feat: LAMOST DR12-14 与子版本体系接入、观测层安全加固与并发异步化
- LAMOST 新增 DR12/13/14 及子版本（v0/v1.0/v1.1/v2.0）维度，Internal 发布标记需登录认证并前端灰显，release×subtype 交叉约束下沉至 capabilities 统一声明 - ObservationFetcher trait 扩展版本/认证/交叉约束能力声明，version 参数贯穿 client→service→API→Agent tool→前端全链路 - 安全：observation cache SQL 全参数绑定 + LIKE 转义、cone_cache_hash 加长度前缀防碰撞、DESI survey/program 白名单防穿越 - 异步化：persist/cached_files_total_size/maybe_persist_tool_result迁移到 tokio::fs；cancelled_runs 与 session_permission_checkers改用 DashMap；auth 读锁优先 + 60s 节流 - Gaia 去 native-tls 改禁用连接池规避 UnexpectedEof，reqwest 移除 native-tls feature - 重构：Source/ProductType from_str 集中解析、download 模块拆分为 try_download_pdf/html、AgentRuntime::init 抽取共享逻辑 - 部署：新增 deploy.sh 一键打包推送脚本、catch-panic 启用
This commit is contained in:
parent
2f1fd19d74
commit
2c8d0b8f8b
135
CLAUDE.md
135
CLAUDE.md
@ -42,6 +42,9 @@ cargo test --lib
|
||||
# Run a specific test
|
||||
cargo test test_name
|
||||
|
||||
# Coverage (requires cargo-llvm-cov)
|
||||
cargo llvm-cov --fail-under-lines 80
|
||||
|
||||
# Frontend (cd dashboard first)
|
||||
npm run dev # HMR dev server on :5173, proxies /api to :8000
|
||||
npm run build # TypeScript check + Vite production build → dashboard/dist/
|
||||
@ -62,45 +65,102 @@ src/
|
||||
├── main.rs # Axum server entry: logging, DB pool, migrations, vec0 table,
|
||||
│ # client/service construction, route registration, AppState assembly
|
||||
├── lib.rs # Config struct + from_env() loading from .env
|
||||
├── api/ # HTTP handlers, AppState, StandardPaper type
|
||||
│ ├── mod.rs # AppState (shared state), StandardPaper, handlers re-exports
|
||||
├── api/ # HTTP handlers, AppState, re-exports via handlers namespace
|
||||
│ ├── mod.rs # AppState (shared state), handler re-exports, permission/event types
|
||||
│ ├── agent.rs # SSE chat_agent endpoint, session CRUD, metrics, audit log
|
||||
│ ├── papers.rs # Search, download, parse, translate, embed, citations, library, export
|
||||
│ ├── catalog.rs # VizieR TAP catalog queries + cone search endpoints
|
||||
│ ├── observation.rs # Observation data search/download (LAMOST/Gaia/SDSS/DESI)
|
||||
│ ├── notes.rs # Highlight/note CRUD
|
||||
│ ├── sync.rs # Meta-sync and asset-batch endpoints
|
||||
│ ├── targets.rs # Target query/associate/extract, RAG chat, figure chat
|
||||
│ └── helpers.rs # Shared DB helpers, format conversion, path validation
|
||||
│ ├── auth.rs # Login/logout/auth-check (token-based, session memory)
|
||||
│ ├── permissions.rs # Agent tool permission rule management
|
||||
│ ├── search.rs # Search history endpoint
|
||||
│ ├── error.rs # Unified API error type (status code + user-safe message)
|
||||
│ └── helpers.rs # Shared DB helpers, format conversion, path validation (private)
|
||||
├── agent/ # ReAct-based research agent (LLM-driven tool-use loop)
|
||||
│ ├── tools/ # AgentTool trait, ToolRegistry, 25+ tool implementations per domain file
|
||||
│ ├── runtime/ # ReAct loop core: session, context, streaming, executor, token_budget,
|
||||
│ │ # permission, permission_profile, checkpoint, circuit_breaker, error_recovery,
|
||||
│ │ # hardline, partitioner, file_cache, system_prompt, finalize, denial_tracker
|
||||
│ ├── tools/ # AgentTool trait, ToolRegistry, tool implementations per domain
|
||||
│ │ ├── mod.rs # AgentTool trait, ToolContext (session_id, sse_tx, silent, etc.)
|
||||
│ │ ├── filesystem/ # read_file, grep_files, write, edit
|
||||
│ │ ├── astro/ # search_papers, download_paper, parse_paper, rag_search,
|
||||
│ │ │ # query_target, save_note, analyze_image, research/*
|
||||
│ │ ├── ask_user.rs # Permission-gated interactive user question
|
||||
│ │ ├── background.rs# bg_task_run, bg_task_check (async slow tasks)
|
||||
│ │ ├── compress.rs # Context compression trigger
|
||||
│ │ ├── memory.rs # save_memory, load_memory
|
||||
│ │ ├── persist.rs # Data persistence (used by memory/compaction)
|
||||
│ │ ├── skill.rs # load_skill
|
||||
│ │ ├── subagent.rs # Context-isolated sub-agent spawner
|
||||
│ │ ├── team.rs # spawn_teammate, send_teammate_message, etc.
|
||||
│ │ └── todo.rs # todo_write (task tracking)
|
||||
│ ├── runtime/ # ReAct loop core
|
||||
│ │ ├── mod.rs # AgentRuntime: session create/resume → context → ReAct → finalize
|
||||
│ │ ├── executor/ # Step executor + helpers (tool dispatch, parallel tool calls)
|
||||
│ │ ├── streaming.rs # SSE AgentStreamEvent types
|
||||
│ │ ├── streaming_executor.rs # SSE-emitting executor wrapper
|
||||
│ │ ├── session.rs # Session CRUD, DB persist, agent_config
|
||||
│ │ ├── token_budget.rs # Soft/hard token limits, tiktoken counting
|
||||
│ │ ├── permission.rs # Deny > Allow > Ask rule chain, pattern matching
|
||||
│ │ ├── permission_profile.rs # Named profiles (research, readonly)
|
||||
│ │ ├── checkpoint.rs # Rewind/retry state snapshots
|
||||
│ │ ├── circuit_breaker.rs# CompactionCircuitBreaker (cooldown rate limiting)
|
||||
│ │ ├── error_recovery.rs # Error classification + retry strategies
|
||||
│ │ ├── hardline.rs # Terminal conditions (too many denials, loops, etc.)
|
||||
│ │ ├── partitioner.rs # Message window partitioning for compaction
|
||||
│ │ ├── file_cache.rs # ReadFileState — file content dedup cache
|
||||
│ │ ├── system_prompt.rs # System prompt assembly (mode, skills, memory, tools)
|
||||
│ │ ├── finalize.rs # Post-loop cleanup: memory extraction, audit log
|
||||
│ │ └── denial_tracker.rs # Consecutive denial detection
|
||||
│ ├── compact/ # Context compression (micro/auto/manual layers + collapse)
|
||||
│ ├── memory/ # Persistent memory manager: extraction, dedup, decay, age, guardrails
|
||||
│ ├── hooks/ # Lifecycle hooks: registry, dispatch, builtins, matcher, traits (PreToolUse/PostToolUse/Stop/etc.)
|
||||
│ ├── modes/ # Agent session modes: default, deep-research, literature-reader (identity/tools/config presets)
|
||||
│ ├── hooks/ # Lifecycle hooks: registry, dispatch, builtins, matcher, traits
|
||||
│ ├── modes/ # Agent session modes (see Agent Session Modes section below)
|
||||
│ ├── skills.rs # SkillRegistry: hot-loads SKILL.md files from skills/ directory
|
||||
│ ├── subagent.rs # Context-isolated sub-agent runner (subagent tool)
|
||||
│ ├── subagent.rs # Context-isolated sub-agent runner
|
||||
│ ├── team/ # Multi-agent team: file-based inbox, lead/teammate coordination
|
||||
│ ├── background.rs# BgNotificationQueue for async slow-task (download/parse) notifications
|
||||
│ ├── background.rs# BgNotificationQueue for async slow-task notifications
|
||||
│ ├── task_board.rs# Persistent task board (agent_tasks table)
|
||||
│ ├── trajectory.rs# Session trajectory recording for audit/debug
|
||||
│ └── terminal.rs # Escape sequence filter for ANSI-heavy tool outputs
|
||||
├── clients/ # External API wrappers
|
||||
│ ├── llm.rs # LlmClient (OpenAI-compatible chat + streaming), EmbeddingClient
|
||||
├── clients/ # External API wrappers (pure communication, no business logic)
|
||||
│ ├── llm/ # LlmClient (OpenAI-compatible chat + streaming), EmbeddingClient
|
||||
│ ├── ads.rs # NASA ADS API
|
||||
│ ├── arxiv.rs # arXiv Atom XML API
|
||||
│ ├── cds/ # CDS services (sesame.rs — Sesame name resolver; vizier.rs — TAP queries)
|
||||
│ ├── gaia/ # Gaia archive (TAP + DataLink)
|
||||
│ ├── lamost/ # LAMOST archive (ConeSearch + FITS.gz download)
|
||||
│ ├── sdss/ # SDSS archive (Data Lab TAP + SAS bulk download)
|
||||
│ ├── desi/ # DESI archive (Data Lab TAP + HEALPix coadd SAS)
|
||||
│ ├── vo/ # Virtual Observatory generic client
|
||||
│ └── qiniu.rs # Qiniu cloud storage
|
||||
├── services/ # Business logic
|
||||
│ ├── search.rs # Unified cross-source search (ADS + arXiv dedup)
|
||||
│ ├── download.rs # PDF/HTML download with anti-bot measures and fallback chain
|
||||
│ ├── parser/ # HTML/PDF → Markdown parsers (A&A, IOP, ar5iv, generic, PDF via MinerU)
|
||||
│ ├── translation.rs# LLM bilingual translation with Trie-based astronomy glossary
|
||||
│ ├── paper/ # Paper lifecycle: model, db (CRUD), ingest (ADS/arXiv→DB), reader (markdown)
|
||||
│ ├── translation.rs # LLM bilingual translation with Trie-based astronomy glossary
|
||||
│ ├── rag.rs # Embedding ingest + vector similarity retrieval + LLM answer generation
|
||||
│ ├── target.rs # Celestial target extraction (IAU name regex) + CDS Sesame lookup
|
||||
│ ├── chunker.rs # Markdown text chunking for embedding
|
||||
│ ├── cds/ # CDS business layer: target.rs (Sesame name resolver), vizier.rs (TAP + cache)
|
||||
│ ├── observation/ # Cross-source observation data (spectra, light curves, photometry, images)
|
||||
│ │ ├── mod.rs # 双轴正交架构: Source × ProductType → ObservationFetcher → Registry
|
||||
│ │ ├── types.rs # Source/ProductType/ProductSpec/Artifact/ObservationProduct/Candidate
|
||||
│ │ ├── fetcher.rs # ObservationFetcher trait + cone search cache template methods
|
||||
│ │ ├── registry.rs # ObservationRegistry (fetcher registration, capability discovery)
|
||||
│ │ ├── dispatch.rs # search_observation + download_observation (unified entry points)
|
||||
│ │ ├── cache.rs # observation_cache table read/write + file staging
|
||||
│ │ ├── lamost.rs # LamostSpectrumFetcher
|
||||
│ │ ├── gaia.rs # GaiaSpectrumFetcher + GaiaLightCurveFetcher
|
||||
│ │ ├── sdss.rs # SdssSpectrumFetcher + SdssApogeeFetcher
|
||||
│ │ └── desi.rs # DesiSpectrumFetcher
|
||||
│ ├── batch/ # Meta-sync (ADS bulk harvest) and asset-batch processing engines
|
||||
│ ├── query_parser.rs# Advanced search query syntax parser
|
||||
│ ├── citation.rs # Citation network graph data (references + citations)
|
||||
│ ├── session.rs # Agent session listing, metrics, audit log (for API consumption)
|
||||
│ ├── vision.rs # Image analysis via dedicated vision model
|
||||
│ ├── note.rs # Highlight/note storage and retrieval
|
||||
│ ├── pipeline.rs # Synchronous paper processing pipeline (download→parse→translate→embed)
|
||||
│ ├── chunker.rs # Markdown text chunking for embedding
|
||||
│ ├── query_parser.rs # Advanced search query syntax parser
|
||||
│ └── logging.rs # Pretty console + rolling file logger
|
||||
└── bin/
|
||||
├── health_check.rs # Library consistency checker and auto-repair
|
||||
@ -113,11 +173,18 @@ src/
|
||||
All handlers access state via `Arc<AppState>`. Key fields:
|
||||
|
||||
- `db: SqlitePool` — SQLite connection pool (5 max connections, foreign keys enforced)
|
||||
- `llm: LlmClient` / `embedding: EmbeddingClient` — OpenAI-compatible LLM clients
|
||||
- `llm: LlmClient` / `medium_llm` / `fast_llm` — OpenAI-compatible LLM clients (multi-tier)
|
||||
- `vision_llm: Option<LlmClient>` — dedicated vision model client
|
||||
- `embedding: EmbeddingClient` — embedding model client
|
||||
- `ads: AdsClient` / `arxiv: ArxivClient` — academic search clients
|
||||
- `vizier: VizierClient` — CDS VizieR TAP catalog query client
|
||||
- `lamost` / `gaia` / `sdss` / `desi` — observation data source clients
|
||||
- `observation_registry: Arc<ObservationRegistry>` — cross-source observation fetcher registry
|
||||
- `skill_registry: Arc<RwLock<SkillRegistry>>` — hot-reloaded agent skills
|
||||
- `memory_manager: Arc<MemoryManager>` — persistent agent memory (MEMORY.md + decay)
|
||||
- `cancelled_runs: Arc<Mutex<HashSet<String>>>` — agent cancellation tokens
|
||||
- `cancelled_runs: Arc<DashMap<String, ()>>` — agent cancellation tokens
|
||||
- `session_permission_checkers: Arc<DashMap<String, PermissionChecker>>` — per-session permission rules
|
||||
- `pending_questions` / `pending_permissions` — interactive agent-user handshake channels
|
||||
- `harvest_status` / `batch_status` — async batch operation status tracking
|
||||
|
||||
### Agent System Design
|
||||
@ -131,7 +198,7 @@ The agent (`src/agent/`) implements a **ReAct** (Thought → Action → Observat
|
||||
5. **Tool call ID tracking**: LLM may not return tool_call IDs — `LlmClient` generates UUID fallbacks. `ToolCall` and `ToolResult` SSE events carry matching IDs for precise frontend pairing.
|
||||
6. **ToolContext** (`tools/mod.rs`): Injected into every tool execution — holds `app_state`, `session_id`, `sse_tx` (for intermediate events), `enable_thinking`, `read_file_state` (file cache for dedup), `silent` (sub-agents skip permission prompts).
|
||||
7. **Context compression**: Four layers — micro (placeholder replacement), snip (old-message truncation), auto (LLM summarization), aggro_micro (aggressive placeholder). Protected by `CompactionCircuitBreaker`. Transcripts persisted in `agent_messages` table, not filesystem snapshots.
|
||||
8. **Skills** (`skills.rs`): Two-layer loading — system-reminder lists names (~20 tokens each), LLM calls `load_skill` to inject full SKILL.md content
|
||||
8. **Skills** (`skills.rs`): Two-layer loading — system-reminder lists names (~20 tokens each), LLM calls `load_skill` to inject full SKILL.md content. Skills are organized into subdirectories: `methodology/`, `plotting/`, `presentation/`.
|
||||
9. **Sub-agents** (`subagent.rs`): `subagent` tool spawns a context-isolated sub-agent with its own ReAct loop. Sub-agent messages (system/user/assistant/tool) are persisted to `agent_messages` with `agent_name` identifier. Returns final summary + activity log. SSE progress forwarded to parent via ToolContext.
|
||||
10. **Memory** (`memory/`): File-based persistent memory (MEMORY.md). `MemoryManager` handles extraction from conversation, dedup, recency decay, age-based pruning, and guardrails. Tools: `save_memory`, `load_memory` (auto-injected in system prompt).
|
||||
11. **Teams** (`team/`): File-based inbox directory per session for lead/teammate message passing
|
||||
@ -141,17 +208,29 @@ Environment variables for agent tuning: `AGENT_TOKEN_SOFT_LIMIT` (default 80000)
|
||||
|
||||
### Database
|
||||
|
||||
SQLite via `sqlx::sqlite`. Migrations in `migrations/` are auto-run on startup (`sqlx::migrate!("./migrations")`). Key tables: `papers`, `citations_references`, `notes`, `agent_sessions`, `agent_messages`, `agent_tasks`, `agent_audit_log`, `paper_chunks_content`. Vector embeddings use `sqlite-vec` (`vec_paper_chunks` virtual table, auto-registered before any DB connection).
|
||||
SQLite via `sqlx::sqlite`. Migrations in `migrations/` are auto-run on startup (`sqlx::migrate!("./migrations")`). Key tables: `papers`, `citations_references`, `notes`, `agent_sessions`, `agent_messages`, `agent_tasks`, `agent_audit_log`, `paper_chunks_content`, `vizier_cache`, `observation_cache`. Vector embeddings use `sqlite-vec` (`vec_paper_chunks` virtual table, auto-registered before any DB connection).
|
||||
|
||||
The embedding dimension is controlled by `EMBEDDING_DIM` env var (default 1536). On dimension mismatch, the vec table and chunk content are dropped and recreated.
|
||||
|
||||
### Observation Data System (`src/services/observation/`)
|
||||
|
||||
A cross-source astronomical observation data subsystem with a **dual-axis orthogonal** architecture: **Source** (LAMOST/Gaia/SDSS/DESI) × **ProductType** (Spectrum/LightCurve/Photometry/Image). Each valid combination implements an `ObservationFetcher`, registered in `ObservationRegistry`.
|
||||
|
||||
New fetchers follow OCP (pure addition, no modification to existing code):
|
||||
|
||||
1. Create `{source}.rs`, implement `ObservationFetcher` trait
|
||||
2. Add `register!(YourFetcher)` in `registry.rs::default()`
|
||||
3. Add new variants to `Source`/`ProductType` enums in `types.rs` if needed
|
||||
|
||||
External callers use `dispatch::search_observation()` (metadata only) and `dispatch::download_observation()` (metadata + file download) via `ObservationRequest`. Results are cached in `observation_cache` table with file staging to `library/`.
|
||||
|
||||
### Frontend (dashboard/)
|
||||
|
||||
React 19 + TypeScript + Vite + Tailwind CSS 4. Organized by technical layer (Type-Based):
|
||||
|
||||
- `pages/` — Page-level panel view components (SearchPanel, LibraryPanel, ReaderPanel, CitationPanel, SyncPanel, ResearchAgentPanel, SettingsPanel)
|
||||
- `pages/` — Page-level panel view components (SearchPanel, LibraryPanel, ReaderPanel, CitationPanel, SyncPanel, ResearchAgentPanel, ObservationPanel, SettingsPanel)
|
||||
- `components/` — Reusable and layout components (sub-folders: agent, reader, sync, layout, dialogs)
|
||||
- `hooks/` — Global and feature-specific custom stateful React Hooks (e.g., useLibrary, useSearch, useNotes)
|
||||
- `hooks/` — Global and feature-specific custom stateful React Hooks (e.g., useLibrary, useSearch, useNotes, useObservation)
|
||||
- `types/` — Global TypeScript type definitions (types/index.ts)
|
||||
- `utils/` — Common utility helper functions
|
||||
- `assets/` — Static assets and global stylesheet styles
|
||||
@ -160,7 +239,7 @@ Dependencies: `react-markdown` + `rehype-katex` + `remark-math` for Markdown/LaT
|
||||
|
||||
### Obscura In-Process Browser
|
||||
|
||||
The `obscura-inprocess` feature compiles `obscura-browser` and `obscura-net` directly into the binary, eliminating the need for external browser binaries. This is used for bypassing Cloudflare/WAF on PDF download. Enabled via `--features obscura-inprocess`.
|
||||
The `obscura-inprocess` feature compiles `obscura-browser` and `obscura-net` from `libs/obscura/` directly into the binary, eliminating the need for external browser binaries. This is used for bypassing Cloudflare/WAF on PDF download. Enabled via `--features obscura-inprocess`.
|
||||
|
||||
### Astronomy Glossary (dictionary.txt)
|
||||
|
||||
@ -235,6 +314,10 @@ Controlled by env vars:
|
||||
- `EXTRACT_MEMORY_THROTTLE_TURNS` (default `3`) — extract every N turns
|
||||
- `EXTRACT_MEMORY_MAX_STEPS` (default `3`) — sub-agent max steps for extraction
|
||||
|
||||
### Deployment (`deploy.sh`)
|
||||
|
||||
The `deploy.sh` script builds the project locally (release-min profile, Composer-based) and deploys to a remote server via rsync + SSH. It handles: source compilation, frontend build, remote directory sync, service restart via docker compose, and health check verification.
|
||||
|
||||
## Code Conventions
|
||||
|
||||
- Use `anyhow` for application errors, `thiserror` for library-style typed errors
|
||||
@ -243,3 +326,7 @@ Controlled by env vars:
|
||||
- API handlers take `State(Arc<AppState>)` and return Axum-compatible responses
|
||||
- Agent tools implement `AgentTool` trait; new tools register in `ToolRegistry::new()`
|
||||
- Front-end build is triggered by `build.rs` (auto npm install + build when `dashboard/src/` changes)
|
||||
- New observation fetchers follow OCP: add module + register — don't modify existing fetchers or dispatch
|
||||
- External API clients (`src/clients/`) are pure communication; business logic lives in `src/services/`
|
||||
- Route registration happens in `src/main.rs`; handler re-exports are maintained in `src/api/mod.rs` → `handlers` namespace
|
||||
- Agent hooks (`src/agent/hooks/`) implement lifecycles: PreToolUse, PostToolUse, Stop, etc. — register in `builtins.rs`
|
||||
|
||||
168
Cargo.lock
generated
168
Cargo.lock
generated
@ -138,6 +138,7 @@ dependencies = [
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"clap",
|
||||
"dashmap",
|
||||
"dotenvy",
|
||||
"flate2",
|
||||
"futures-util",
|
||||
@ -481,7 +482,7 @@ checksum = "2c5e60b8c8d282c86360cab651ded04ab0335a7b5390c8d34145cbeab8cacf5f"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"btls-sys",
|
||||
"foreign-types 0.5.0",
|
||||
"foreign-types",
|
||||
"libc",
|
||||
"openssl-macros",
|
||||
]
|
||||
@ -761,16 +762,6 @@ dependencies = [
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation-sys"
|
||||
version = "0.8.7"
|
||||
@ -900,6 +891,20 @@ dependencies = [
|
||||
"cmov",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dashmap"
|
||||
version = "6.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"hashbrown 0.14.5",
|
||||
"lock_api",
|
||||
"once_cell",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "data-encoding"
|
||||
version = "2.11.0"
|
||||
@ -1242,15 +1247,6 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
|
||||
dependencies = [
|
||||
"foreign-types-shared 0.1.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types"
|
||||
version = "0.5.0"
|
||||
@ -1258,7 +1254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965"
|
||||
dependencies = [
|
||||
"foreign-types-macros",
|
||||
"foreign-types-shared 0.3.1",
|
||||
"foreign-types-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1272,12 +1268,6 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types-shared"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types-shared"
|
||||
version = "0.3.1"
|
||||
@ -1760,29 +1750,13 @@ dependencies = [
|
||||
"http",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"rustls 0.23.40",
|
||||
"rustls 0.23.41",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tower-service",
|
||||
"webpki-roots 1.0.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-tls"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"native-tls",
|
||||
"tokio",
|
||||
"tokio-native-tls",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-util"
|
||||
version = "0.1.20"
|
||||
@ -2419,23 +2393,6 @@ dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "native-tls"
|
||||
version = "0.2.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"openssl",
|
||||
"openssl-probe",
|
||||
"openssl-sys",
|
||||
"schannel",
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "new_debug_unreachable"
|
||||
version = "1.0.6"
|
||||
@ -2624,20 +2581,6 @@ version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.80"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"cfg-if",
|
||||
"foreign-types 0.3.2",
|
||||
"libc",
|
||||
"openssl-macros",
|
||||
"openssl-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-macros"
|
||||
version = "0.1.1"
|
||||
@ -2649,24 +2592,6 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-probe"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
||||
|
||||
[[package]]
|
||||
name = "openssl-sys"
|
||||
version = "0.9.117"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "outref"
|
||||
version = "0.5.2"
|
||||
@ -2954,7 +2879,7 @@ dependencies = [
|
||||
"quinn-proto",
|
||||
"quinn-udp",
|
||||
"rustc-hash",
|
||||
"rustls 0.23.40",
|
||||
"rustls 0.23.41",
|
||||
"socket2",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
@ -2974,7 +2899,7 @@ dependencies = [
|
||||
"rand 0.9.4",
|
||||
"ring",
|
||||
"rustc-hash",
|
||||
"rustls 0.23.40",
|
||||
"rustls 0.23.41",
|
||||
"rustls-pki-types",
|
||||
"slab",
|
||||
"thiserror 2.0.18",
|
||||
@ -3147,23 +3072,20 @@ dependencies = [
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"hyper-tls",
|
||||
"hyper-util",
|
||||
"js-sys",
|
||||
"log",
|
||||
"mime_guess",
|
||||
"native-tls",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"quinn",
|
||||
"rustls 0.23.40",
|
||||
"rustls 0.23.41",
|
||||
"rustls-pki-types",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-native-tls",
|
||||
"tokio-rustls",
|
||||
"tokio-util",
|
||||
"tower",
|
||||
@ -3256,9 +3178,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.40"
|
||||
version = "0.23.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
|
||||
checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"ring",
|
||||
@ -3329,15 +3251,6 @@ dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schannel"
|
||||
version = "0.1.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
@ -3354,29 +3267,6 @@ dependencies = [
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "3.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"core-foundation",
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
"security-framework-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "security-framework-sys"
|
||||
version = "2.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "selectors"
|
||||
version = "0.26.0"
|
||||
@ -4210,23 +4100,13 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-native-tls"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
|
||||
dependencies = [
|
||||
"native-tls",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-rustls"
|
||||
version = "0.26.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
|
||||
dependencies = [
|
||||
"rustls 0.23.40",
|
||||
"rustls 0.23.41",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
|
||||
@ -21,11 +21,11 @@ path = "src/bin/cli.rs"
|
||||
[dependencies]
|
||||
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "signal", "time", "fs", "net", "sync", "process"] }
|
||||
axum = { version = "0.7", features = ["macros", "multipart"] }
|
||||
tower-http = { version = "0.5", features = ["cors", "fs", "trace", "set-header"] }
|
||||
tower-http = { version = "0.5", features = ["cors", "fs", "trace", "set-header", "catch-panic"] }
|
||||
sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "sqlite", "chrono", "json"] }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "multipart", "cookies", "rustls-tls", "native-tls"] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "multipart", "cookies", "rustls-tls"] }
|
||||
dotenvy = "0.15"
|
||||
quick-xml = { version = "0.31", features = ["serialize"] }
|
||||
anyhow = "1.0"
|
||||
@ -62,6 +62,7 @@ glob = "0.3"
|
||||
walkdir = "2"
|
||||
lru = "0.12"
|
||||
git2 = { version = "0.18", default-features = false, features = ["vendored-libgit2"] }
|
||||
dashmap = "6"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="referrer" content="no-referrer" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<link rel="manifest" href="/manifest.json" />
|
||||
<!-- iOS PWA 独立全屏支持 -->
|
||||
|
||||
@ -61,7 +61,13 @@ export default function App() {
|
||||
|
||||
// 2. 局部状态定义 (多组件共用或全局网络进度状态)
|
||||
const [activeTab, setActiveTab] = useState<
|
||||
'search' | 'library' | 'reader' | 'citation' | 'sync' | 'observation' | 'agent'
|
||||
| 'search'
|
||||
| 'library'
|
||||
| 'reader'
|
||||
| 'citation'
|
||||
| 'sync'
|
||||
| 'observation'
|
||||
| 'agent'
|
||||
>(() => {
|
||||
const saved = localStorage.getItem('astro_active_tab');
|
||||
const validTabs = [
|
||||
|
||||
@ -642,7 +642,9 @@ export function VizierResultCard({ metadata }: VizierResultCardProps) {
|
||||
if (val === null || val === undefined) return '—';
|
||||
if (typeof val === 'number') {
|
||||
// 数字保留合理精度
|
||||
return Number.isInteger(val) ? String(val) : val.toFixed(6).replace(/\.?0+$/, '');
|
||||
return Number.isInteger(val)
|
||||
? String(val)
|
||||
: val.toFixed(6).replace(/\.?0+$/, '');
|
||||
}
|
||||
if (typeof val === 'boolean') return val ? 'true' : 'false';
|
||||
return String(val);
|
||||
@ -703,11 +705,17 @@ export function VizierResultCard({ metadata }: VizierResultCardProps) {
|
||||
<th
|
||||
key={i}
|
||||
className="px-2 py-1.5 text-left font-bold text-slate-600 border-b border-slate-200 whitespace-nowrap"
|
||||
title={f.description || f.unit ? `${f.description || ''} ${f.unit ? `[${f.unit}]` : ''}`.trim() : undefined}
|
||||
title={
|
||||
f.description || f.unit
|
||||
? `${f.description || ''} ${f.unit ? `[${f.unit}]` : ''}`.trim()
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
{f.name}
|
||||
{f.unit && (
|
||||
<span className="text-slate-400 font-normal ml-1">[{f.unit}]</span>
|
||||
<span className="text-slate-400 font-normal ml-1">
|
||||
[{f.unit}]
|
||||
</span>
|
||||
)}
|
||||
</th>
|
||||
))}
|
||||
@ -744,7 +752,6 @@ export function VizierResultCard({ metadata }: VizierResultCardProps) {
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
// ==========================================
|
||||
// 6. 统一观测数据下载结果卡片 (find_observation)
|
||||
// 跨 (LAMOST/Gaia/SDSS/DESI) × (Spectrum/LightCurve/Photometry/Image)
|
||||
|
||||
@ -108,7 +108,12 @@ export function ToolCallCard({
|
||||
metadata={
|
||||
metadata as {
|
||||
table_name?: string;
|
||||
fields: { name: string; description?: string; unit?: string; datatype?: string }[];
|
||||
fields: {
|
||||
name: string;
|
||||
description?: string;
|
||||
unit?: string;
|
||||
datatype?: string;
|
||||
}[];
|
||||
rows: unknown[][];
|
||||
row_count: number;
|
||||
truncated: boolean;
|
||||
|
||||
@ -7,12 +7,7 @@
|
||||
// 2. （可选）SpecialToolRenderers 的 FindObservationCard 可改为薄封装调用本组件
|
||||
//
|
||||
// 视觉与 SpecialToolRenderers::FindObservationCard 保持一致，统一数据源主题色。
|
||||
import {
|
||||
Activity,
|
||||
CheckCircle2,
|
||||
AlertTriangle,
|
||||
Download,
|
||||
} from 'lucide-react';
|
||||
import { Activity, CheckCircle2, AlertTriangle, Download } from 'lucide-react';
|
||||
import {
|
||||
SOURCE_THEME,
|
||||
PRODUCT_LABEL,
|
||||
@ -119,8 +114,7 @@ export function ObservationResultCard({ result }: ObservationResultCardProps) {
|
||||
{a.band}
|
||||
</span>
|
||||
)}
|
||||
{a.file_format.toUpperCase()} ·{' '}
|
||||
{formatFileSize(a.size_bytes)}
|
||||
{a.file_format.toUpperCase()} · {formatFileSize(a.size_bytes)}
|
||||
</span>
|
||||
<a
|
||||
href={a.file_url}
|
||||
|
||||
@ -12,11 +12,7 @@
|
||||
import { useState, useEffect, useCallback, useRef, useMemo } from 'react';
|
||||
import axios from 'axios';
|
||||
import { extractErrorMessage } from '../utils/apiError';
|
||||
import type {
|
||||
ObservationRecord,
|
||||
Candidate,
|
||||
CapabilitySpec,
|
||||
} from '../types';
|
||||
import type { ObservationRecord, Candidate, CapabilitySpec } from '../types';
|
||||
import type { ObservationBatchResult } from '../components/observation/constants';
|
||||
|
||||
interface UseObservationProps {
|
||||
@ -32,6 +28,8 @@ export interface SearchForm {
|
||||
dec: string;
|
||||
radius: string;
|
||||
release: string;
|
||||
/// 数据发布的子版本（仅 LAMOST 有意义）；空串表示用该 DR 的默认子版本
|
||||
version: string;
|
||||
}
|
||||
|
||||
export const EMPTY_SEARCH_FORM: SearchForm = {
|
||||
@ -42,6 +40,7 @@ export const EMPTY_SEARCH_FORM: SearchForm = {
|
||||
dec: '',
|
||||
radius: '0.1',
|
||||
release: '',
|
||||
version: '',
|
||||
};
|
||||
|
||||
export function useObservation({ isAuthenticated }: UseObservationProps) {
|
||||
@ -65,7 +64,9 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (isAuthenticated === true) fetchCapabilities();
|
||||
if (isAuthenticated !== true) return;
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect -- async fetch, setState in microtask
|
||||
fetchCapabilities();
|
||||
}, [isAuthenticated, fetchCapabilities]);
|
||||
|
||||
// ════════════════════════════════════════════════════
|
||||
@ -88,6 +89,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
||||
const valid = currentSpec.releases;
|
||||
const hardMax = currentSpec.hard_max_radius_deg;
|
||||
const suggested = currentSpec.suggested_max_radius_deg;
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect -- derived form sync on spec change
|
||||
setSearchForm((f) => {
|
||||
let next = f;
|
||||
// release 回落（仅在该源区分版本时）
|
||||
@ -103,6 +105,27 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
||||
});
|
||||
}, [currentSpec]);
|
||||
|
||||
// 当 release 切换后：若该 release 有子版本列表，则校验当前 version 是否在列表里，
|
||||
// 不在则清空（清空后端用默认子版本）。无子版本概念的源（release_versions 为空）也清空。
|
||||
useEffect(() => {
|
||||
if (!currentSpec) return;
|
||||
const versionsMap = currentSpec.release_versions ?? {};
|
||||
const versions = searchForm.release
|
||||
? versionsMap[searchForm.release]
|
||||
: undefined;
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect -- derived version reset on release change
|
||||
setSearchForm((f) => {
|
||||
if (!versions || versions.length === 0) {
|
||||
// 该 release 无子版本概念
|
||||
return f.version ? { ...f, version: '' } : f;
|
||||
}
|
||||
if (!f.version || !versions.includes(f.version)) {
|
||||
return { ...f, version: '' }; // 清空 = 用默认（最新公开）子版本
|
||||
}
|
||||
return f;
|
||||
});
|
||||
}, [currentSpec, searchForm.release]);
|
||||
|
||||
const [searchResults, setSearchResults] = useState<Candidate[]>([]);
|
||||
const [searching, setSearching] = useState(false);
|
||||
const [searchError, setSearchError] = useState<string | null>(null);
|
||||
@ -131,6 +154,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
||||
dec,
|
||||
radius: Math.max(0.0001, parseFloat(searchForm.radius) || 0.1),
|
||||
release: searchForm.release || undefined,
|
||||
version: searchForm.version || undefined,
|
||||
},
|
||||
});
|
||||
setSearchResults(res.data ?? []);
|
||||
@ -169,13 +193,6 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
||||
useState<ObservationBatchResult | null>(null);
|
||||
const [downloadError, setDownloadError] = useState<string | null>(null);
|
||||
|
||||
/** 下载选中的候选源（标识符模式） */
|
||||
const downloadSelected = useCallback(async () => {
|
||||
const ids = Array.from(selectedSourceIds);
|
||||
if (ids.length === 0) return;
|
||||
await downloadByIds(ids);
|
||||
}, [selectedSourceIds]); // eslint-disable-line react-hooks/exhaustive-deps
|
||||
|
||||
/** 按标识符列表下载 */
|
||||
const downloadByIds = useCallback(
|
||||
async (sourceIds: string[]) => {
|
||||
@ -190,6 +207,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
||||
product: searchForm.product,
|
||||
subtype: searchForm.subtype || undefined,
|
||||
release: searchForm.release || undefined,
|
||||
version: searchForm.version || undefined,
|
||||
force: false,
|
||||
mode: 'identifiers',
|
||||
source_ids: sourceIds,
|
||||
@ -205,6 +223,13 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
||||
[searchForm]
|
||||
);
|
||||
|
||||
/** 下载选中的候选源（标识符模式） */
|
||||
const downloadSelected = useCallback(async () => {
|
||||
const ids = Array.from(selectedSourceIds);
|
||||
if (ids.length === 0) return;
|
||||
await downloadByIds(ids);
|
||||
}, [selectedSourceIds, downloadByIds]);
|
||||
|
||||
/** 按坐标下载（cone 检索 + 全部下载） */
|
||||
const downloadByCoordinates = useCallback(
|
||||
async (strategy: 'nearest' | 'all' = 'nearest') => {
|
||||
@ -224,6 +249,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
||||
product: searchForm.product,
|
||||
subtype: searchForm.subtype || undefined,
|
||||
release: searchForm.release || undefined,
|
||||
version: searchForm.version || undefined,
|
||||
force: false,
|
||||
mode: 'coordinates',
|
||||
ra,
|
||||
@ -265,9 +291,10 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
||||
setLibraryLoading(true);
|
||||
setLibraryError(null);
|
||||
try {
|
||||
const res = await axios.get<{ items: ObservationRecord[]; total: number }>(
|
||||
'/api/observation/list',
|
||||
{
|
||||
const res = await axios.get<{
|
||||
items: ObservationRecord[];
|
||||
total: number;
|
||||
}>('/api/observation/list', {
|
||||
params: {
|
||||
source: libSource !== 'all' ? libSource : undefined,
|
||||
product: libProduct !== 'all' ? libProduct : undefined,
|
||||
@ -276,14 +303,11 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
||||
limit: libPageSize,
|
||||
offset: (libPage - 1) * libPageSize,
|
||||
},
|
||||
}
|
||||
);
|
||||
});
|
||||
setLibraryItems(res.data.items ?? []);
|
||||
setLibraryTotal(res.data.total ?? 0);
|
||||
} catch (e: unknown) {
|
||||
setLibraryError(
|
||||
extractErrorMessage(e, '加载缓存库失败，请检查后端连接')
|
||||
);
|
||||
setLibraryError(extractErrorMessage(e, '加载缓存库失败，请检查后端连接'));
|
||||
setLibraryItems([]);
|
||||
} finally {
|
||||
setLibraryLoading(false);
|
||||
@ -300,6 +324,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
||||
|
||||
// 任意筛选/分页变更 → 重新请求后端
|
||||
useEffect(() => {
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect -- async fetch, setState in microtask
|
||||
fetchLibrary();
|
||||
}, [fetchLibrary]);
|
||||
|
||||
|
||||
@ -25,6 +25,7 @@ import {
|
||||
Database,
|
||||
Send,
|
||||
RotateCw,
|
||||
Lock,
|
||||
} from 'lucide-react';
|
||||
import { CustomSelect } from '../components/CustomSelect';
|
||||
import { ObservationResultCard } from '../components/observation/ObservationResultCard';
|
||||
@ -56,11 +57,13 @@ interface ObservationPanelProps {
|
||||
selectNone: () => void;
|
||||
// 下载
|
||||
downloading: boolean;
|
||||
downloadResult: import('../components/observation/constants').ObservationBatchResult | null;
|
||||
downloadResult:
|
||||
import('../components/observation/constants').ObservationBatchResult | null;
|
||||
downloadError: string | null;
|
||||
setDownloadResult: React.Dispatch<
|
||||
React.SetStateAction<
|
||||
import('../components/observation/constants').ObservationBatchResult | null
|
||||
| import('../components/observation/constants').ObservationBatchResult
|
||||
| null
|
||||
>
|
||||
>;
|
||||
downloadSelected: () => Promise<void>;
|
||||
@ -195,8 +198,7 @@ function SearchView({
|
||||
|
||||
const subtypeOptions = useMemo(() => {
|
||||
const cap = capabilities.find(
|
||||
(c) =>
|
||||
c.source === searchForm.source && c.product === searchForm.product
|
||||
(c) => c.source === searchForm.source && c.product === searchForm.product
|
||||
);
|
||||
return (cap?.subtypes ?? []).map((s) => ({ value: s, label: s }));
|
||||
}, [capabilities, searchForm.source, searchForm.product]);
|
||||
@ -208,22 +210,42 @@ function SearchView({
|
||||
|
||||
// 当前源+产品支持的版本列表（供下拉）
|
||||
const releaseOptions = useMemo(() => {
|
||||
const authRequired = new Set(currentSpec?.releases_requiring_auth ?? []);
|
||||
return (currentSpec?.releases ?? []).map((r) => ({
|
||||
value: r,
|
||||
label: r.toUpperCase(),
|
||||
label: authRequired.has(r)
|
||||
? `${r.toUpperCase()} · 需登录`
|
||||
: r.toUpperCase(),
|
||||
}));
|
||||
}, [currentSpec]);
|
||||
|
||||
// LAMOST MRS 联动约束：MRS（中分辨率）从 DR7 起才支持，DR5/DR6 无 MRS 数据
|
||||
const lamostMrsBlocked = useMemo(() => {
|
||||
if (searchForm.source !== 'lamost' || searchForm.subtype !== 'mrs')
|
||||
return null;
|
||||
const rel = searchForm.release;
|
||||
if (rel === 'dr5' || rel === 'dr6') {
|
||||
return 'LAMOST 中分辨率（MRS）从 DR7 起才支持，当前版本无 MRS 数据';
|
||||
// 当前选中的 release 是否需要登录（Internal DR）
|
||||
const currentReleaseNeedsAuth = useMemo(() => {
|
||||
if (!searchForm.release || !currentSpec) return false;
|
||||
return (currentSpec.releases_requiring_auth ?? []).includes(
|
||||
searchForm.release
|
||||
);
|
||||
}, [searchForm.release, currentSpec]);
|
||||
|
||||
// 交叉约束：当前 release 是否支持当前 subtype（从 capabilities 派生，不硬编码）
|
||||
const subtypeBlocked = useMemo(() => {
|
||||
if (!currentSpec || !searchForm.release || !searchForm.subtype) return null;
|
||||
const map = currentSpec.subtypes_per_release;
|
||||
if (!map || Object.keys(map).length === 0) return null; // 无交叉约束
|
||||
const valid = map[searchForm.release];
|
||||
if (valid && !valid.includes(searchForm.subtype)) {
|
||||
return `'${searchForm.release.toUpperCase()}' 不支持子类型 '${searchForm.subtype}'`;
|
||||
}
|
||||
return null;
|
||||
}, [searchForm.source, searchForm.subtype, searchForm.release]);
|
||||
}, [currentSpec, searchForm.release, searchForm.subtype]);
|
||||
|
||||
// 当前 release 对应的子版本列表（仅 LAMOST 有内容）
|
||||
const versionOptions = useMemo(() => {
|
||||
if (!currentSpec || !searchForm.release) return [];
|
||||
const versionsMap = currentSpec.release_versions ?? {};
|
||||
const versions = versionsMap[searchForm.release] ?? [];
|
||||
return versions.map((v) => ({ value: v, label: v.toUpperCase() }));
|
||||
}, [currentSpec, searchForm.release]);
|
||||
|
||||
const handleSubmitIds = () => {
|
||||
const ids = idsText
|
||||
@ -275,7 +297,7 @@ function SearchView({
|
||||
.map((c) => c.product);
|
||||
const safeProduct = newProducts.includes(f.product)
|
||||
? f.product
|
||||
: newProducts[0] ?? f.product;
|
||||
: (newProducts[0] ?? f.product);
|
||||
return { ...f, source: v, product: safeProduct, subtype: '' };
|
||||
});
|
||||
}}
|
||||
@ -331,27 +353,67 @@ function SearchView({
|
||||
}
|
||||
className="w-full"
|
||||
options={[
|
||||
{ value: '__none__', label: `默认${currentSpec?.default_release ? `（${currentSpec.default_release.toUpperCase()}）` : ''}` },
|
||||
{
|
||||
value: '__none__',
|
||||
label: `默认${currentSpec?.default_release ? `（${currentSpec.default_release.toUpperCase()}）` : ''}`,
|
||||
},
|
||||
...releaseOptions,
|
||||
]}
|
||||
/>
|
||||
)}
|
||||
</Field>
|
||||
{/* 子版本（仅 LAMOST 有内容；其他源该 Field 不渲染） */}
|
||||
{versionOptions.length > 0 && (
|
||||
<Field label="子版本">
|
||||
<CustomSelect
|
||||
value={searchForm.version || '__none__'}
|
||||
onChange={(v) =>
|
||||
setSearchForm((f) => ({
|
||||
...f,
|
||||
version: v === '__none__' ? '' : v,
|
||||
}))
|
||||
}
|
||||
className="w-full"
|
||||
options={[
|
||||
{ value: '__none__', label: '默认（最新公开）' },
|
||||
...versionOptions,
|
||||
]}
|
||||
/>
|
||||
</Field>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* LAMOST MRS 版本约束提示 */}
|
||||
{lamostMrsBlocked && (
|
||||
{/* Internal DR 需登录提示 */}
|
||||
{currentReleaseNeedsAuth && (
|
||||
<div className="flex items-center gap-2 text-xs text-rose-700 bg-rose-50 border border-rose-200 rounded-md p-2.5">
|
||||
<Lock className="w-3.5 h-3.5 shrink-0" />
|
||||
<span>
|
||||
{searchForm.release.toUpperCase()} 当前为 Internal 数据发布，需经
|
||||
oauth.china-vo.org 登录认证后才能访问，公开检索暂不可用
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* 交叉约束提示：当前 release 不支持该 subtype */}
|
||||
{subtypeBlocked && (
|
||||
<div className="flex items-center gap-2 text-xs text-amber-700 bg-amber-50 border border-amber-200 rounded-md p-2.5">
|
||||
<AlertTriangle className="w-3.5 h-3.5 shrink-0" />
|
||||
<span>{lamostMrsBlocked}</span>
|
||||
<span>{subtypeBlocked}</span>
|
||||
{currentSpec?.subtypes && currentSpec.subtypes.length > 0 && (
|
||||
<button
|
||||
onClick={() =>
|
||||
setSearchForm((f) => ({ ...f, subtype: 'lrs' }))
|
||||
}
|
||||
onClick={() => {
|
||||
// 切换到该 release 支持的首个 subtype
|
||||
const map = currentSpec?.subtypes_per_release;
|
||||
const valid =
|
||||
(map && searchForm.release && map[searchForm.release]) ||
|
||||
currentSpec?.subtypes;
|
||||
setSearchForm((f) => ({ ...f, subtype: valid?.[0] ?? '' }));
|
||||
}}
|
||||
className="ml-auto px-2 py-0.5 rounded bg-amber-100 hover:bg-amber-200 text-amber-800 font-bold text-[10px]"
|
||||
>
|
||||
切换为 LRS
|
||||
切换可用子类型
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
@ -410,14 +472,15 @@ function SearchView({
|
||||
<AlertTriangle className="w-3.5 h-3.5 shrink-0" />
|
||||
<span>
|
||||
当前半径 {searchForm.radius}° 超出建议值（≤
|
||||
{currentSpec.suggested_max_radius_deg}°），大范围查询可能因主表过大被服务端超时拒绝
|
||||
{currentSpec.suggested_max_radius_deg}
|
||||
°），大范围查询可能因主表过大被服务端超时拒绝
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
<div className="flex items-center gap-2 pt-1">
|
||||
<button
|
||||
onClick={runSearch}
|
||||
disabled={searching}
|
||||
disabled={searching || currentReleaseNeedsAuth}
|
||||
className="btn-console btn-console-primary px-4 py-2 rounded-md text-xs font-bold flex items-center gap-2 disabled:opacity-50"
|
||||
>
|
||||
{searching ? (
|
||||
@ -432,7 +495,7 @@ function SearchView({
|
||||
</span>
|
||||
<button
|
||||
onClick={() => downloadByCoordinates('nearest')}
|
||||
disabled={downloading}
|
||||
disabled={downloading || currentReleaseNeedsAuth}
|
||||
className="ml-auto px-3 py-2 rounded-md text-xs font-bold flex items-center gap-1.5 bg-emerald-50 text-emerald-700 border border-emerald-200 hover:bg-emerald-100 disabled:opacity-50"
|
||||
>
|
||||
<Download className="w-3.5 h-3.5" />
|
||||
@ -786,7 +849,10 @@ function LibraryView({
|
||||
<>
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-3">
|
||||
{libraryItems.map((rec) => (
|
||||
<LibraryCard key={`${rec.source}|${rec.product}|${rec.source_id}`} rec={rec} />
|
||||
<LibraryCard
|
||||
key={`${rec.source}|${rec.product}|${rec.source_id}`}
|
||||
rec={rec}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
|
||||
@ -863,7 +929,11 @@ function Field({
|
||||
compact?: boolean;
|
||||
}) {
|
||||
return (
|
||||
<div className={compact ? 'w-full sm:w-36 space-y-1.5 font-bold' : 'space-y-1.5'}>
|
||||
<div
|
||||
className={
|
||||
compact ? 'w-full sm:w-36 space-y-1.5 font-bold' : 'space-y-1.5'
|
||||
}
|
||||
>
|
||||
<label className="block text-slate-500 font-bold text-xs">{label}</label>
|
||||
{children}
|
||||
</div>
|
||||
@ -894,7 +964,12 @@ function LibraryCard({ rec }: { rec: ObservationRecord }) {
|
||||
const theme = SOURCE_THEME[rec.source] ?? SOURCE_THEME.lamost;
|
||||
const productLabel = PRODUCT_LABEL[rec.product] ?? rec.product;
|
||||
|
||||
let artifacts: Array<{ path: string; format: string; size?: number; band?: string }> = [];
|
||||
let artifacts: Array<{
|
||||
path: string;
|
||||
format: string;
|
||||
size?: number;
|
||||
band?: string;
|
||||
}> = [];
|
||||
try {
|
||||
const parsed = JSON.parse(rec.artifacts_json);
|
||||
if (Array.isArray(parsed)) artifacts = parsed;
|
||||
@ -910,7 +985,9 @@ function LibraryCard({ rec }: { rec: ObservationRecord }) {
|
||||
<div className="bg-white border border-slate-200 rounded-lg p-3.5 shadow-xs hover:shadow-md hover:border-slate-300 transition-all">
|
||||
<div className="flex items-center justify-between mb-2 pb-2 border-b border-slate-100">
|
||||
<div className="flex items-center gap-1.5">
|
||||
<span className={`px-2 py-0.5 rounded text-[10px] font-bold ${theme.badge}`}>
|
||||
<span
|
||||
className={`px-2 py-0.5 rounded text-[10px] font-bold ${theme.badge}`}
|
||||
>
|
||||
{theme.label}
|
||||
</span>
|
||||
<span className="px-2 py-0.5 rounded text-[10px] font-bold bg-slate-200 text-slate-700">
|
||||
@ -927,7 +1004,10 @@ function LibraryCard({ rec }: { rec: ObservationRecord }) {
|
||||
<div className="text-[9px] font-bold text-slate-400 tracking-widest uppercase">
|
||||
Source ID
|
||||
</div>
|
||||
<div className="font-mono text-xs text-slate-800 font-semibold truncate" title={rec.source_id}>
|
||||
<div
|
||||
className="font-mono text-xs text-slate-800 font-semibold truncate"
|
||||
title={rec.source_id}
|
||||
>
|
||||
{rec.source_id}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -250,6 +250,12 @@ export interface CapabilitySpec {
|
||||
subtypes: string[];
|
||||
releases: string[];
|
||||
default_release?: string | null;
|
||||
/// 每个 release 对应的子版本列表（仅 LAMOST 有内容，如 {dr11: ["v0","v1.0","v1.1","v2.0"]}）
|
||||
release_versions?: Record<string, string[]>;
|
||||
/// 需要登录认证才能访问的 release 列表（如 LAMOST Internal DR12-14），空表示全部公开
|
||||
releases_requiring_auth?: string[];
|
||||
/// 每个 release 实际支持的 subtype（交叉约束，如 LAMOST DR5/6 无 MRS）；空表示全部支持
|
||||
subtypes_per_release?: Record<string, string[]>;
|
||||
suggested_max_radius_deg: number;
|
||||
hard_max_radius_deg: number;
|
||||
identifier_format?: string | null;
|
||||
|
||||
@ -16,10 +16,7 @@ interface AxiosLikeError {
|
||||
///
|
||||
/// 优先级：response.data 为字符串 → 对象里的 error/message/detail/description
|
||||
/// → 整体 JSON 序列化兜底 → axios message → fallback
|
||||
export function extractErrorMessage(
|
||||
e: unknown,
|
||||
fallback: string
|
||||
): string {
|
||||
export function extractErrorMessage(e: unknown, fallback: string): string {
|
||||
const axiosError = e as AxiosLikeError;
|
||||
const data = axiosError?.response?.data;
|
||||
if (typeof data === 'string') return data;
|
||||
|
||||
86
deploy.sh
Executable file
86
deploy.sh
Executable file
@ -0,0 +1,86 @@
|
||||
#!/bin/bash
|
||||
# deploy.sh — AstroResearch 一键本地打包并推送服务器脚本 (Compose 部署版)
|
||||
|
||||
# 退出脚本如果发生错误
|
||||
set -e
|
||||
|
||||
# =============================================================================
|
||||
# 配置参数
|
||||
# =============================================================================
|
||||
REMOTE_USER="fmq"
|
||||
REMOTE_IP="100.66.1.2"
|
||||
REMOTE_PORT="22"
|
||||
REMOTE_DIR="/vol1/1000/program/astroresearch"
|
||||
IMAGE_NAME="astroresearch"
|
||||
TAG="latest"
|
||||
TAR_NAME="astroresearch.tar.gz"
|
||||
|
||||
echo "=== 1. 开始在本地构建 Docker 镜像 (基于 Dockerfile.modeB) ==="
|
||||
docker build -f Dockerfile.modeB -t ${IMAGE_NAME}:${TAG} .
|
||||
|
||||
echo "=== 2. 将镜像导出并压缩为本地临时文件 ${TAR_NAME} ==="
|
||||
docker save ${IMAGE_NAME}:${TAG} | gzip > ${TAR_NAME}
|
||||
|
||||
echo "=== 3. 上传镜像包到服务器目标目录 ==="
|
||||
scp -P ${REMOTE_PORT} ${TAR_NAME} ${REMOTE_USER}@${REMOTE_IP}:${REMOTE_DIR}/
|
||||
|
||||
echo "=== 4. 连接服务器：载入镜像并使用 Docker Compose 启动 ==="
|
||||
# 使用 'EOF' 锁死环境，防止本地变量混淆远程环境变量
|
||||
ssh -p ${REMOTE_PORT} ${REMOTE_USER}@${REMOTE_IP} << 'EOF'
|
||||
set -e
|
||||
|
||||
# 远程运行参数
|
||||
REMOTE_DIR="/vol1/1000/program/astroresearch"
|
||||
IMAGE_NAME="astroresearch"
|
||||
TAG="latest"
|
||||
TAR_NAME="astroresearch.tar.gz"
|
||||
|
||||
echo " -> 正在导入 Docker 镜像..."
|
||||
docker load < ${REMOTE_DIR}/${TAR_NAME}
|
||||
|
||||
echo " -> 清理服务器上的临时镜像压缩包..."
|
||||
rm -f ${REMOTE_DIR}/${TAR_NAME}
|
||||
|
||||
echo " -> 在服务器上写入/更新生产环境的 docker-compose.yml..."
|
||||
cat > ${REMOTE_DIR}/docker-compose.yml << 'INNER_EOF'
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
astroresearch:
|
||||
image: astroresearch:latest
|
||||
container_name: astroresearch
|
||||
restart: always
|
||||
ports:
|
||||
- "8001:8000"
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
- DATABASE_URL=sqlite:///app/library/astro_research.db
|
||||
- LIBRARY_DIR=/app/library
|
||||
- SKILLS_DIR=/app/skills
|
||||
- LOG_DIR=/app/logs
|
||||
- PORT=8000
|
||||
volumes:
|
||||
- ./library:/app/library
|
||||
- ./logs:/app/logs
|
||||
- ./skills:/app/skills:ro
|
||||
INNER_EOF
|
||||
|
||||
# 进入远程部署目录
|
||||
cd ${REMOTE_DIR}
|
||||
|
||||
echo " -> 停止并移除已有的旧服务容器 (docker compose down)..."
|
||||
docker compose down 2>/dev/null || true
|
||||
|
||||
echo " -> 使用 Docker Compose 启动新服务 (docker compose up -d)..."
|
||||
docker compose up -d
|
||||
|
||||
echo " -> 部署成功！正在等待 3 秒检查服务运行状态..."
|
||||
sleep 3
|
||||
docker compose ps
|
||||
EOF
|
||||
|
||||
echo "=== 5. 清理本地临时压缩包 ==="
|
||||
rm -f ${TAR_NAME}
|
||||
|
||||
echo "🎉 所有部署操作已圆满完成！服务已在服务器上通过 Docker Compose 后台运行。"
|
||||
@ -25,7 +25,7 @@ open http://localhost:8000
|
||||
### 两种镜像模式对比
|
||||
|
||||
| | Mode A (`Dockerfile`) | Mode B (`Dockerfile.modeB`) |
|
||||
|---|---|---|
|
||||
| --------------------- | ----------------------- | ------------------------------------------------ |
|
||||
| **镜像大小** | ~23 MB | ~87 MB |
|
||||
| **基础镜像** | `alpine:3.21` | `distroless/cc-debian12:nonroot` |
|
||||
| **libc** | musl (静态链接) | glibc (动态链接) |
|
||||
@ -33,7 +33,7 @@ open http://localhost:8000
|
||||
| **二进制包含** | Rust + C 依赖 | Rust + C + V8 + BoringSSL |
|
||||
| **运行时依赖** | 无 (静态链接) | `libc` + `libstdc++` + `libgcc_s` (已内置) |
|
||||
| **空闲内存** | ~8.6 MiB | ~8.7 MiB |
|
||||
| **Obscura** | 外部 `bin/` bind mount | 编译在二进制内 |
|
||||
| **Obscura** | 外部`bin/` bind mount | 编译在二进制内 |
|
||||
| **容器 Shell** | 有 (`/bin/sh`) | 无 (distroless) |
|
||||
| **Healthcheck** | `wget` TCP probe | 编排层替代 |
|
||||
|
||||
@ -44,11 +44,13 @@ open http://localhost:8000
|
||||
镜像最小（23 MB），Obscura 作为外部二进制通过 bind mount 注入，可独立更新。
|
||||
|
||||
**构建：**
|
||||
|
||||
```bash
|
||||
docker build -t astroresearch:latest .
|
||||
```
|
||||
|
||||
**运行：**
|
||||
|
||||
```bash
|
||||
docker run -d --name astro -p 8000:8000 --env-file .env \
|
||||
-v ./library:/app/library \
|
||||
@ -61,6 +63,7 @@ docker run -d --name astro -p 8000:8000 --env-file .env \
|
||||
> `bin/` 目录需包含编译好的 Obscura 二进制文件（`obscura` 和 `obscura-worker`）。
|
||||
|
||||
**docker-compose.yml（已内置）：**
|
||||
|
||||
```yaml
|
||||
services:
|
||||
astroresearch:
|
||||
@ -90,17 +93,20 @@ services:
|
||||
Obscura (V8 + BoringSSL) 编译在二进制内，单容器零外部二进制依赖。适合对外交付的一键部署包。
|
||||
|
||||
**前置条件：**
|
||||
|
||||
```bash
|
||||
mkdir -p libs
|
||||
git clone https://github.com/h4ckf0r0day/obscura libs/obscura
|
||||
```
|
||||
|
||||
**构建：**
|
||||
|
||||
```bash
|
||||
docker build -f Dockerfile.modeB -t astroresearch-all:latest .
|
||||
```
|
||||
|
||||
**运行：**
|
||||
|
||||
```bash
|
||||
docker run -d --name astro -p 8000:8000 --env-file .env \
|
||||
-v ./library:/app/library \
|
||||
@ -124,7 +130,7 @@ docker build --build-arg USE_MIRRORS=0 -t astroresearch:latest .
|
||||
**加速源：**
|
||||
|
||||
| 工具 | 镜像 |
|
||||
|------|------|
|
||||
| ---------- | -------------------------- |
|
||||
| npm | `registry.npmmirror.com` |
|
||||
| Alpine apk | `mirrors.aliyun.com` |
|
||||
| Debian apt | `mirrors.ustc.edu.cn` |
|
||||
@ -134,9 +140,9 @@ docker build --build-arg USE_MIRRORS=0 -t astroresearch:latest .
|
||||
### 持久化数据卷
|
||||
|
||||
| 容器路径 | 说明 | 推荐权限 |
|
||||
|----------|------|---------|
|
||||
| ---------------- | ----------------------------------------- | -------------- |
|
||||
| `/app/library` | SQLite 数据库 + PDF/HTML 全文 | 读写 |
|
||||
| `/app/logs` | 应用日志（仅 `LOG_OUTPUTS=file` 时写入） | 读写 |
|
||||
| `/app/logs` | 应用日志（仅`LOG_OUTPUTS=file` 时写入） | 读写 |
|
||||
| `/app/skills` | Agent 技能文件 (SKILL.md) | 只读 (`:ro`) |
|
||||
| `/app/bin` | Obscura 外部二进制（仅 Mode A） | 只读 (`:ro`) |
|
||||
|
||||
@ -157,21 +163,26 @@ docker build --build-arg USE_MIRRORS=0 -t astroresearch:latest .
|
||||
### 构建步骤
|
||||
|
||||
**步骤 1：构建前端**
|
||||
|
||||
```bash
|
||||
cd dashboard
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
产物位于 `dashboard/dist/`。
|
||||
|
||||
**步骤 2：编译后端**
|
||||
|
||||
```bash
|
||||
cd ..
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
产物位于 `target/release/astroresearch`。
|
||||
|
||||
**步骤 3（可选）：健康检查工具**
|
||||
|
||||
```bash
|
||||
cargo build --release --bin health_check
|
||||
```
|
||||
@ -195,11 +206,12 @@ cp .env.example .env # 编辑填入 API Key
|
||||
Obscura 作为独立二进制运行，与主进程隔离。适合常规生产环境。
|
||||
|
||||
| 部署方式 | 步骤 |
|
||||
|----------|------|
|
||||
| ---------------- | ----------------------------------------------------- |
|
||||
| **Docker** | `docker compose up -d`（自动 bind mount `./bin`） |
|
||||
| **源码** | 下载 obscura 到 `bin/`，赋予执行权限后启动 |
|
||||
| **源码** | 下载 obscura 到`bin/`，赋予执行权限后启动 |
|
||||
|
||||
二进制安装（源码部署时）：
|
||||
|
||||
```bash
|
||||
mkdir -p bin/
|
||||
# 下载 obscura 和 obscura-worker 到 bin/，赋予执行权限
|
||||
@ -211,7 +223,7 @@ chmod +x bin/obscura bin/obscura-worker
|
||||
Obscura (V8 + BoringSSL) 编译进二进制，单文件零外部依赖。
|
||||
|
||||
| 部署方式 | 构建命令 |
|
||||
|----------|---------|
|
||||
| ---------------- | ------------------------------------------------------------------ |
|
||||
| **Docker** | `docker build -f Dockerfile.modeB -t astroresearch-all:latest .` |
|
||||
| **源码** | `cargo build --release --features obscura-inprocess` |
|
||||
|
||||
@ -228,7 +240,7 @@ Obscura (V8 + BoringSSL) 编译进二进制，单文件零外部依赖。
|
||||
**优化指标 (Mode A, release-min)：**
|
||||
|
||||
| 指标 | release | release-min | 降幅 |
|
||||
|------|---------|-------------|------|
|
||||
| --------------- | ------- | ----------- | ---- |
|
||||
| 二进制大小 | 17.0 MB | 8.3 MB | 51% |
|
||||
| 启动 RSS | 34.8 MB | 32.9 MB | 5% |
|
||||
| 虚拟内存 (VSZ) | 1.27 GB | 302 MB | 76% |
|
||||
@ -243,6 +255,7 @@ docker build -t astroresearch:latest .
|
||||
```
|
||||
|
||||
**限制 Tokio 线程数：**
|
||||
|
||||
```bash
|
||||
PORT=8000 TOKIO_WORKER_THREADS=1 ./astroresearch
|
||||
```
|
||||
@ -252,14 +265,14 @@ PORT=8000 TOKIO_WORKER_THREADS=1 ./astroresearch
|
||||
## 5. 环境变量
|
||||
|
||||
| 变量名 | 必需 | 默认值 | 说明 |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| :-------------------------- | :--- | :------------------------------------------ | :------------------------------- |
|
||||
| `DATABASE_URL` | 否 | `sqlite:///app/library/astro_research.db` | SQLite 连接 URL |
|
||||
| `ADS_API_KEY` | 是 | - | NASA ADS API Token |
|
||||
| `LLM_API_KEY` | 是 | - | LLM API Key |
|
||||
| `LLM_API_BASE` | 否 | `https://api.openai.com/v1` | LLM API 地址 |
|
||||
| `LLM_MODEL` | 否 | `gpt-4o-mini` | 对话模型 |
|
||||
| `EMBEDDING_API_KEY` | 否 | 同 `LLM_API_KEY` | Embedding API Key |
|
||||
| `EMBEDDING_API_BASE` | 否 | 同 `LLM_API_BASE` | Embedding API 地址 |
|
||||
| `EMBEDDING_API_KEY` | 否 | 同`LLM_API_KEY` | Embedding API Key |
|
||||
| `EMBEDDING_API_BASE` | 否 | 同`LLM_API_BASE` | Embedding API 地址 |
|
||||
| `EMBEDDING_MODEL` | 否 | `text-embedding-3-small` | Embedding 模型 |
|
||||
| `LIBRARY_DIR` | 否 | `/app/library` | 文献馆藏根目录 |
|
||||
| `SKILLS_DIR` | 否 | `/app/skills` | Agent 技能目录 |
|
||||
@ -277,11 +290,13 @@ PORT=8000 TOKIO_WORKER_THREADS=1 ./astroresearch
|
||||
## 6. 健康检查与维护
|
||||
|
||||
**Docker 健康检查（Mode A）：**
|
||||
|
||||
```bash
|
||||
docker ps --filter “health=healthy” --filter “name=astroresearch”
|
||||
```
|
||||
|
||||
Mode B (distroless) 无内置 healthcheck，可在编排层配置：
|
||||
|
||||
```yaml
|
||||
# docker-compose 或 k8s
|
||||
healthcheck:
|
||||
@ -289,6 +304,7 @@ healthcheck:
|
||||
```
|
||||
|
||||
**源码部署健康检查：**
|
||||
|
||||
```bash
|
||||
# 只读扫描
|
||||
./health_check
|
||||
|
||||
@ -250,7 +250,7 @@ pub fn rough_estimate_tokens(messages: &[ChatMessage]) -> usize {
|
||||
messages
|
||||
.iter()
|
||||
.map(|m| {
|
||||
let content_len = m.text().map_or(0, |c| estimate_tokens_from_text(c));
|
||||
let content_len = m.text().map_or(0, estimate_tokens_from_text);
|
||||
content_len + 4 // 消息结构 overhead 约 4 token
|
||||
})
|
||||
.sum()
|
||||
@ -697,7 +697,6 @@ fn spawn_memory_extraction(
|
||||
memory_manager: std::sync::Arc<tokio::sync::Mutex<crate::agent::memory::MemoryManager>>,
|
||||
app_state: std::sync::Arc<crate::api::AppState>,
|
||||
) {
|
||||
|
||||
let sid = session_id.to_string();
|
||||
let mem_mgr = memory_manager.clone();
|
||||
let app = app_state.clone();
|
||||
|
||||
@ -2,7 +2,6 @@
|
||||
//
|
||||
// 内置 Hooks — CancellationHook、MetricsHook、AuditLogHook。
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
@ -48,11 +47,11 @@ impl ContextDeduplicator {
|
||||
|
||||
/// 取消检查 Hook — 在每次工具执行前检查用户是否中止了会话。
|
||||
pub struct CancellationHook {
|
||||
cancelled_runs: Arc<Mutex<HashSet<String>>>,
|
||||
cancelled_runs: Arc<dashmap::DashMap<String, ()>>,
|
||||
}
|
||||
|
||||
impl CancellationHook {
|
||||
pub fn new(cancelled_runs: Arc<Mutex<HashSet<String>>>) -> Self {
|
||||
pub fn new(cancelled_runs: Arc<dashmap::DashMap<String, ()>>) -> Self {
|
||||
CancellationHook { cancelled_runs }
|
||||
}
|
||||
}
|
||||
@ -68,8 +67,7 @@ impl AgentHook for CancellationHook {
|
||||
}
|
||||
|
||||
async fn pre_tool_use(&self, ctx: &PreToolUseContext) -> PreToolUseAction {
|
||||
let runs = self.cancelled_runs.lock().await;
|
||||
if runs.contains(&ctx.session_id) {
|
||||
if self.cancelled_runs.contains_key(&ctx.session_id) {
|
||||
warn!(
|
||||
"[CancellationHook] 会话 {} 已被用户取消，阻止工具 {} 执行",
|
||||
ctx.session_id, ctx.tool_name
|
||||
@ -83,8 +81,7 @@ impl AgentHook for CancellationHook {
|
||||
|
||||
async fn on_session_stop(&self, ctx: &SessionStopContext<'_>) {
|
||||
// 清理取消状态
|
||||
let mut runs = self.cancelled_runs.lock().await;
|
||||
runs.remove(&ctx.session_id);
|
||||
self.cancelled_runs.remove(&ctx.session_id);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -201,9 +201,8 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cancellation_hook_blocks_when_cancelled() {
|
||||
let mut cancelled = HashSet::new();
|
||||
cancelled.insert("test_session".to_string());
|
||||
let cancelled_runs = Arc::new(tokio::sync::Mutex::new(cancelled));
|
||||
let cancelled_runs = Arc::new(dashmap::DashMap::new());
|
||||
cancelled_runs.insert("test_session".to_string(), ());
|
||||
|
||||
let hook = CancellationHook::new(cancelled_runs);
|
||||
let ctx = PreToolUseContext {
|
||||
@ -219,7 +218,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cancellation_hook_allows_when_not_cancelled() {
|
||||
let cancelled_runs = Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new()));
|
||||
let cancelled_runs = Arc::new(dashmap::DashMap::new());
|
||||
|
||||
let hook = CancellationHook::new(cancelled_runs);
|
||||
let ctx = PreToolUseContext {
|
||||
|
||||
@ -52,7 +52,7 @@ impl HookRegistry {
|
||||
/// 使得 AgentRuntime.get_metrics() 能查询到实际运行数据。
|
||||
pub fn with_builtins(
|
||||
db: SqlitePool,
|
||||
cancelled_runs: Arc<tokio::sync::Mutex<std::collections::HashSet<String>>>,
|
||||
cancelled_runs: Arc<dashmap::DashMap<String, ()>>,
|
||||
metrics_data: Option<Arc<tokio::sync::Mutex<MetricsData>>>,
|
||||
) -> Self {
|
||||
let mut registry = Self::new();
|
||||
|
||||
@ -138,6 +138,7 @@ pub(super) async fn process_single_result(
|
||||
max_output_chars,
|
||||
&tool_results_dir,
|
||||
)
|
||||
.await
|
||||
};
|
||||
|
||||
// PostToolUse hook
|
||||
|
||||
@ -538,8 +538,7 @@ pub async fn execute_parallel(
|
||||
let cancel_handle = tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
let locked = app_state_ref.cancelled_runs.lock().await;
|
||||
if locked.contains(&sid_ref) {
|
||||
if app_state_ref.cancelled_runs.contains_key(&sid_ref) {
|
||||
cancel_flag.store(true, Ordering::SeqCst);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -96,10 +96,12 @@ pub struct AgentRuntime {
|
||||
}
|
||||
|
||||
impl AgentRuntime {
|
||||
/// 创建新的运行时实例
|
||||
pub fn new(app_state: Arc<AppState>) -> Self {
|
||||
let mut config = AgentConfig::default();
|
||||
let mode_registry = ModeRegistry::builtins();
|
||||
/// 共享初始化逻辑：根据 config + mode 构建完整运行时。
|
||||
fn init(
|
||||
app_state: Arc<AppState>,
|
||||
mut config: AgentConfig,
|
||||
mode_registry: ModeRegistry,
|
||||
) -> Self {
|
||||
let mode = mode_registry.get(&config.mode).copied().unwrap_or_else(|| {
|
||||
tracing::warn!("[AgentRuntime] 未知模式 '{}'，回退到默认模式", config.mode);
|
||||
mode_registry
|
||||
@ -123,14 +125,13 @@ impl AgentRuntime {
|
||||
tool_registry.add_tool(Box::new(crate::agent::tools::memory::SaveMemoryTool::new(
|
||||
app_state.memory_manager.clone(),
|
||||
)));
|
||||
// 替换 DelegateResearchTool 为带有 permission_checker 的版本（SSE 通道通过 ToolContext 注入）
|
||||
// 替换 DelegateResearchTool 为带有 permission_checker 的版本
|
||||
tool_registry.replace_tool(Box::new(
|
||||
crate::agent::tools::subagent::SubAgentTool::new_with_hooks(
|
||||
None,
|
||||
permission_checker.clone(),
|
||||
),
|
||||
));
|
||||
// 会话级权限检查器将在 run_react_loop 中按 session_id 注册
|
||||
|
||||
// 视觉模型可用时注册 analyze_image 工具
|
||||
if app_state.vision_llm.is_some() {
|
||||
@ -140,9 +141,9 @@ impl AgentRuntime {
|
||||
// ── 应用模式的工具集过滤 ──
|
||||
apply_mode_tool_filter(&mut tool_registry, mode);
|
||||
|
||||
// 初始化 checkpoint 管理器
|
||||
// 初始化 checkpoint 管理器（存储在 library_dir/.checkpoints 下）
|
||||
let checkpoint_enabled = true;
|
||||
let checkpoint_store = app_state.config.library_dir.join("..").join(".checkpoints");
|
||||
let checkpoint_store = app_state.config.library_dir.join(".checkpoints");
|
||||
let checkpoint_manager = Arc::new(checkpoint::CheckpointManager::new(
|
||||
std::fs::canonicalize(&checkpoint_store).unwrap_or(checkpoint_store),
|
||||
checkpoint_enabled,
|
||||
@ -168,78 +169,25 @@ impl AgentRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
/// 创建新的运行时实例
|
||||
pub fn new(app_state: Arc<AppState>) -> Self {
|
||||
let config = AgentConfig::default();
|
||||
let mode_registry = ModeRegistry::builtins();
|
||||
Self::init(app_state, config, mode_registry)
|
||||
}
|
||||
|
||||
/// 创建带自定义配置的运行时实例
|
||||
pub fn with_config(app_state: Arc<AppState>, mut config: AgentConfig) -> Self {
|
||||
pub fn with_config(app_state: Arc<AppState>, config: AgentConfig) -> Self {
|
||||
let mode_registry = ModeRegistry::builtins();
|
||||
let mode = mode_registry.get(&config.mode).copied().unwrap_or_else(|| {
|
||||
tracing::warn!("[AgentRuntime] 未知模式 '{}'，回退到默认模式", config.mode);
|
||||
mode_registry
|
||||
.get(ModeRegistry::default_id())
|
||||
.copied()
|
||||
.unwrap_or(&modes::default::DEFAULT_MODE)
|
||||
});
|
||||
// 合并模式配置预设
|
||||
apply_mode_config(&mut config, mode);
|
||||
|
||||
let queue = Arc::new(BgNotificationQueue::new());
|
||||
let metrics_data = Arc::new(tokio::sync::Mutex::new(super::hooks::MetricsData::default()));
|
||||
let permission_checker = Arc::new(permission::PermissionChecker::from_config(&config));
|
||||
let denial_tracker = Arc::new(std::sync::Mutex::new(denial_tracker::DenialTracker::new(
|
||||
config.denial_max_consecutive,
|
||||
config.denial_max_total,
|
||||
)));
|
||||
let skill_registry = app_state.skill_registry.clone();
|
||||
let mut tool_registry = ToolRegistry::new_with_queue(Some(queue.clone()), skill_registry);
|
||||
tool_registry.add_tool(Box::new(crate::agent::tools::memory::SaveMemoryTool::new(
|
||||
app_state.memory_manager.clone(),
|
||||
)));
|
||||
tool_registry.replace_tool(Box::new(
|
||||
crate::agent::tools::subagent::SubAgentTool::new_with_hooks(
|
||||
None,
|
||||
permission_checker.clone(),
|
||||
),
|
||||
));
|
||||
// 会话级权限检查器将在 run_react_loop 中按 session_id 注册
|
||||
|
||||
// 视觉模型可用时注册 analyze_image 工具
|
||||
if app_state.vision_llm.is_some() {
|
||||
tool_registry.add_tool(Box::new(crate::agent::tools::astro::AnalyzeImageTool));
|
||||
}
|
||||
|
||||
// ── 应用模式的工具集过滤 ──
|
||||
apply_mode_tool_filter(&mut tool_registry, mode);
|
||||
|
||||
// 初始化 checkpoint 管理器
|
||||
let checkpoint_enabled = true;
|
||||
let checkpoint_store = app_state.config.library_dir.join("..").join(".checkpoints");
|
||||
let checkpoint_manager = Arc::new(checkpoint::CheckpointManager::new(
|
||||
std::fs::canonicalize(&checkpoint_store).unwrap_or(checkpoint_store),
|
||||
checkpoint_enabled,
|
||||
));
|
||||
|
||||
AgentRuntime {
|
||||
app_state,
|
||||
config,
|
||||
tool_registry,
|
||||
bg_notification_queue: queue,
|
||||
metrics_data,
|
||||
compaction_breaker: Arc::new(std::sync::Mutex::new(
|
||||
circuit_breaker::CompactionCircuitBreaker::new(),
|
||||
)),
|
||||
permission_checker,
|
||||
denial_tracker,
|
||||
read_file_state: Arc::new(std::sync::Mutex::new(file_cache::FileStateCache::new())),
|
||||
prompt_cache: std::sync::Mutex::new(SystemPromptCache::new()),
|
||||
collapse_log: Arc::new(std::sync::Mutex::new(compact::collapse::CollapseLog::new())),
|
||||
checkpoint_manager,
|
||||
mode,
|
||||
mode_registry,
|
||||
}
|
||||
Self::init(app_state, config, mode_registry)
|
||||
}
|
||||
|
||||
/// 返回当前运行指标快照（锁异常时返回默认值并记录警告）
|
||||
pub fn get_metrics(&self) -> super::hooks::MetricsData {
|
||||
self.metrics_data.try_lock().map(|m| m.clone()).unwrap_or_else(|_| {
|
||||
self.metrics_data
|
||||
.try_lock()
|
||||
.map(|m| m.clone())
|
||||
.unwrap_or_else(|_| {
|
||||
warn!("[AgentRuntime] 获取指标锁失败，返回默认值");
|
||||
super::hooks::MetricsData::default()
|
||||
})
|
||||
@ -456,7 +404,7 @@ impl AgentRuntime {
|
||||
|
||||
// Phase 4: 会话收尾（传入实际的终止原因 + trajectory 导出参数）
|
||||
let system_prompt = self.system_prompt();
|
||||
let model_name = self.app_state.llm.model();
|
||||
let model_name = self.app_state.llm.model().await;
|
||||
finalize::finalize_turn(
|
||||
db,
|
||||
&session_info.session_id,
|
||||
@ -491,12 +439,10 @@ impl AgentRuntime {
|
||||
let turn_index = session_info.turn_index;
|
||||
|
||||
// 注册当前会话的权限检查器（如不存在则从全局配置初始化）
|
||||
{
|
||||
let mut checkers = self.app_state.session_permission_checkers.write().await;
|
||||
checkers
|
||||
self.app_state
|
||||
.session_permission_checkers
|
||||
.entry(sid.clone())
|
||||
.or_insert_with(|| (*self.permission_checker).clone());
|
||||
}
|
||||
|
||||
let tool_defs = self.tool_registry.definitions();
|
||||
let mut duplicate_detector = DuplicateDetector::default();
|
||||
@ -522,10 +468,7 @@ impl AgentRuntime {
|
||||
self.checkpoint_manager.new_turn();
|
||||
|
||||
// 检查用户取消
|
||||
let is_cancelled = {
|
||||
let mut cancelled = self.app_state.cancelled_runs.lock().await;
|
||||
cancelled.remove(sid)
|
||||
};
|
||||
let is_cancelled = self.app_state.cancelled_runs.remove(sid).is_some();
|
||||
|
||||
if is_cancelled {
|
||||
warn!("[AgentRuntime] 用户手动中止了会话 {} 的智能体执行", sid);
|
||||
@ -868,14 +811,11 @@ impl AgentRuntime {
|
||||
}
|
||||
|
||||
// 并行执行工具（带权限检查、checkpoint 和分区器）
|
||||
let session_checker_snapshot = {
|
||||
self.app_state
|
||||
let session_checker_snapshot = self
|
||||
.app_state
|
||||
.session_permission_checkers
|
||||
.read()
|
||||
.await
|
||||
.get(sid)
|
||||
.cloned()
|
||||
};
|
||||
.map(|r| r.value().clone());
|
||||
let exec_result = executor::execute_parallel(
|
||||
&prepared_calls,
|
||||
&self.tool_registry,
|
||||
@ -947,8 +887,7 @@ impl AgentRuntime {
|
||||
}
|
||||
|
||||
if exec_result.was_cancelled {
|
||||
let mut locked = self.app_state.cancelled_runs.lock().await;
|
||||
locked.remove(sid);
|
||||
self.app_state.cancelled_runs.remove(sid);
|
||||
warn!(
|
||||
"[AgentRuntime] 工具执行期间被用户手动中止，会话 ID: {}",
|
||||
sid
|
||||
@ -1016,8 +955,7 @@ impl AgentRuntime {
|
||||
match output.status {
|
||||
StreamStatus::Success => return Some(output),
|
||||
StreamStatus::Cancelled => {
|
||||
let mut cancelled = self.app_state.cancelled_runs.lock().await;
|
||||
cancelled.remove(session_id);
|
||||
self.app_state.cancelled_runs.remove(session_id);
|
||||
warn!(
|
||||
"[AgentRuntime] 流式调用期间被用户手动中止，会话 ID: {}",
|
||||
session_id
|
||||
@ -1077,7 +1015,7 @@ impl AgentRuntime {
|
||||
"[AgentRuntime] 连续 {} 次过载，切换到备用模型: {}",
|
||||
consecutive_overloads, fallback
|
||||
);
|
||||
llm.set_model(fallback.clone());
|
||||
llm.set_model(fallback.clone()).await;
|
||||
consecutive_overloads = 0;
|
||||
} else if !self.app_state.config.llm_fallback_chain.is_empty() {
|
||||
let idx = ((consecutive_overloads as usize - 3)
|
||||
@ -1088,15 +1026,14 @@ impl AgentRuntime {
|
||||
"[AgentRuntime] 连续 {} 次过载，从链中切换: {}",
|
||||
consecutive_overloads, alt
|
||||
);
|
||||
llm.set_model(alt.clone());
|
||||
llm.set_model(alt.clone()).await;
|
||||
consecutive_overloads = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 检查用户取消
|
||||
let cancelled = self.app_state.cancelled_runs.lock().await;
|
||||
if cancelled.contains(session_id) {
|
||||
if self.app_state.cancelled_runs.contains_key(session_id) {
|
||||
warn!("[AgentRuntime] 退避重试期间被用户取消");
|
||||
let _ = tx.send(AgentStreamEvent::Error {
|
||||
message: "用户已手动中止执行。".to_string(),
|
||||
@ -1123,8 +1060,7 @@ impl AgentRuntime {
|
||||
return Some(retry_output);
|
||||
}
|
||||
StreamStatus::Cancelled => {
|
||||
let mut cancelled = self.app_state.cancelled_runs.lock().await;
|
||||
cancelled.remove(session_id);
|
||||
self.app_state.cancelled_runs.remove(session_id);
|
||||
return None;
|
||||
}
|
||||
StreamStatus::Error(_) => {
|
||||
@ -1232,8 +1168,7 @@ impl AgentRuntime {
|
||||
return Some(retry_output);
|
||||
}
|
||||
StreamStatus::Cancelled => {
|
||||
let mut cancelled = self.app_state.cancelled_runs.lock().await;
|
||||
cancelled.remove(session_id);
|
||||
self.app_state.cancelled_runs.remove(session_id);
|
||||
warn!("[AgentRuntime] 恢复期间被用户中止");
|
||||
let _ = tx.send(AgentStreamEvent::Error {
|
||||
message: "用户已手动中止执行。".to_string(),
|
||||
@ -1400,7 +1335,7 @@ impl AgentRuntime {
|
||||
};
|
||||
|
||||
let today = chrono::Utc::now().format("%Y-%m-%d").to_string();
|
||||
let model_name = self.app_state.llm.model().to_string();
|
||||
let model_name = self.app_state.llm.model_name_snapshot();
|
||||
|
||||
let mut lines = vec![
|
||||
"# 环境信息".to_string(),
|
||||
|
||||
@ -83,7 +83,7 @@ pub async fn create_or_resume_session(
|
||||
)
|
||||
.bind(&new_id)
|
||||
.bind("")
|
||||
.bind(llm.model())
|
||||
.bind(llm.model().await)
|
||||
.bind(mode)
|
||||
.execute(db)
|
||||
.await?;
|
||||
@ -544,7 +544,9 @@ pub async fn branch_session(db: &SqlitePool, session_id: &str) -> anyhow::Result
|
||||
|
||||
let forked_at = last_active_id.unwrap_or(0);
|
||||
|
||||
// 4. 创建新会话
|
||||
// 4. 创建新会话与复制消息（同一个事务内）
|
||||
let mut tx = db.begin().await?;
|
||||
|
||||
let branch_id = uuid::Uuid::new_v4().to_string();
|
||||
let branch_title = if title.is_empty() {
|
||||
format!("分支 (来自 {})", &session_id[..8.min(session_id.len())])
|
||||
@ -565,7 +567,7 @@ pub async fn branch_session(db: &SqlitePool, session_id: &str) -> anyhow::Result
|
||||
.bind(&branch_title)
|
||||
.bind(session_id)
|
||||
.bind(serde_json::to_string(&branch_meta).unwrap_or_default())
|
||||
.execute(db)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
// 5. 复制所有 active=1 的消息到新会话
|
||||
@ -573,7 +575,7 @@ pub async fn branch_session(db: &SqlitePool, session_id: &str) -> anyhow::Result
|
||||
"SELECT COUNT(*) FROM agent_messages WHERE session_id = ? AND active = 1",
|
||||
)
|
||||
.bind(session_id)
|
||||
.fetch_one(db)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query(
|
||||
@ -588,9 +590,11 @@ pub async fn branch_session(db: &SqlitePool, session_id: &str) -> anyhow::Result
|
||||
)
|
||||
.bind(&branch_id)
|
||||
.bind(session_id)
|
||||
.execute(db)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
info!(
|
||||
"[Session] 分叉完成: parent={}, branch={}, copied={} messages, forked_at={}",
|
||||
session_id, branch_id, copied, forked_at
|
||||
|
||||
@ -43,7 +43,7 @@ pub async fn process_llm_stream(
|
||||
tx: &mpsc::UnboundedSender<AgentStreamEvent>,
|
||||
step: usize,
|
||||
session_id: &str,
|
||||
cancelled_runs: Arc<tokio::sync::Mutex<std::collections::HashSet<String>>>,
|
||||
cancelled_runs: Arc<dashmap::DashMap<String, ()>>,
|
||||
enable_thinking: bool,
|
||||
) -> StreamOutput {
|
||||
// 1. 发起 LLM 流式调用
|
||||
@ -76,8 +76,7 @@ pub async fn process_llm_stream(
|
||||
let cancel_fut = async {
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
let cancelled = cancelled_runs.lock().await;
|
||||
if cancelled.contains(&sid) {
|
||||
if cancelled_runs.contains_key(&sid) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@ -590,10 +590,7 @@ mod tests {
|
||||
ads,
|
||||
arxiv,
|
||||
vizier,
|
||||
lamost: crate::clients::lamost::LamostClient::new(
|
||||
"https://www.lamost.org",
|
||||
60,
|
||||
)
|
||||
lamost: crate::clients::lamost::LamostClient::new("https://www.lamost.org", 60)
|
||||
.unwrap(),
|
||||
gaia: crate::clients::gaia::GaiaClient::new(
|
||||
"https://gea.esac.esa.int/tap-server/tap",
|
||||
@ -621,7 +618,7 @@ mod tests {
|
||||
harvest_status: Arc::new(tokio::sync::Mutex::new(MetaSyncStatus::default())),
|
||||
batch_status: Arc::new(tokio::sync::Mutex::new(AssetBatchStatus::default())),
|
||||
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
|
||||
cancelled_runs: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())),
|
||||
cancelled_runs: Arc::new(dashmap::DashMap::new()),
|
||||
skill_registry: Arc::new(tokio::sync::RwLock::new(SkillRegistry::new(PathBuf::from(
|
||||
"/tmp/sk",
|
||||
)))),
|
||||
@ -629,16 +626,16 @@ mod tests {
|
||||
pending_permissions: Arc::new(
|
||||
tokio::sync::Mutex::new(std::collections::HashMap::new()),
|
||||
),
|
||||
session_permission_checkers: Arc::new(tokio::sync::RwLock::new(
|
||||
std::collections::HashMap::new(),
|
||||
)),
|
||||
session_permission_checkers: Arc::new(dashmap::DashMap::new()),
|
||||
sse_broadcast: None,
|
||||
memory_manager: Arc::new(tokio::sync::Mutex::new(MemoryManager::new(PathBuf::from(
|
||||
"/tmp/test_mem",
|
||||
)))),
|
||||
sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
|
||||
login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
||||
upload_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
||||
upload_rate_limiter: Arc::new(
|
||||
tokio::sync::Mutex::new(std::collections::HashMap::new()),
|
||||
),
|
||||
});
|
||||
|
||||
ToolContext::new(app_state)
|
||||
|
||||
@ -26,7 +26,7 @@ impl SkillCreator {
|
||||
///
|
||||
/// 返回创建的 skill 名称列表。
|
||||
/// 如果目标 skill 目录已存在则跳过（不覆盖已有 skill）。
|
||||
pub fn create_from_patterns(
|
||||
pub async fn create_from_patterns(
|
||||
&self,
|
||||
patterns: &[pattern_detector::DetectedPattern],
|
||||
) -> std::io::Result<Vec<String>> {
|
||||
@ -46,13 +46,13 @@ impl SkillCreator {
|
||||
}
|
||||
|
||||
// 创建 skill 目录
|
||||
std::fs::create_dir_all(&skill_dir)?;
|
||||
tokio::fs::create_dir_all(&skill_dir).await?;
|
||||
|
||||
// 生成 SKILL.md 内容
|
||||
let content = self.generate_skill_md(pattern, &skill_name);
|
||||
|
||||
let md_path = skill_dir.join("SKILL.md");
|
||||
std::fs::write(&md_path, &content)?;
|
||||
tokio::fs::write(&md_path, &content).await?;
|
||||
|
||||
info!(
|
||||
"[SkillCreator] 已从模式创建 Skill '{}': {}",
|
||||
@ -217,7 +217,7 @@ impl SelfImprovePipeline {
|
||||
info!("[SelfImprove] 检测到 {} 个候选模式", patterns.len());
|
||||
|
||||
// Step 2: 创建 Skills
|
||||
let created = self.creator.create_from_patterns(&patterns)?;
|
||||
let created = self.creator.create_from_patterns(&patterns).await?;
|
||||
|
||||
// Step 3: Curator 分析
|
||||
let curator_report = self.curator.analyze(usage_stats, skill_metas);
|
||||
|
||||
@ -340,9 +340,12 @@ impl Curator {
|
||||
/// 归档指定的 stale skills（移动到 archive 目录而非删除）。
|
||||
///
|
||||
/// 返回成功归档的 skill 名称列表。
|
||||
pub fn archive_stale_skills(&self, skill_names: &[String]) -> std::io::Result<Vec<String>> {
|
||||
pub async fn archive_stale_skills(
|
||||
&self,
|
||||
skill_names: &[String],
|
||||
) -> std::io::Result<Vec<String>> {
|
||||
// 确保归档目录存在
|
||||
std::fs::create_dir_all(&self.archive_dir)?;
|
||||
tokio::fs::create_dir_all(&self.archive_dir).await?;
|
||||
|
||||
let mut archived = Vec::new();
|
||||
|
||||
@ -355,17 +358,16 @@ impl Curator {
|
||||
|
||||
let archive_path = self.archive_dir.join(name);
|
||||
if archive_path.exists() {
|
||||
// 避免覆盖已有归档：添加时间戳后缀
|
||||
let ts = Utc::now().format("%Y%m%d%H%M%S");
|
||||
let renamed = self.archive_dir.join(format!("{}-{}", name, ts));
|
||||
info!(
|
||||
"[Curator] 归档目录已存在，使用新名称: {}",
|
||||
renamed.display()
|
||||
);
|
||||
std::fs::rename(&skill_dir, &renamed)?;
|
||||
tokio::fs::rename(&skill_dir, &renamed).await?;
|
||||
archived.push(format!("{}-{}", name, ts));
|
||||
} else {
|
||||
std::fs::rename(&skill_dir, &archive_path)?;
|
||||
tokio::fs::rename(&skill_dir, &archive_path).await?;
|
||||
archived.push(name.clone());
|
||||
}
|
||||
|
||||
|
||||
@ -64,29 +64,39 @@ pub fn inbox_path(team_dir: &Path, agent_name: &str) -> PathBuf {
|
||||
}
|
||||
|
||||
/// 向收件箱追加一条消息
|
||||
pub fn append_message(team_dir: &Path, agent_name: &str, msg: &TeamMessage) -> std::io::Result<()> {
|
||||
pub async fn append_message(
|
||||
team_dir: &Path,
|
||||
agent_name: &str,
|
||||
msg: &TeamMessage,
|
||||
) -> std::io::Result<()> {
|
||||
let inbox = inbox_path(team_dir, agent_name);
|
||||
if let Some(parent) = inbox.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
tokio::fs::create_dir_all(parent).await?;
|
||||
}
|
||||
let line = serde_json::to_string(msg).unwrap_or_default();
|
||||
let inbox_clone = inbox.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&inbox)?;
|
||||
.open(&inbox_clone)?;
|
||||
file.write_all(line.as_bytes())?;
|
||||
file.write_all(b"\n")?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
|
||||
}
|
||||
|
||||
/// 读取并清空收件箱
|
||||
pub fn drain_inbox(team_dir: &Path, agent_name: &str) -> Vec<TeamMessage> {
|
||||
pub async fn drain_inbox(team_dir: &Path, agent_name: &str) -> Vec<TeamMessage> {
|
||||
let inbox = inbox_path(team_dir, agent_name);
|
||||
if !inbox.exists() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let content = match std::fs::read_to_string(&inbox) {
|
||||
let inbox_clone = inbox.clone();
|
||||
let content = match tokio::fs::read_to_string(&inbox_clone).await {
|
||||
Ok(c) => c,
|
||||
Err(_) => return Vec::new(),
|
||||
};
|
||||
@ -98,7 +108,7 @@ pub fn drain_inbox(team_dir: &Path, agent_name: &str) -> Vec<TeamMessage> {
|
||||
.collect();
|
||||
|
||||
// 清空文件
|
||||
let _ = std::fs::write(&inbox, "");
|
||||
let _ = tokio::fs::write(&inbox, "").await;
|
||||
|
||||
messages
|
||||
}
|
||||
|
||||
@ -157,9 +157,15 @@ impl TeamManager {
|
||||
}
|
||||
|
||||
/// 发送消息给指定队友
|
||||
pub fn send_message(&self, from: &str, to: &str, content: &str, msg_type: TeamMessageType) {
|
||||
pub async fn send_message(
|
||||
&self,
|
||||
from: &str,
|
||||
to: &str,
|
||||
content: &str,
|
||||
msg_type: TeamMessageType,
|
||||
) {
|
||||
let msg = TeamMessage::new(from, to, content, msg_type);
|
||||
if let Err(e) = inbox::append_message(&self.team_dir, to, &msg) {
|
||||
if let Err(e) = inbox::append_message(&self.team_dir, to, &msg).await {
|
||||
warn!("[TeamManager] 发送消息失败: {}", e);
|
||||
} else {
|
||||
info!(
|
||||
@ -172,7 +178,7 @@ impl TeamManager {
|
||||
}
|
||||
|
||||
/// 广播消息给所有队友
|
||||
pub fn broadcast(&self, from: &str, content: &str) {
|
||||
pub async fn broadcast(&self, from: &str, content: &str) {
|
||||
info!(
|
||||
"[TeamManager] 广播消息: {}",
|
||||
content.chars().take(80).collect::<String>()
|
||||
@ -180,14 +186,14 @@ impl TeamManager {
|
||||
let msg = TeamMessage::new(from, "all", content, TeamMessageType::Status);
|
||||
for member in &self.config.members {
|
||||
if member.name != from {
|
||||
let _ = inbox::append_message(&self.team_dir, &member.name, &msg);
|
||||
let _ = inbox::append_message(&self.team_dir, &member.name, &msg).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 检查并清空指定 agent 的收件箱
|
||||
pub fn check_inbox(&self, agent_name: &str) -> Vec<TeamMessage> {
|
||||
inbox::drain_inbox(&self.team_dir, agent_name)
|
||||
pub async fn check_inbox(&self, agent_name: &str) -> Vec<TeamMessage> {
|
||||
inbox::drain_inbox(&self.team_dir, agent_name).await
|
||||
}
|
||||
|
||||
/// 列出所有队友状态
|
||||
|
||||
@ -61,14 +61,14 @@ pub async fn run_teammate_loop(
|
||||
&format!("队友 {} ({}) 已退出。", name, role),
|
||||
TeamMessageType::Status,
|
||||
);
|
||||
let _ = inbox::append_message(&team_dir, "lead", &goodbye);
|
||||
let _ = inbox::append_message(&team_dir, "lead", &goodbye).await;
|
||||
return;
|
||||
}
|
||||
|
||||
// ── IDLE 阶段：检查收件箱 ──
|
||||
*status.lock().await = MemberStatus::Idle;
|
||||
|
||||
let inbox_msgs = inbox::drain_inbox(&team_dir, &name);
|
||||
let inbox_msgs = inbox::drain_inbox(&team_dir, &name).await;
|
||||
|
||||
if inbox_msgs.is_empty() {
|
||||
// 等待新消息或取消，poll 间隔 5 秒，最长 60 秒
|
||||
@ -84,7 +84,7 @@ pub async fn run_teammate_loop(
|
||||
}
|
||||
|
||||
// 再次 drain（可能因为 pending 标志被唤醒）
|
||||
let inbox_msgs = inbox::drain_inbox(&team_dir, &name);
|
||||
let inbox_msgs = inbox::drain_inbox(&team_dir, &name).await;
|
||||
if inbox_msgs.is_empty() {
|
||||
continue; // 超时，没有新消息，继续 idle
|
||||
}
|
||||
@ -115,7 +115,7 @@ pub async fn run_teammate_loop(
|
||||
if let Some(ref summary) = result {
|
||||
let reply =
|
||||
super::inbox::TeamMessage::new(&name, "lead", summary, TeamMessageType::Result);
|
||||
let _ = inbox::append_message(&team_dir, "lead", &reply);
|
||||
let _ = inbox::append_message(&team_dir, "lead", &reply).await;
|
||||
info!(
|
||||
"[Teammate:{}] 任务完成，已发送结果 ({}字符)",
|
||||
name,
|
||||
@ -147,7 +147,7 @@ pub async fn run_teammate_loop(
|
||||
if let Some(ref summary) = result {
|
||||
let reply =
|
||||
super::inbox::TeamMessage::new(&name, "lead", summary, TeamMessageType::Result);
|
||||
let _ = inbox::append_message(&team_dir, "lead", &reply);
|
||||
let _ = inbox::append_message(&team_dir, "lead", &reply).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -44,7 +44,7 @@ impl AnalyzeImageTool {
|
||||
json!({
|
||||
"image_path": image_path,
|
||||
"question": question,
|
||||
"model": vision_llm.model(),
|
||||
"model": vision_llm.model().await,
|
||||
}),
|
||||
),
|
||||
Err(e) => {
|
||||
@ -237,7 +237,7 @@ mod tests {
|
||||
.filter(|e| matches!(e, AgentStreamEvent::TextDelta { .. }))
|
||||
.count();
|
||||
|
||||
println!("视觉模型: {}", vision_llm.model());
|
||||
println!("视觉模型: {}", vision_llm.model().await);
|
||||
println!("SSE 事件总数: {}, TextDelta: {}", events.len(), delta_count);
|
||||
println!("分析结果:\n{}", output.content);
|
||||
|
||||
|
||||
@ -253,7 +253,14 @@ impl AgentTool for GetCitationNetworkTool {
|
||||
|
||||
// 有 query → 引用查找模式：在关联文献中按作者+年份搜索
|
||||
if let Some(ref q) = query {
|
||||
match crate::services::citation::search_citations(&state.db, &paper.bibcode, direction, q).await {
|
||||
match crate::services::citation::search_citations(
|
||||
&state.db,
|
||||
&paper.bibcode,
|
||||
direction,
|
||||
q,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(results) => {
|
||||
if results.is_empty() {
|
||||
let dir_label = if direction == "citations" {
|
||||
@ -315,7 +322,14 @@ impl AgentTool for GetCitationNetworkTool {
|
||||
}
|
||||
} else {
|
||||
// 无 query → 分页浏览模式
|
||||
match crate::services::citation::get_citations_paginated(&state.db, &paper.bibcode, direction, sort, offset, limit)
|
||||
match crate::services::citation::get_citations_paginated(
|
||||
&state.db,
|
||||
&paper.bibcode,
|
||||
direction,
|
||||
sort,
|
||||
offset,
|
||||
limit,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok((rows, total)) => {
|
||||
|
||||
@ -11,9 +11,9 @@ pub mod target;
|
||||
pub mod vizier;
|
||||
|
||||
pub use library::{GetCitationNetworkTool, SearchLocalLibraryTool};
|
||||
pub use observation::FindObservationTool;
|
||||
pub use metadata::GetPaperMetadataTool;
|
||||
pub use note::SaveNoteTool;
|
||||
pub use observation::FindObservationTool;
|
||||
pub use paper::{GetPaperContentTool, GetPaperOutlineTool};
|
||||
pub use rag::RagSearchTool;
|
||||
pub use target::QueryTargetTool;
|
||||
|
||||
@ -58,6 +58,7 @@ impl AgentTool for SaveNoteTool {
|
||||
let state = &ctx.app_state;
|
||||
|
||||
match crate::services::note::save_note_service(&state.config.library_dir, &title, &content)
|
||||
.await
|
||||
{
|
||||
Ok((filename, filepath, size)) => ToolOutput::success(
|
||||
format!("笔记已保存: {}", filename),
|
||||
|
||||
@ -52,23 +52,26 @@ impl AgentTool for FindObservationTool {
|
||||
}
|
||||
|
||||
fn parameters(&self) -> serde_json::Value {
|
||||
// enum 列表从枚举的 valid_values() 动态生成，新增源/产品时自动同步
|
||||
let source_enum: Vec<&str> = Source::valid_values().to_vec();
|
||||
let product_enum: Vec<&str> = ProductType::valid_values().to_vec();
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"source": {
|
||||
"type": "string",
|
||||
"enum": ["lamost", "gaia", "sdss", "desi"],
|
||||
"enum": source_enum,
|
||||
"description": "数据源"
|
||||
},
|
||||
"product": {
|
||||
"type": "string",
|
||||
"enum": ["spectrum", "lightcurve", "photometry", "image"],
|
||||
"enum": product_enum,
|
||||
"description": "产品类型，默认 spectrum",
|
||||
"default": "spectrum"
|
||||
},
|
||||
"subtype": {
|
||||
"type": "string",
|
||||
"description": "产品子类型（可选）。LAMOST spectrum: lrs/mrs；Gaia spectrum: xp_continuous/xp_sampled/rvs；Gaia lightcurve: epoch_photometry；SDSS spectrum: spec/apstar/aspcap；DESI spectrum: coadd"
|
||||
"description": "产品子类型（可选）。各源可选值通过 GET /api/observation/capabilities 查询；LAMOST spectrum: lrs/mrs；Gaia spectrum: xp_continuous/xp_sampled/rvs；Gaia lightcurve: epoch_photometry；SDSS spectrum: spec/apstar/aspcap；DESI spectrum: coadd"
|
||||
},
|
||||
"ra": {
|
||||
"type": "number",
|
||||
@ -80,7 +83,7 @@ impl AgentTool for FindObservationTool {
|
||||
},
|
||||
"radius_deg": {
|
||||
"type": "number",
|
||||
"description": "检索半径（度），坐标模式用，默认 0.1。各源无硬性 API 上限，但建议值：LAMOST≤5°，Gaia/SDSS/DESI≤1°（主表很大，超出易超时）。硬性上限 30°，超出报错。可通过 GET /api/observation/capabilities 查询各源建议值",
|
||||
"description": "检索半径（度），坐标模式用，默认 0.1。建议值与硬上限可通过 GET /api/observation/capabilities 查询（LAMOST 建议≤5°，其他≤1°；硬上限 30°）",
|
||||
"default": 0.1
|
||||
},
|
||||
"strategy": {
|
||||
@ -96,7 +99,11 @@ impl AgentTool for FindObservationTool {
|
||||
},
|
||||
"release": {
|
||||
"type": "string",
|
||||
"description": "数据发布版本（可选，留空用各源默认）。可选值与默认值可通过 GET /api/observation/capabilities 查询；LAMOST: dr5..dr11（默认dr10，MRS 需 dr7+）；Gaia: dr3；SDSS spec: dr16/dr17（默认dr17）；SDSS apstar/aspcap: dr17；DESI: edr/dr1（默认dr1）"
|
||||
"description": "数据发布版本（可选，留空用各源默认）。可选值与默认值通过 GET /api/observation/capabilities 查询"
|
||||
},
|
||||
"version": {
|
||||
"type": "string",
|
||||
"description": "数据发布的子版本（可选，仅 LAMOST 有意义）。留空用该 DR 的默认（最新公开）子版本。可选值通过 GET /api/observation/capabilities 的 release_versions 字段查询"
|
||||
},
|
||||
"force": {
|
||||
"type": "boolean",
|
||||
@ -124,36 +131,41 @@ impl AgentTool for FindObservationTool {
|
||||
let state = &ctx.app_state;
|
||||
|
||||
let source = match args.get("source").and_then(|v| v.as_str()) {
|
||||
Some(s) => match s.to_lowercase().as_str() {
|
||||
"lamost" => Source::Lamost,
|
||||
"gaia" => Source::Gaia,
|
||||
"sdss" => Source::Sdss,
|
||||
"desi" => Source::Desi,
|
||||
other => return ToolOutput::error(format!(
|
||||
"不支持的 source '{}'，可选: {:?}", other, Source::valid_values()
|
||||
)),
|
||||
Some(s) => match Source::from_str(s) {
|
||||
Ok(src) => src,
|
||||
Err(e) => return ToolOutput::error(e),
|
||||
},
|
||||
None => return ToolOutput::error("缺少必需参数 'source'（lamost/gaia/sdss/desi）"),
|
||||
};
|
||||
let product_type = match args.get("product").and_then(|v| v.as_str()) {
|
||||
None | Some("spectrum") => ProductType::Spectrum,
|
||||
Some("lightcurve") | Some("light_curve") | Some("lc") => ProductType::LightCurve,
|
||||
Some("photometry") => ProductType::Photometry,
|
||||
Some("image") => ProductType::Image,
|
||||
Some(other) => return ToolOutput::error(format!(
|
||||
"不支持的 product '{}'，可选: {:?}", other, ProductType::valid_values()
|
||||
)),
|
||||
None => ProductType::Spectrum,
|
||||
Some(s) => match ProductType::from_str(s) {
|
||||
Ok(p) => p,
|
||||
Err(e) => return ToolOutput::error(e),
|
||||
},
|
||||
};
|
||||
let subtype = args.get("subtype")
|
||||
let subtype = args
|
||||
.get("subtype")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
let product = ProductSpec { product: product_type, subtype };
|
||||
let product = ProductSpec {
|
||||
product: product_type,
|
||||
subtype,
|
||||
};
|
||||
let force = args.get("force").and_then(|v| v.as_bool()).unwrap_or(false);
|
||||
let release = args.get("release").and_then(|v| v.as_str()).map(|s| s.to_string());
|
||||
let release = args
|
||||
.get("release")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
let version = args
|
||||
.get("version")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
|
||||
// 标识符模式 vs 坐标模式
|
||||
let request = if let Some(ids) = args.get("source_ids").and_then(|v| v.as_array()) {
|
||||
let identifiers: Vec<String> = ids.iter()
|
||||
let identifiers: Vec<String> = ids
|
||||
.iter()
|
||||
.filter_map(|v| v.as_str().map(|s| s.to_string()))
|
||||
.collect();
|
||||
if identifiers.is_empty() {
|
||||
@ -161,23 +173,38 @@ impl AgentTool for FindObservationTool {
|
||||
}
|
||||
info!(
|
||||
"[FindObservation] by_id source={:?} product={:?} count={}",
|
||||
source, product.product, identifiers.len()
|
||||
source,
|
||||
product.product,
|
||||
identifiers.len()
|
||||
);
|
||||
ObservationRequest::ByIdentifier { source, product, identifiers, release }
|
||||
ObservationRequest::ByIdentifier {
|
||||
source,
|
||||
product,
|
||||
identifiers,
|
||||
release,
|
||||
version,
|
||||
}
|
||||
} else {
|
||||
let ra = match args.get("ra").and_then(|v| v.as_f64()) {
|
||||
Some(v) => v,
|
||||
None => return ToolOutput::error(
|
||||
None => {
|
||||
return ToolOutput::error(
|
||||
"坐标模式缺少必需参数 'ra'（赤经，度），或改用 source_ids 标识符模式",
|
||||
),
|
||||
)
|
||||
}
|
||||
};
|
||||
let dec = match args.get("dec").and_then(|v| v.as_f64()) {
|
||||
Some(v) => v,
|
||||
None => return ToolOutput::error(
|
||||
None => {
|
||||
return ToolOutput::error(
|
||||
"坐标模式缺少必需参数 'dec'（赤纬，度），或改用 source_ids 标识符模式",
|
||||
),
|
||||
)
|
||||
}
|
||||
};
|
||||
let radius = args.get("radius_deg").and_then(|v| v.as_f64()).unwrap_or(0.1);
|
||||
let radius = args
|
||||
.get("radius_deg")
|
||||
.and_then(|v| v.as_f64())
|
||||
.unwrap_or(0.1);
|
||||
let strategy = match args.get("strategy").and_then(|v| v.as_str()) {
|
||||
Some("all") => FindStrategy::All,
|
||||
_ => FindStrategy::Nearest,
|
||||
@ -187,7 +214,14 @@ impl AgentTool for FindObservationTool {
|
||||
source, product.product, ra, dec, radius, strategy
|
||||
);
|
||||
ObservationRequest::ByCoordinates {
|
||||
source, product, ra, dec, radius_deg: radius, strategy, release,
|
||||
source,
|
||||
product,
|
||||
ra,
|
||||
dec,
|
||||
radius_deg: radius,
|
||||
strategy,
|
||||
release,
|
||||
version,
|
||||
}
|
||||
};
|
||||
|
||||
@ -203,7 +237,11 @@ impl AgentTool for FindObservationTool {
|
||||
|
||||
/// 渲染 ObservationBatch 为可读文本（支持多 artifact 展示）
|
||||
fn render_batch(b: &crate::services::observation::ObservationBatch) -> String {
|
||||
let mut content = format!("{} {} 下载", b.source.display(), b.product.product.display());
|
||||
let mut content = format!(
|
||||
"{} {} 下载",
|
||||
b.source.display(),
|
||||
b.product.product.display()
|
||||
);
|
||||
if let (Some(ra), Some(dec), Some(r)) = (b.ra, b.dec, b.radius_deg) {
|
||||
content.push_str(&format!("（ra={}, dec={}, radius={}°）", ra, dec, r));
|
||||
}
|
||||
@ -219,18 +257,30 @@ fn render_batch(b: &crate::services::observation::ObservationBatch) -> String {
|
||||
|
||||
for p in &b.products {
|
||||
// 多 artifact 展示（如 Gaia 光变 G/BP/RP 三波段）
|
||||
let artifact_summary: Vec<String> = p.artifacts.iter().map(|a| {
|
||||
match &a.band {
|
||||
Some(band) => format!("{}波段 {} ({})",
|
||||
band, a.file_format.to_uppercase(), format_size(a.size_bytes)),
|
||||
None => format!("{} ({})",
|
||||
a.file_format.to_uppercase(), format_size(a.size_bytes)),
|
||||
}
|
||||
}).collect();
|
||||
let artifact_summary: Vec<String> = p
|
||||
.artifacts
|
||||
.iter()
|
||||
.map(|a| match &a.band {
|
||||
Some(band) => format!(
|
||||
"{}波段 {} ({})",
|
||||
band,
|
||||
a.file_format.to_uppercase(),
|
||||
format_size(a.size_bytes)
|
||||
),
|
||||
None => format!(
|
||||
"{} ({})",
|
||||
a.file_format.to_uppercase(),
|
||||
format_size(a.size_bytes)
|
||||
),
|
||||
})
|
||||
.collect();
|
||||
content.push_str(&format!(
|
||||
"\n✓ 已下载（{}）: {} —— {}\n",
|
||||
if p.artifacts.iter().all(|a| a.cached) { "缓存" } else { "新下载" }
|
||||
.to_string(),
|
||||
if p.artifacts.iter().all(|a| a.cached) {
|
||||
"缓存"
|
||||
} else {
|
||||
"新下载"
|
||||
},
|
||||
p.source_label,
|
||||
artifact_summary.join(", ")
|
||||
));
|
||||
@ -238,7 +288,10 @@ fn render_batch(b: &crate::services::observation::ObservationBatch) -> String {
|
||||
content.push_str(&format!(
|
||||
" {}{}\n",
|
||||
a.file_path,
|
||||
a.band.as_ref().map(|b| format!(" [{}]", b)).unwrap_or_default()
|
||||
a.band
|
||||
.as_ref()
|
||||
.map(|b| format!(" [{}]", b))
|
||||
.unwrap_or_default()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@ -185,10 +185,7 @@ impl AgentTool for CatalogOperationTool {
|
||||
}
|
||||
}
|
||||
|
||||
async fn do_catalog_search(
|
||||
state: &crate::api::AppState,
|
||||
args: &serde_json::Value,
|
||||
) -> ToolOutput {
|
||||
async fn do_catalog_search(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
|
||||
let keyword = match args.get("keyword").and_then(|v| v.as_str()) {
|
||||
Some(k) => k,
|
||||
None => return ToolOutput::error("search 需要 'keyword' 参数"),
|
||||
@ -226,10 +223,7 @@ async fn do_catalog_search(
|
||||
ToolOutput::success(content, json!(items))
|
||||
}
|
||||
|
||||
async fn do_catalog_describe(
|
||||
state: &crate::api::AppState,
|
||||
args: &serde_json::Value,
|
||||
) -> ToolOutput {
|
||||
async fn do_catalog_describe(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
|
||||
let table = match args.get("table").and_then(|v| v.as_str()) {
|
||||
Some(t) => t,
|
||||
None => return ToolOutput::error("describe 需要 'table' 参数"),
|
||||
@ -259,10 +253,7 @@ async fn do_catalog_describe(
|
||||
ToolOutput::success(content, json!(items))
|
||||
}
|
||||
|
||||
async fn do_catalog_query(
|
||||
state: &crate::api::AppState,
|
||||
args: &serde_json::Value,
|
||||
) -> ToolOutput {
|
||||
async fn do_catalog_query(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
|
||||
let limit = args
|
||||
.get("limit")
|
||||
.and_then(|v| v.as_i64())
|
||||
@ -275,8 +266,7 @@ async fn do_catalog_query(
|
||||
limit,
|
||||
adql.chars().take(150).collect::<String>()
|
||||
);
|
||||
crate::services::cds::vizier::query_adql_cached(&state.db, &state.vizier, adql, limit)
|
||||
.await
|
||||
crate::services::cds::vizier::query_adql_cached(&state.db, &state.vizier, adql, limit).await
|
||||
} else if let Some(table) = args.get("table").and_then(|v| v.as_str()) {
|
||||
let columns: Vec<String> = args
|
||||
.get("columns")
|
||||
@ -284,13 +274,7 @@ async fn do_catalog_query(
|
||||
.map(|c| c.split(',').map(|s| s.trim().to_string()).collect())
|
||||
.unwrap_or_default();
|
||||
info!("[CatalogOp:query] table={} limit={}", table, limit);
|
||||
crate::services::cds::vizier::query_table(
|
||||
&state.db,
|
||||
&state.vizier,
|
||||
table,
|
||||
&columns,
|
||||
limit,
|
||||
)
|
||||
crate::services::cds::vizier::query_table(&state.db, &state.vizier, table, &columns, limit)
|
||||
.await
|
||||
} else {
|
||||
return ToolOutput::error("query 需要 'adql' 或 'table' 参数之一");
|
||||
@ -305,10 +289,7 @@ async fn do_catalog_query(
|
||||
}
|
||||
}
|
||||
|
||||
async fn do_catalog_cone(
|
||||
state: &crate::api::AppState,
|
||||
args: &serde_json::Value,
|
||||
) -> ToolOutput {
|
||||
async fn do_catalog_cone(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
|
||||
let coords = match args.get("coords").and_then(|v| v.as_object()) {
|
||||
Some(c) => c,
|
||||
None => return ToolOutput::error("cone 需要 'coords' 参数（含 ra/dec）"),
|
||||
@ -338,7 +319,11 @@ async fn do_catalog_cone(
|
||||
|
||||
info!(
|
||||
"[CatalogOp:cone] ra={} dec={} radius={}° table={} strategy={}",
|
||||
ra, dec, radius, table, if nearest { "nearest" } else { "all" }
|
||||
ra,
|
||||
dec,
|
||||
radius,
|
||||
table,
|
||||
if nearest { "nearest" } else { "all" }
|
||||
);
|
||||
|
||||
match crate::services::cds::vizier::cone_search(
|
||||
@ -365,10 +350,7 @@ async fn do_catalog_cone(
|
||||
}
|
||||
}
|
||||
|
||||
async fn do_catalog_export(
|
||||
state: &crate::api::AppState,
|
||||
args: &serde_json::Value,
|
||||
) -> ToolOutput {
|
||||
async fn do_catalog_export(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
|
||||
let limit = args
|
||||
.get("limit")
|
||||
.and_then(|v| v.as_i64())
|
||||
@ -384,13 +366,16 @@ async fn do_catalog_export(
|
||||
}
|
||||
|
||||
let catalog = crate::services::cds::vizier::VizierCatalog::new(&state.db, &state.vizier);
|
||||
let result = match catalog.export_to_file(
|
||||
let result = match catalog
|
||||
.export_to_file(
|
||||
state.config.library_dir.to_str().unwrap_or("."),
|
||||
adql,
|
||||
table,
|
||||
columns,
|
||||
limit,
|
||||
).await {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => return ToolOutput::error(format!("导出失败: {}", e)),
|
||||
};
|
||||
@ -410,10 +395,7 @@ async fn do_catalog_export(
|
||||
)
|
||||
}
|
||||
|
||||
async fn do_catalog_lookup(
|
||||
state: &crate::api::AppState,
|
||||
args: &serde_json::Value,
|
||||
) -> ToolOutput {
|
||||
async fn do_catalog_lookup(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
|
||||
let bibcode = match args.get("bibcode").and_then(|v| v.as_str()) {
|
||||
Some(b) => b,
|
||||
None => return ToolOutput::error("lookup 需要 'bibcode' 参数"),
|
||||
@ -424,11 +406,8 @@ async fn do_catalog_lookup(
|
||||
.unwrap_or(10)
|
||||
.clamp(1, 30) as usize;
|
||||
|
||||
let catalog = crate::services::cds::vizier::VizierCatalog::with_ads(
|
||||
&state.db,
|
||||
&state.vizier,
|
||||
&state.ads,
|
||||
);
|
||||
let catalog =
|
||||
crate::services::cds::vizier::VizierCatalog::with_ads(&state.db, &state.vizier, &state.ads);
|
||||
let results = match catalog.lookup(bibcode, limit).await {
|
||||
Ok(r) => r,
|
||||
Err(e) => return ToolOutput::error(format!("查找失败: {}", e)),
|
||||
|
||||
@ -245,7 +245,12 @@ impl AgentTool for FileEditTool {
|
||||
);
|
||||
}
|
||||
|
||||
let original = match std::fs::read_to_string(&file_path) {
|
||||
let read_path = file_path.clone();
|
||||
let original =
|
||||
match tokio::task::spawn_blocking(move || std::fs::read_to_string(&read_path))
|
||||
.await
|
||||
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
|
||||
{
|
||||
Ok(c) => c,
|
||||
Err(e) => return ToolOutput::error(format!("读取文件失败: {}", e)),
|
||||
};
|
||||
@ -317,7 +322,12 @@ impl AgentTool for FileEditTool {
|
||||
);
|
||||
}
|
||||
|
||||
match std::fs::write(&file_path, &modified) {
|
||||
let write_path = file_path.clone();
|
||||
let write_content = modified.clone();
|
||||
match tokio::task::spawn_blocking(move || std::fs::write(&write_path, &write_content))
|
||||
.await
|
||||
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
|
||||
{
|
||||
Ok(()) => {
|
||||
let count = if replace_all { match_count } else { 1 };
|
||||
info!("[FileEdit] {} 处替换: {}", count, file_path.display());
|
||||
|
||||
@ -89,20 +89,27 @@ impl AgentTool for GrepFilesTool {
|
||||
Err(e) => return ToolOutput::error(format!("正则表达式无效: {}", e)),
|
||||
};
|
||||
|
||||
let search_re = re.clone();
|
||||
let search_inc = include_pattern.map(|s| s.to_string());
|
||||
let search_p = search_path.clone();
|
||||
let (results, count) = tokio::task::spawn_blocking(move || {
|
||||
let mut results = Vec::new();
|
||||
let mut count = 0;
|
||||
|
||||
if search_path.is_file() {
|
||||
count += Self::search_file(&search_path, &re, &mut results);
|
||||
} else if search_path.is_dir() {
|
||||
if search_p.is_file() {
|
||||
count += Self::search_file(&search_p, &search_re, &mut results);
|
||||
} else if search_p.is_dir() {
|
||||
count += Self::search_dir(
|
||||
&search_path,
|
||||
&re,
|
||||
include_pattern,
|
||||
&search_p,
|
||||
&search_re,
|
||||
search_inc.as_deref(),
|
||||
max_results,
|
||||
&mut results,
|
||||
);
|
||||
}
|
||||
(results, count)
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| (Vec::new(), 0));
|
||||
|
||||
if results.is_empty() {
|
||||
ToolOutput::success(
|
||||
|
||||
@ -113,18 +113,25 @@ impl AgentTool for ReadFileTool {
|
||||
}
|
||||
// ── 去重检查结束 ──
|
||||
|
||||
// 预检文件大小，避免 OOM
|
||||
// 预检文件大小 + 读取文件内容（spawn_blocking 避免阻塞 tokio）
|
||||
const MAX_READ_BYTES: u64 = 10 * 1024 * 1024; // 10MB
|
||||
if let Ok(meta) = std::fs::metadata(&path) {
|
||||
if meta.len() > MAX_READ_BYTES {
|
||||
return ToolOutput::error(format!(
|
||||
let read_path = path.clone();
|
||||
let read_result = tokio::task::spawn_blocking(move || {
|
||||
let meta = std::fs::metadata(&read_path).ok();
|
||||
if let Some(ref m) = meta {
|
||||
if m.len() > MAX_READ_BYTES {
|
||||
return Err(format!(
|
||||
"文件过大 ({} MB)，超过 10MB 限制。请使用 read_file 的 offset/limit 参数分段读取。",
|
||||
meta.len() / (1024 * 1024)
|
||||
m.len() / (1024 * 1024)
|
||||
));
|
||||
}
|
||||
}
|
||||
std::fs::read_to_string(&read_path).map_err(|e| format!("读取文件失败: {}", e))
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|e| Err(format!("任务执行失败: {}", e)));
|
||||
|
||||
match std::fs::read_to_string(&path) {
|
||||
match read_result {
|
||||
Ok(content) => {
|
||||
let total_lines = content.lines().count();
|
||||
let truncated = if max_lines < total_lines {
|
||||
@ -141,8 +148,6 @@ impl AgentTool for ReadFileTool {
|
||||
content.clone()
|
||||
};
|
||||
|
||||
// read_file 不截断输出——LLM 明确要求读取完整内容，
|
||||
// 且 skip_persist 确保不会被二次持久化。
|
||||
let content_len = truncated.len();
|
||||
info!(
|
||||
"[ReadFile] 读取 {}: {} 字符 ({} 行)",
|
||||
@ -163,14 +168,13 @@ impl AgentTool for ReadFileTool {
|
||||
},
|
||||
);
|
||||
}
|
||||
// ── 缓存更新结束 ──
|
||||
|
||||
ToolOutput::success_skip_persist(
|
||||
truncated,
|
||||
json!({"file_path": file_path_str, "total_lines": total_lines}),
|
||||
)
|
||||
}
|
||||
Err(e) => ToolOutput::error(format!("读取文件失败: {}", e)),
|
||||
Err(e) => ToolOutput::error(e),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -69,12 +69,21 @@ impl AgentTool for FileWriteTool {
|
||||
}
|
||||
|
||||
if let Some(parent) = path.parent() {
|
||||
if let Err(e) = std::fs::create_dir_all(parent) {
|
||||
let parent = parent.to_path_buf();
|
||||
if let Err(e) = tokio::task::spawn_blocking(move || std::fs::create_dir_all(&parent))
|
||||
.await
|
||||
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
|
||||
{
|
||||
return ToolOutput::error(format!("创建父目录失败: {}", e));
|
||||
}
|
||||
}
|
||||
|
||||
match std::fs::write(&path, &content) {
|
||||
let write_path = path.clone();
|
||||
let write_content = content.clone();
|
||||
match tokio::task::spawn_blocking(move || std::fs::write(&write_path, &write_content))
|
||||
.await
|
||||
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
|
||||
{
|
||||
Ok(_) => {
|
||||
let line_count = content.lines().count();
|
||||
info!("[FileWrite] 写入 {}: {} 行", file_path_str, line_count);
|
||||
|
||||
@ -17,7 +17,7 @@ use tracing::info;
|
||||
///
|
||||
/// 返回 (最终内容, 持久化文件路径)。
|
||||
/// 如果内容未超过限制，直接返回原内容且持久化路径为 None。
|
||||
pub fn maybe_persist_tool_result(
|
||||
pub async fn maybe_persist_tool_result(
|
||||
content: &str,
|
||||
tool_call_id: &str,
|
||||
max_chars: usize,
|
||||
@ -28,8 +28,12 @@ pub fn maybe_persist_tool_result(
|
||||
}
|
||||
|
||||
// 创建目录（幂等）
|
||||
if let Err(_e) = std::fs::create_dir_all(tool_results_dir) {
|
||||
// 无法创建目录则直接截断（不持久化）
|
||||
let dir = tool_results_dir.to_path_buf();
|
||||
if tokio::task::spawn_blocking(move || std::fs::create_dir_all(&dir))
|
||||
.await
|
||||
.unwrap_or(Ok(()))
|
||||
.is_err()
|
||||
{
|
||||
let truncated: String = content.chars().take(max_chars).collect();
|
||||
return (
|
||||
format!(
|
||||
@ -60,19 +64,20 @@ pub fn maybe_persist_tool_result(
|
||||
let file_path = tool_results_dir.join(format!("{}.txt", safe_id));
|
||||
|
||||
// 独占创建——只在文件不存在时写入，保证幂等性
|
||||
let written = match std::fs::OpenOptions::new()
|
||||
let fp = file_path.clone();
|
||||
let content_clone = content.to_string();
|
||||
let written = tokio::task::spawn_blocking(move || {
|
||||
match std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&file_path)
|
||||
.open(&fp)
|
||||
{
|
||||
Ok(_) => {
|
||||
// 文件创建成功，写入内容
|
||||
match std::fs::write(&file_path, content) {
|
||||
Ok(_) => match std::fs::write(&fp, &content_clone) {
|
||||
Ok(_) => {
|
||||
info!(
|
||||
"[Persist] 工具结果已持久化: {} ({} 字符)",
|
||||
file_path.display(),
|
||||
content.len()
|
||||
fp.display(),
|
||||
content_clone.len()
|
||||
);
|
||||
true
|
||||
}
|
||||
@ -80,18 +85,19 @@ pub fn maybe_persist_tool_result(
|
||||
tracing::warn!("[Persist] 写入失败: {}", e);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
|
||||
// 文件已存在，无需重复写入
|
||||
info!("[Persist] 工具结果已存在，跳过: {}", file_path.display());
|
||||
info!("[Persist] 工具结果已存在，跳过: {}", fp.display());
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("[Persist] 创建文件失败: {}", e);
|
||||
false
|
||||
}
|
||||
};
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
|
||||
if !written {
|
||||
// 持久化失败，回退到截断
|
||||
@ -148,19 +154,19 @@ mod tests {
|
||||
(dir, Cleanup(dir_clone))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_small_content_not_persisted() {
|
||||
#[tokio::test]
|
||||
async fn test_small_content_not_persisted() {
|
||||
let (dir, _cleanup) = temp_dir();
|
||||
let (content, path) = maybe_persist_tool_result("small result", "call_1", 4000, &dir);
|
||||
let (content, path) = maybe_persist_tool_result("small result", "call_1", 4000, &dir).await;
|
||||
assert_eq!(content, "small result");
|
||||
assert!(path.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_large_content_persisted() {
|
||||
#[tokio::test]
|
||||
async fn test_large_content_persisted() {
|
||||
let (dir, _cleanup) = temp_dir();
|
||||
let large = "x".repeat(5000);
|
||||
let (content, path) = maybe_persist_tool_result(&large, "call_2", 100, &dir);
|
||||
let (content, path) = maybe_persist_tool_result(&large, "call_2", 100, &dir).await;
|
||||
assert!(content.contains("<persisted-output>"));
|
||||
assert!(content.contains("call_2.txt"));
|
||||
assert!(path.is_some());
|
||||
@ -170,14 +176,14 @@ mod tests {
|
||||
assert_eq!(written, large);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_idempotent_write() {
|
||||
#[tokio::test]
|
||||
async fn test_idempotent_write() {
|
||||
let (dir, _cleanup) = temp_dir();
|
||||
let large1 = "a".repeat(5000);
|
||||
let large2 = "b".repeat(5000);
|
||||
|
||||
let (_, path1) = maybe_persist_tool_result(&large1, "call_3", 100, &dir);
|
||||
let (_content2, path2) = maybe_persist_tool_result(&large2, "call_3", 100, &dir);
|
||||
let (_, path1) = maybe_persist_tool_result(&large1, "call_3", 100, &dir).await;
|
||||
let (_content2, path2) = maybe_persist_tool_result(&large2, "call_3", 100, &dir).await;
|
||||
|
||||
assert!(path1.is_some());
|
||||
assert!(path2.is_some());
|
||||
@ -185,11 +191,11 @@ mod tests {
|
||||
assert_eq!(written, large1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_preview_at_newline_boundary() {
|
||||
#[tokio::test]
|
||||
async fn test_preview_at_newline_boundary() {
|
||||
let (dir, _cleanup) = temp_dir();
|
||||
let large = format!("Short line\n{}", "x".repeat(5000));
|
||||
let (content, _) = maybe_persist_tool_result(&large, "call_4", 100, &dir);
|
||||
let (content, _) = maybe_persist_tool_result(&large, "call_4", 100, &dir).await;
|
||||
let preview_start = content.find("preview:").unwrap();
|
||||
let preview_section = &content[preview_start..];
|
||||
assert!(preview_section.contains("Short line"));
|
||||
|
||||
@ -151,7 +151,8 @@ impl AgentTool for SendTeammateMessageTool {
|
||||
let tm_lock = self.team_manager.lock().await;
|
||||
match tm_lock.as_ref() {
|
||||
Some(tm) => {
|
||||
tm.send_message("lead", &to, &content, TeamMessageType::Task);
|
||||
tm.send_message("lead", &to, &content, TeamMessageType::Task)
|
||||
.await;
|
||||
ToolOutput::success(
|
||||
format!(
|
||||
"📤 消息已发送给 {}: {}",
|
||||
@ -214,7 +215,7 @@ impl AgentTool for TeamBroadcastTool {
|
||||
let tm_lock = self.team_manager.lock().await;
|
||||
match tm_lock.as_ref() {
|
||||
Some(tm) => {
|
||||
tm.broadcast("lead", &content);
|
||||
tm.broadcast("lead", &content).await;
|
||||
ToolOutput::success("📢 已广播消息给所有队友。", json!({}))
|
||||
}
|
||||
None => ToolOutput::error("团队管理器未初始化"),
|
||||
@ -278,7 +279,7 @@ impl AgentTool for CheckTeamInboxTool {
|
||||
let tm_lock = self.team_manager.lock().await;
|
||||
match tm_lock.as_ref() {
|
||||
Some(tm) => {
|
||||
let msgs = tm.check_inbox(agent);
|
||||
let msgs = tm.check_inbox(agent).await;
|
||||
if msgs.is_empty() {
|
||||
ToolOutput::success(
|
||||
format!("📭 {} 的收件箱为空。", agent),
|
||||
|
||||
@ -259,10 +259,7 @@ mod tests {
|
||||
ads,
|
||||
arxiv,
|
||||
vizier,
|
||||
lamost: crate::clients::lamost::LamostClient::new(
|
||||
"https://www.lamost.org",
|
||||
60,
|
||||
)
|
||||
lamost: crate::clients::lamost::LamostClient::new("https://www.lamost.org", 60)
|
||||
.unwrap(),
|
||||
gaia: crate::clients::gaia::GaiaClient::new(
|
||||
"https://gea.esac.esa.int/tap-server/tap",
|
||||
@ -290,7 +287,7 @@ mod tests {
|
||||
harvest_status: Arc::new(tokio::sync::Mutex::new(MetaSyncStatus::default())),
|
||||
batch_status: Arc::new(tokio::sync::Mutex::new(AssetBatchStatus::default())),
|
||||
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
|
||||
cancelled_runs: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())),
|
||||
cancelled_runs: Arc::new(dashmap::DashMap::new()),
|
||||
skill_registry: Arc::new(tokio::sync::RwLock::new(
|
||||
crate::agent::skills::SkillRegistry::new(PathBuf::from("/tmp/test_skills")),
|
||||
)),
|
||||
@ -298,16 +295,16 @@ mod tests {
|
||||
pending_permissions: Arc::new(
|
||||
tokio::sync::Mutex::new(std::collections::HashMap::new()),
|
||||
),
|
||||
session_permission_checkers: Arc::new(tokio::sync::RwLock::new(
|
||||
std::collections::HashMap::new(),
|
||||
)),
|
||||
session_permission_checkers: Arc::new(dashmap::DashMap::new()),
|
||||
sse_broadcast: None,
|
||||
memory_manager: Arc::new(tokio::sync::Mutex::new(
|
||||
crate::agent::memory::MemoryManager::new(PathBuf::from("/tmp/test_memory")),
|
||||
)),
|
||||
sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
|
||||
login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
||||
upload_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
||||
upload_rate_limiter: Arc::new(
|
||||
tokio::sync::Mutex::new(std::collections::HashMap::new()),
|
||||
),
|
||||
});
|
||||
|
||||
ToolContext {
|
||||
|
||||
@ -299,8 +299,7 @@ pub async fn stop_agent(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Path(session_id): Path<String>,
|
||||
) -> ApiResult<Json<serde_json::Value>> {
|
||||
let mut cancelled = state.cancelled_runs.lock().await;
|
||||
cancelled.insert(session_id.clone());
|
||||
state.cancelled_runs.insert(session_id.clone(), ());
|
||||
info!("已接收并记录手动中止请求，会话 ID: {}", session_id);
|
||||
Ok(Json(
|
||||
serde_json::json!({ "status": "stopping", "session_id": session_id }),
|
||||
@ -412,7 +411,12 @@ pub async fn respond_permission(
|
||||
|
||||
match perm_id {
|
||||
Some(id) => {
|
||||
let perm = perms.remove(&id).unwrap();
|
||||
let perm = match perms.remove(&id) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
return Err(AppError::internal("权限请求在查找后消失"));
|
||||
}
|
||||
};
|
||||
match perm.response_tx.send(req) {
|
||||
Ok(()) => {
|
||||
info!(
|
||||
|
||||
@ -124,14 +124,9 @@ pub async fn login(
|
||||
let mut limiter = state.login_rate_limiter.lock().await;
|
||||
// 批量清理过期条目，防止内存无限增长
|
||||
limiter.retain(|_, (_, first_fail)| first_fail.elapsed().as_secs() < WINDOW_SECS);
|
||||
// 容量保护：超过上限时强制清理最旧的一半条目
|
||||
// 容量保护：超过上限时直接清空（比 O(n log n) 排序更高效，且不影响安全性）
|
||||
if limiter.len() > MAX_RATE_LIMITER_ENTRIES {
|
||||
let mut entries: Vec<_> = limiter.iter().map(|(k, v)| (k.clone(), v.1)).collect();
|
||||
entries.sort_by_key(|(_, t)| *t);
|
||||
let cutoff = entries.len() / 2;
|
||||
for (key, _) in &entries[..cutoff] {
|
||||
limiter.remove(key);
|
||||
}
|
||||
limiter.clear();
|
||||
}
|
||||
|
||||
if let Some((count, first_fail)) = limiter.get(&client_ip) {
|
||||
@ -242,16 +237,42 @@ pub async fn auth_middleware(
|
||||
let token = extract_auth_token_from_headers(req.headers());
|
||||
|
||||
let mut is_valid = false;
|
||||
if let Some(tok) = token {
|
||||
if let Some(ref tok) = token {
|
||||
// 1. 优先校验内存中的动态 Session
|
||||
// 先用读锁快速检查会话是否存在，并获取其最后活跃时间
|
||||
let mut last_active_opt = None;
|
||||
{
|
||||
let sessions = state.sessions.read().await;
|
||||
if let Some(last_active) = sessions.get(tok) {
|
||||
is_valid = true;
|
||||
last_active_opt = Some(*last_active);
|
||||
}
|
||||
}
|
||||
|
||||
// 会话存在时，如果最后活跃时间超过 60 秒，则获取写锁更新最后活跃时间 + 过期清理
|
||||
if is_valid {
|
||||
let needs_update = match last_active_opt {
|
||||
Some(last_active) => {
|
||||
if let Ok(dur) = chrono::Utc::now()
|
||||
.signed_duration_since(last_active)
|
||||
.to_std()
|
||||
{
|
||||
dur.as_secs() > 60
|
||||
} else {
|
||||
true // 时钟回拨等异常情况，强制更新
|
||||
}
|
||||
}
|
||||
None => true,
|
||||
};
|
||||
|
||||
if needs_update {
|
||||
let mut sessions = state.sessions.write().await;
|
||||
// 顺便执行过期会话自动清理垃圾收集
|
||||
prune_expired_sessions(&mut sessions);
|
||||
|
||||
if let Some(last_active) = sessions.get_mut(&tok) {
|
||||
// 会话存在且未过期，更新最后活跃时间
|
||||
if let Some(last_active) = sessions.get_mut(tok) {
|
||||
*last_active = chrono::Utc::now();
|
||||
is_valid = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. 校验书签专属永久 API 密钥（严格隔离限制：仅允许用于特定的书签导入和同步路径）
|
||||
@ -259,7 +280,7 @@ pub async fn auth_middleware(
|
||||
let path = req.uri().path();
|
||||
if path == "/api/upload" || path == "/api/active_bibcode" {
|
||||
let expected_key = get_bookmarklet_api_key(&state.config.admin_password);
|
||||
if tok == expected_key {
|
||||
if *tok == expected_key {
|
||||
is_valid = true;
|
||||
}
|
||||
}
|
||||
|
||||
@ -85,18 +85,15 @@ pub struct AppState {
|
||||
pub harvest_status: Arc<tokio::sync::Mutex<crate::services::batch::MetaSyncStatus>>,
|
||||
pub batch_status: Arc<tokio::sync::Mutex<crate::services::batch::AssetBatchStatus>>,
|
||||
pub active_bibcode: Arc<tokio::sync::Mutex<Option<String>>>,
|
||||
pub cancelled_runs: Arc<Mutex<std::collections::HashSet<String>>>,
|
||||
pub cancelled_runs: Arc<dashmap::DashMap<String, ()>>,
|
||||
pub skill_registry: Arc<RwLock<SkillRegistry>>,
|
||||
/// ask_user 工具 — 待回答的问题
|
||||
pub pending_questions: Arc<Mutex<HashMap<String, PendingQuestion>>>,
|
||||
/// 权限检查 — 待处理的权限确认请求
|
||||
pub pending_permissions: Arc<Mutex<HashMap<String, PendingPermission>>>,
|
||||
/// 会话级权限检查器注册表（按 session_id 隔离，支持 API 动态添加/移除规则）
|
||||
pub session_permission_checkers: Arc<
|
||||
RwLock<
|
||||
std::collections::HashMap<String, crate::agent::runtime::permission::PermissionChecker>,
|
||||
>,
|
||||
>,
|
||||
pub session_permission_checkers:
|
||||
Arc<dashmap::DashMap<String, crate::agent::runtime::permission::PermissionChecker>>,
|
||||
/// SSE 广播通道（agent 运行时向所有连接的客户端推送事件）
|
||||
pub sse_broadcast: Option<broadcast::Sender<AppEvent>>,
|
||||
/// 项目记忆管理器（跨会话持久化）
|
||||
|
||||
@ -31,47 +31,15 @@ fn default_radius() -> f64 {
|
||||
0.1
|
||||
}
|
||||
|
||||
fn parse_source(s: &str) -> Result<crate::services::observation::Source, AppError> {
|
||||
use crate::services::observation::Source;
|
||||
Ok(match s.to_lowercase().as_str() {
|
||||
"lamost" => Source::Lamost,
|
||||
"gaia" => Source::Gaia,
|
||||
"sdss" => Source::Sdss,
|
||||
"desi" => Source::Desi,
|
||||
other => {
|
||||
return Err(AppError::bad_request(format!(
|
||||
"不支持的 source '{}'，可选: {:?}",
|
||||
other,
|
||||
Source::valid_values()
|
||||
)))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_product(s: &str) -> Result<crate::services::observation::ProductType, AppError> {
|
||||
use crate::services::observation::ProductType;
|
||||
Ok(match s.to_lowercase().as_str() {
|
||||
"spectrum" => ProductType::Spectrum,
|
||||
"lightcurve" | "light_curve" | "lc" => ProductType::LightCurve,
|
||||
"photometry" => ProductType::Photometry,
|
||||
"image" => ProductType::Image,
|
||||
other => {
|
||||
return Err(AppError::bad_request(format!(
|
||||
"不支持的 product '{}'，可选: {:?}",
|
||||
other,
|
||||
ProductType::valid_values()
|
||||
)))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// 把 (source, product, subtype) 字符串参数聚合为 ProductSpec
|
||||
/// 把 (source, product, subtype) 字符串参数聚合为 ProductSpec。
|
||||
/// source/product 的解析统一委托给枚举的 from_str() 方法，不再重复 match。
|
||||
fn build_product_spec(
|
||||
product: &str,
|
||||
subtype: &Option<String>,
|
||||
) -> Result<crate::services::observation::ProductSpec, AppError> {
|
||||
Ok(crate::services::observation::ProductSpec {
|
||||
product: parse_product(product)?,
|
||||
product: crate::services::observation::ProductType::from_str(product)
|
||||
.map_err(AppError::bad_request)?,
|
||||
subtype: subtype.clone(),
|
||||
})
|
||||
}
|
||||
@ -91,13 +59,16 @@ pub struct ObservationSearchParams {
|
||||
#[serde(default = "default_radius")]
|
||||
pub radius: f64,
|
||||
pub release: Option<String>,
|
||||
/// 数据发布的子版本（仅 LAMOST 有意义，如 "v2.0"）；None 时用该 DR 的默认版本
|
||||
pub version: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn observation_search(
|
||||
State(state): State<std::sync::Arc<AppState>>,
|
||||
Query(params): Query<ObservationSearchParams>,
|
||||
) -> ApiResult<Json<Vec<crate::services::observation::Candidate>>> {
|
||||
let source = parse_source(¶ms.source)?;
|
||||
let source = crate::services::observation::Source::from_str(¶ms.source)
|
||||
.map_err(AppError::bad_request)?;
|
||||
let product = build_product_spec(¶ms.product, ¶ms.subtype)?;
|
||||
|
||||
let candidates = crate::services::observation::search_observation(
|
||||
@ -109,6 +80,7 @@ pub async fn observation_search(
|
||||
params.dec,
|
||||
params.radius,
|
||||
params.release.as_deref(),
|
||||
params.version.as_deref(),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
@ -140,6 +112,8 @@ pub struct ObservationDownloadRequest {
|
||||
pub product: String,
|
||||
pub subtype: Option<String>,
|
||||
pub release: Option<String>,
|
||||
/// 数据发布的子版本（仅 LAMOST 有意义，如 "v2.0"）；None 时用该 DR 的默认版本
|
||||
pub version: Option<String>,
|
||||
#[serde(default)]
|
||||
pub force: bool,
|
||||
#[serde(flatten)]
|
||||
@ -168,7 +142,8 @@ pub async fn observation_download(
|
||||
) -> ApiResult<Json<crate::services::observation::ObservationBatch>> {
|
||||
use crate::services::observation::{download_observation, ObservationRequest};
|
||||
|
||||
let source = parse_source(&req.source)?;
|
||||
let source = crate::services::observation::Source::from_str(&req.source)
|
||||
.map_err(AppError::bad_request)?;
|
||||
let product = build_product_spec(&req.product, &req.subtype)?;
|
||||
|
||||
let request = match req.mode {
|
||||
@ -185,6 +160,7 @@ pub async fn observation_download(
|
||||
radius_deg,
|
||||
strategy,
|
||||
release: req.release.clone(),
|
||||
version: req.version.clone(),
|
||||
},
|
||||
DownloadMode::Identifiers { source_ids } => {
|
||||
let identifiers: Vec<String> = source_ids
|
||||
@ -200,6 +176,7 @@ pub async fn observation_download(
|
||||
product,
|
||||
identifiers,
|
||||
release: req.release.clone(),
|
||||
version: req.version.clone(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@ -350,6 +350,11 @@ pub async fn upload_paper_file(
|
||||
return Err(AppError::bad_request("bibcode 包含非法字符"));
|
||||
}
|
||||
|
||||
// 防御性安全边界：虽然合法 bibcode 允许双点（..），旧 arXiv 允许斜杠（/），但二者不能共存。
|
||||
if bibcode.contains("..") && (bibcode.contains('/') || bibcode.contains('\\')) {
|
||||
return Err(AppError::bad_request("bibcode 包含非法的路径遍历字符"));
|
||||
}
|
||||
|
||||
let updated_paper = crate::services::paper::upload_paper_file_service(
|
||||
&state.db,
|
||||
&state.config.library_dir,
|
||||
|
||||
@ -130,19 +130,19 @@ pub async fn update_permission_rules(
|
||||
},
|
||||
};
|
||||
|
||||
let mut checkers = state.session_permission_checkers.write().await;
|
||||
let checker = checkers
|
||||
let mut entry = state
|
||||
.session_permission_checkers
|
||||
.entry(session_id.clone())
|
||||
.or_insert_with(PermissionChecker::new);
|
||||
checker.add_rule_dynamic(session_rule);
|
||||
entry.add_rule_dynamic(session_rule);
|
||||
Json(PermissionActionResponse {
|
||||
success: true,
|
||||
message: format!("已添加 {}:{} 规则", req.kind, req.rule),
|
||||
})
|
||||
}
|
||||
"remove" => {
|
||||
let mut checkers = state.session_permission_checkers.write().await;
|
||||
let removed = if let Some(checker) = checkers.get_mut(&session_id) {
|
||||
let removed =
|
||||
if let Some(mut checker) = state.session_permission_checkers.get_mut(&session_id) {
|
||||
checker.remove_rule_dynamic(&req.rule, &req.kind)
|
||||
} else {
|
||||
0
|
||||
@ -182,19 +182,15 @@ pub async fn update_permission_mode(
|
||||
session_id, mode_str
|
||||
);
|
||||
|
||||
let mut checkers = state.session_permission_checkers.write().await;
|
||||
if let Some(checker) = checkers.get_mut(&session_id) {
|
||||
let mut checker = state
|
||||
.session_permission_checkers
|
||||
.entry(session_id.clone())
|
||||
.or_insert_with(PermissionChecker::new);
|
||||
checker.set_mode(mode);
|
||||
Json(PermissionActionResponse {
|
||||
success: true,
|
||||
message: format!("权限模式已切换为: {}", mode_str),
|
||||
})
|
||||
} else {
|
||||
Json(PermissionActionResponse {
|
||||
success: false,
|
||||
message: format!("会话 {} 不存在或无活跃 Agent", session_id),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 列出指定会话的所有权限规则。
|
||||
@ -202,9 +198,7 @@ pub async fn list_permission_rules(
|
||||
Path(session_id): Path<String>,
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Json<PermissionRulesResponse> {
|
||||
let checkers = state.session_permission_checkers.read().await;
|
||||
|
||||
if let Some(checker) = checkers.get(&session_id) {
|
||||
if let Some(checker) = state.session_permission_checkers.get(&session_id) {
|
||||
let mut deny_rules = Vec::new();
|
||||
let mut allow_rules = Vec::new();
|
||||
let mut ask_rules = Vec::new();
|
||||
|
||||
@ -259,13 +259,13 @@ async fn main() -> anyhow::Result<()> {
|
||||
astroresearch::services::batch::AssetBatchStatus::new(),
|
||||
)),
|
||||
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
|
||||
cancelled_runs: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())),
|
||||
cancelled_runs: Arc::new(dashmap::DashMap::new()),
|
||||
skill_registry: Arc::new(tokio::sync::RwLock::new(
|
||||
astroresearch::agent::skills::SkillRegistry::new(config.skills_dir.clone()),
|
||||
)),
|
||||
pending_questions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
||||
pending_permissions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
||||
session_permission_checkers: Arc::new(tokio::sync::RwLock::new(HashMap::new())),
|
||||
session_permission_checkers: Arc::new(dashmap::DashMap::new()),
|
||||
sse_broadcast: {
|
||||
let (tx, _) = tokio::sync::broadcast::channel(256);
|
||||
Some(tx)
|
||||
@ -631,7 +631,8 @@ async fn main() -> anyhow::Result<()> {
|
||||
&config.library_dir,
|
||||
&title,
|
||||
&content,
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!(
|
||||
"✅ 笔记已成功保存！\n 文件名: {}\n 物理路径: {}\n 总字符数: {}",
|
||||
|
||||
@ -112,12 +112,11 @@ impl GaiaClient {
|
||||
Ok(GaiaClient {
|
||||
client: reqwest::Client::builder()
|
||||
// Gaia DataLink 端点（data-server/data）在传输完 ZIP 后直接断 TCP 而不发送
|
||||
// TLS close_notify 警报。rustls 严格遵守 RFC 5246，会把这种"无 close_notify 的
|
||||
// 关闭"判为 UnexpectedEof 致命错误（哪怕响应体已完整到达）。ESA 官方 Python
|
||||
// 客户端 astroquery 走系统 OpenSSL（native-tls），后者容忍此行为，故不受影响。
|
||||
// 这里对 Gaia 客户端单独启用 native-tls，使其与 astroquery 走相同 TLS 栈，
|
||||
// 从而正确处理 DataLink 的连接关闭。项目其余 client 仍用默认 rustls。
|
||||
.use_native_tls()
|
||||
// TLS close_notify 警报。rustls 严格遵守 RFC 5246 会判 UnexpectedEof 致命错误。
|
||||
// 多数 UnexpectedEof 只在复用 idle 连接时触发，禁用连接池可避免：
|
||||
// 每次请求新建连接，响应读完即关闭，不会复用"已被对端不洁关闭"的连接。
|
||||
.pool_max_idle_per_host(0)
|
||||
.pool_idle_timeout(Some(Duration::from_secs(0)))
|
||||
.redirect(crate::utils::ssrf::safe_redirect_policy())
|
||||
.timeout(Duration::from_secs(timeout_secs))
|
||||
.connect_timeout(Duration::from_secs(15))
|
||||
|
||||
@ -99,13 +99,17 @@ impl LamostClient {
|
||||
/// GET {base}/{dr}/{ver}/{voservice|medvoservice}/conesearch?RA={ra}&DEC={dec}&SR={radius_deg}
|
||||
/// - LRS（低分辨率）→ voservice 子路径
|
||||
/// - MRS（中分辨率）→ medvoservice 子路径
|
||||
///
|
||||
/// 服务端返回 VOTable，用共享解析器解析后映射为 LamostSpectrumRow。
|
||||
///
|
||||
/// `version` 为该 DR 的子版本路径段（如 "v2.0"/"v1.0"），由上层 parse_lamost_version 解析后传入。
|
||||
pub async fn cone_search(
|
||||
&self,
|
||||
ra: f64,
|
||||
dec: f64,
|
||||
radius_deg: f64,
|
||||
release: crate::services::observation::lamost::LamostRelease,
|
||||
version: &str,
|
||||
resolution: crate::services::observation::lamost::LamostResolution,
|
||||
) -> anyhow::Result<LamostConeResult> {
|
||||
use crate::services::observation::lamost::LamostResolution;
|
||||
@ -125,12 +129,13 @@ impl LamostClient {
|
||||
"{}/{}/{}/{}/conesearch",
|
||||
self.base_url,
|
||||
release.path_segment(),
|
||||
release.version_segment(),
|
||||
version,
|
||||
service_segment,
|
||||
);
|
||||
info!(
|
||||
"[LAMOST] ConeSearch {} {} ra={:.4} dec={:.4} radius={:.4}°",
|
||||
"[LAMOST] ConeSearch {}/{} {} ra={:.4} dec={:.4} radius={:.4}°",
|
||||
release.path_segment(),
|
||||
version,
|
||||
resolution.display(),
|
||||
ra,
|
||||
dec,
|
||||
@ -172,21 +177,25 @@ impl LamostClient {
|
||||
///
|
||||
/// GET {base}/{dr}/{ver}/spectrum/fits/{obsid} → application/gzip
|
||||
/// 返回的 bytes 可能是 gzip 压缩（magic bytes 0x1f 0x8b），由业务层解压。
|
||||
///
|
||||
/// `version` 为该 DR 的子版本路径段（如 "v2.0"），由上层 parse_lamost_version 解析后传入。
|
||||
pub async fn download_fits(
|
||||
&self,
|
||||
obsid: i64,
|
||||
release: crate::services::observation::lamost::LamostRelease,
|
||||
version: &str,
|
||||
) -> anyhow::Result<Vec<u8>> {
|
||||
let url = format!(
|
||||
"{}/{}/{}/spectrum/fits/{}",
|
||||
self.base_url,
|
||||
release.path_segment(),
|
||||
release.version_segment(),
|
||||
version,
|
||||
obsid
|
||||
);
|
||||
info!(
|
||||
"[LAMOST] 下载 FITS {} obsid={}",
|
||||
"[LAMOST] 下载 FITS {}/{} obsid={}",
|
||||
release.path_segment(),
|
||||
version,
|
||||
obsid
|
||||
);
|
||||
|
||||
@ -366,13 +375,14 @@ mod tests {
|
||||
async fn test_live_cone_search() {
|
||||
use crate::services::observation::lamost::{LamostRelease, LamostResolution};
|
||||
let client = LamostClient::new("https://www.lamost.org", 60).unwrap();
|
||||
// M31 附近，DR10 LRS
|
||||
// M31 附近，DR10 v2.0 LRS
|
||||
let result = client
|
||||
.cone_search(
|
||||
10.6847,
|
||||
41.2687,
|
||||
0.1,
|
||||
LamostRelease::Dr10,
|
||||
"v2.0",
|
||||
LamostResolution::Lrs,
|
||||
)
|
||||
.await
|
||||
@ -403,6 +413,7 @@ mod tests {
|
||||
41.2687,
|
||||
0.1,
|
||||
LamostRelease::Dr10,
|
||||
"v2.0",
|
||||
LamostResolution::Lrs,
|
||||
)
|
||||
.await
|
||||
@ -410,7 +421,7 @@ mod tests {
|
||||
let obsid = cone.rows.first().expect("应有结果").obsid;
|
||||
|
||||
let bytes = client
|
||||
.download_fits(obsid, LamostRelease::Dr10)
|
||||
.download_fits(obsid, LamostRelease::Dr10, "v2.0")
|
||||
.await
|
||||
.unwrap();
|
||||
println!("===== LAMOST FITS Live (obsid={}) =====", obsid);
|
||||
|
||||
@ -24,7 +24,7 @@ pub trait ChatCompleter: Send + Sync {
|
||||
pub struct LlmClient {
|
||||
api_key: String,
|
||||
api_base: String,
|
||||
model: Arc<std::sync::RwLock<String>>,
|
||||
model: Arc<tokio::sync::RwLock<String>>,
|
||||
client: Client,
|
||||
}
|
||||
|
||||
@ -46,21 +46,25 @@ impl LlmClient {
|
||||
Ok(LlmClient {
|
||||
api_key,
|
||||
api_base,
|
||||
model: Arc::new(std::sync::RwLock::new(model)),
|
||||
model: Arc::new(tokio::sync::RwLock::new(model)),
|
||||
client,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn model(&self) -> String {
|
||||
self.model.read().unwrap_or_else(|e| e.into_inner()).clone()
|
||||
pub async fn model(&self) -> String {
|
||||
self.model.read().await.clone()
|
||||
}
|
||||
|
||||
/// 同步获取模型名称快照（用于非 async 上下文，如环境信息展示）。
|
||||
/// 如果锁被持有则返回空字符串。
|
||||
pub fn model_name_snapshot(&self) -> String {
|
||||
self.model.try_read().map(|m| m.clone()).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// 运行时切换模型（用于故障转移）。
|
||||
pub fn set_model(&self, model: String) {
|
||||
if let Ok(mut m) = self.model.write() {
|
||||
*m = model;
|
||||
tracing::info!("[LlmClient] 模型切换为: {}", m);
|
||||
}
|
||||
pub async fn set_model(&self, model: String) {
|
||||
tracing::info!("[LlmClient] 模型切换为: {}", model);
|
||||
*self.model.write().await = model;
|
||||
}
|
||||
|
||||
pub fn api_base(&self) -> &str {
|
||||
@ -79,7 +83,7 @@ impl LlmClient {
|
||||
let url = format!("{}/chat/completions", self.api_base);
|
||||
|
||||
let payload = serde_json::json!({
|
||||
"model": self.model(),
|
||||
"model": self.model().await,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
@ -143,7 +147,7 @@ impl LlmClient {
|
||||
let url = format!("{}/chat/completions", self.api_base);
|
||||
|
||||
let mut payload = serde_json::json!({
|
||||
"model": self.model(),
|
||||
"model": self.model().await,
|
||||
"messages": messages,
|
||||
"temperature": 0.3
|
||||
});
|
||||
@ -151,7 +155,7 @@ impl LlmClient {
|
||||
// 前端可控的思考模式开关（仅对千问/DashScope 启用 enable_thinking 参数）（仅对千问/DashScope 启用 enable_thinking 参数）
|
||||
if enable_thinking
|
||||
&& (self.api_base.contains("dashscope.aliyuncs.com")
|
||||
|| self.model().to_lowercase().contains("qwen"))
|
||||
|| self.model().await.to_lowercase().contains("qwen"))
|
||||
{
|
||||
if let Some(obj) = payload.as_object_mut() {
|
||||
obj.insert("enable_thinking".to_string(), serde_json::json!(true));
|
||||
@ -224,7 +228,7 @@ impl LlmClient {
|
||||
]);
|
||||
|
||||
let payload = serde_json::json!({
|
||||
"model": self.model(),
|
||||
"model": self.model().await,
|
||||
"messages": [
|
||||
{ "role": "system", "content": system_prompt },
|
||||
{ "role": "user", "content": user_content }
|
||||
@ -297,7 +301,7 @@ impl LlmClient {
|
||||
let url = format!("{}/chat/completions", self.api_base);
|
||||
|
||||
let mut payload = serde_json::json!({
|
||||
"model": self.model(),
|
||||
"model": self.model().await,
|
||||
"messages": messages,
|
||||
"temperature": 0.3,
|
||||
"stream": true,
|
||||
@ -307,7 +311,7 @@ impl LlmClient {
|
||||
// 前端可控的思考模式开关（仅对千问/DashScope 启用 enable_thinking 参数）
|
||||
if enable_thinking
|
||||
&& (self.api_base.contains("dashscope.aliyuncs.com")
|
||||
|| self.model().to_lowercase().contains("qwen"))
|
||||
|| self.model().await.to_lowercase().contains("qwen"))
|
||||
{
|
||||
if let Some(obj) = payload.as_object_mut() {
|
||||
obj.insert("enable_thinking".to_string(), serde_json::json!(true));
|
||||
@ -564,13 +568,13 @@ mod tests {
|
||||
ChatMessage, FunctionCall, MessageRole, ToolCall, ToolDefinition,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn test_llm_client_initialization() {
|
||||
#[tokio::test]
|
||||
async fn test_llm_client_basics() {
|
||||
let client =
|
||||
LlmClient::new("key".to_string(), "base".to_string(), "model".to_string()).unwrap();
|
||||
assert_eq!(client.api_key(), "key");
|
||||
assert_eq!(client.api_base(), "base");
|
||||
assert_eq!(client.model(), "model");
|
||||
assert_eq!(client.model().await, "model");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
63
src/lib.rs
63
src/lib.rs
@ -2,6 +2,19 @@
|
||||
use std::env;
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// 从环境变量解析 u64 值，解析失败时打印警告并返回默认值。
|
||||
fn env_u64(key: &str, default: u64) -> u64 {
|
||||
env::var(key)
|
||||
.ok()
|
||||
.and_then(|v| {
|
||||
v.parse::<u64>().ok().or_else(|| {
|
||||
tracing::warn!("{} '{}' 不是有效数字，使用默认值 {}", key, v, default);
|
||||
None
|
||||
})
|
||||
})
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
// 系统配置结构体，加载并管理从环境变量或 .env 文件读取的参数
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Config {
|
||||
@ -133,65 +146,25 @@ impl Config {
|
||||
|
||||
let vizier_tap_url = env::var("VIZIER_TAP_URL")
|
||||
.unwrap_or_else(|_| "https://tapvizier.cds.unistra.fr/TAPVizieR/tap".to_string());
|
||||
let vizier_timeout_secs = env::var("VIZIER_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|v| {
|
||||
v.parse::<u64>().ok().or_else(|| {
|
||||
tracing::warn!("VIZIER_TIMEOUT_SECS '{}' 不是有效数字，使用默认值 60", v);
|
||||
None
|
||||
})
|
||||
})
|
||||
.unwrap_or(60);
|
||||
let vizier_timeout_secs = env_u64("VIZIER_TIMEOUT_SECS", 60);
|
||||
|
||||
let lamost_base_url =
|
||||
env::var("LAMOST_BASE_URL").unwrap_or_else(|_| "https://www.lamost.org".to_string());
|
||||
let lamost_timeout_secs = env::var("LAMOST_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|v| {
|
||||
v.parse::<u64>().ok().or_else(|| {
|
||||
tracing::warn!("LAMOST_TIMEOUT_SECS '{}' 不是有效数字，使用默认值 60", v);
|
||||
None
|
||||
})
|
||||
})
|
||||
.unwrap_or(60);
|
||||
let lamost_timeout_secs = env_u64("LAMOST_TIMEOUT_SECS", 60);
|
||||
|
||||
let gaia_tap_url = env::var("GAIA_TAP_URL")
|
||||
.unwrap_or_else(|_| "https://gea.esac.esa.int/tap-server/tap".to_string());
|
||||
let gaia_datalink_url = env::var("GAIA_DATALINK_URL")
|
||||
.unwrap_or_else(|_| "https://gea.esac.esa.int/data-server".to_string());
|
||||
let gaia_timeout_secs = env::var("GAIA_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|v| {
|
||||
v.parse::<u64>().ok().or_else(|| {
|
||||
tracing::warn!("GAIA_TIMEOUT_SECS '{}' 不是有效数字，使用默认值 90", v);
|
||||
None
|
||||
})
|
||||
})
|
||||
.unwrap_or(90);
|
||||
let gaia_timeout_secs = env_u64("GAIA_TIMEOUT_SECS", 90);
|
||||
|
||||
let sdss_tap_url = env::var("SDSS_TAP_URL")
|
||||
.unwrap_or_else(|_| "https://datalab.noirlab.edu/tap/sync".to_string());
|
||||
let sdss_timeout_secs = env::var("SDSS_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|v| {
|
||||
v.parse::<u64>().ok().or_else(|| {
|
||||
tracing::warn!("SDSS_TIMEOUT_SECS '{}' 不是有效数字，使用默认值 90", v);
|
||||
None
|
||||
})
|
||||
})
|
||||
.unwrap_or(90);
|
||||
let sdss_timeout_secs = env_u64("SDSS_TIMEOUT_SECS", 90);
|
||||
|
||||
let desi_tap_url = env::var("DESI_TAP_URL")
|
||||
.unwrap_or_else(|_| "https://datalab.noirlab.edu/tap/sync".to_string());
|
||||
let desi_timeout_secs = env::var("DESI_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|v| {
|
||||
v.parse::<u64>().ok().or_else(|| {
|
||||
tracing::warn!("DESI_TIMEOUT_SECS '{}' 不是有效数字，使用默认值 120", v);
|
||||
None
|
||||
})
|
||||
})
|
||||
.unwrap_or(120);
|
||||
let desi_timeout_secs = env_u64("DESI_TIMEOUT_SECS", 120);
|
||||
|
||||
Config {
|
||||
database_url,
|
||||
|
||||
22
src/main.rs
22
src/main.rs
@ -290,7 +290,9 @@ async fn main() -> anyhow::Result<()> {
|
||||
gaia,
|
||||
sdss,
|
||||
desi,
|
||||
observation_registry: Arc::new(astroresearch::services::observation::ObservationRegistry::default()),
|
||||
observation_registry: Arc::new(
|
||||
astroresearch::services::observation::ObservationRegistry::default(),
|
||||
),
|
||||
llm,
|
||||
medium_llm,
|
||||
fast_llm,
|
||||
@ -309,7 +311,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
astroresearch::services::batch::AssetBatchStatus::new(),
|
||||
)),
|
||||
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
|
||||
cancelled_runs: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())),
|
||||
cancelled_runs: Arc::new(dashmap::DashMap::new()),
|
||||
skill_registry,
|
||||
pending_questions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
||||
pending_permissions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
||||
@ -320,9 +322,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
memory_manager: Arc::new(tokio::sync::Mutex::new(
|
||||
astroresearch::agent::memory::MemoryManager::new(config.library_dir.clone()),
|
||||
)),
|
||||
session_permission_checkers: Arc::new(tokio::sync::RwLock::new(
|
||||
std::collections::HashMap::new(),
|
||||
)),
|
||||
session_permission_checkers: Arc::new(dashmap::DashMap::new()),
|
||||
sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
|
||||
login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
||||
upload_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
||||
@ -530,6 +530,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
.nest_service("/api/files", protected_files)
|
||||
.fallback_service(serve_dir)
|
||||
.layer(tower_http::trace::TraceLayer::new_for_http())
|
||||
.layer(tower_http::catch_panic::CatchPanicLayer::new())
|
||||
.layer(SetResponseHeaderLayer::overriding(
|
||||
axum::http::header::X_CONTENT_TYPE_OPTIONS,
|
||||
HeaderValue::from_static("nosniff"),
|
||||
@ -575,17 +576,18 @@ async fn pdf_inline_middleware(
|
||||
req: axum::http::Request<axum::body::Body>,
|
||||
next: axum::middleware::Next,
|
||||
) -> Result<axum::response::Response<axum::body::Body>, axum::http::StatusCode> {
|
||||
let path = req.uri().path().to_lowercase();
|
||||
let is_pdf = path.ends_with(".pdf");
|
||||
let original_path = req.uri().path().to_string();
|
||||
let is_pdf = original_path.to_lowercase().ends_with(".pdf");
|
||||
|
||||
let mut response = next.run(req).await;
|
||||
|
||||
if is_pdf && response.status().is_success() {
|
||||
// 获取文件名
|
||||
let filename = std::path::Path::new(&path)
|
||||
// 使用原始路径提取文件名（保留大小写）
|
||||
let filename = std::path::Path::new(&original_path)
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("document.pdf");
|
||||
.unwrap_or("document.pdf")
|
||||
.replace('"', "_");
|
||||
|
||||
// 强行设置 Content-Type 为 application/pdf
|
||||
response.headers_mut().insert(
|
||||
|
||||
@ -220,7 +220,7 @@ pub(super) async fn build_markdown_with_frontmatter(
|
||||
}
|
||||
|
||||
/// 将 Markdown 内容写入 library_dir 并更新数据库 markdown_path。
|
||||
pub(super) fn write_markdown_and_update_db(
|
||||
pub(super) async fn write_markdown_and_update_db(
|
||||
config: &Config,
|
||||
bibcode: &str,
|
||||
content: &str,
|
||||
@ -228,9 +228,9 @@ pub(super) fn write_markdown_and_update_db(
|
||||
let md_filename = format!("{}.md", bibcode);
|
||||
let md_dest = config.library_dir.join("Markdown").join(&md_filename);
|
||||
if let Some(parent) = md_dest.parent() {
|
||||
let _ = std::fs::create_dir_all(parent);
|
||||
let _ = tokio::fs::create_dir_all(parent).await;
|
||||
}
|
||||
if std::fs::write(&md_dest, content).is_ok() {
|
||||
if tokio::fs::write(&md_dest, content).await.is_ok() {
|
||||
Some(format!("Markdown/{}", md_filename))
|
||||
} else {
|
||||
None
|
||||
|
||||
@ -170,7 +170,7 @@ pub(super) async fn process_single_bibcode(
|
||||
.await
|
||||
{
|
||||
if let Some(rel_path) =
|
||||
write_markdown_and_update_db(config, &bibcode, &parsed_md)
|
||||
write_markdown_and_update_db(config, &bibcode, &parsed_md).await
|
||||
{
|
||||
relative_md_path = rel_path;
|
||||
}
|
||||
@ -288,7 +288,8 @@ pub(super) async fn process_single_bibcode(
|
||||
&config_clone,
|
||||
&bibcode_clone,
|
||||
&parsed_md,
|
||||
);
|
||||
)
|
||||
.await;
|
||||
|
||||
if let Some(rel_path) = rel_md {
|
||||
let _ = sqlx::query("UPDATE papers SET markdown_path = ? WHERE bibcode = ?")
|
||||
|
||||
@ -213,20 +213,36 @@ pub async fn extract_and_cache_targets(
|
||||
let mut results = Vec::new();
|
||||
let mut new_targets = Vec::new();
|
||||
|
||||
for name in names {
|
||||
// 优先在 paper_targets 表中查找本地缓存
|
||||
let row = sqlx::query_as::<_, TargetDbRow>(
|
||||
if names.is_empty() {
|
||||
return results;
|
||||
}
|
||||
|
||||
// 1. 批量查询已存在的本地缓存记录，减少 N+1 数据库往返
|
||||
let mut cached_map = std::collections::HashMap::new();
|
||||
for chunk in names.chunks(crate::services::paper::SQLITE_PARAM_LIMIT) {
|
||||
let placeholders = vec!["?"; chunk.len()].join(", ");
|
||||
let sql = format!(
|
||||
"SELECT target_name, ra, dec, parallax, parallax_err, spectral_type, v_magnitude, \
|
||||
otype, oname, pm_ra, pm_de, radial_velocity, photometry, aliases \
|
||||
FROM paper_targets WHERE target_name = ? LIMIT 1",
|
||||
)
|
||||
.bind(&name)
|
||||
.fetch_optional(pool)
|
||||
.await;
|
||||
FROM paper_targets WHERE target_name IN ({})",
|
||||
placeholders
|
||||
);
|
||||
let mut query = sqlx::query_as::<_, TargetDbRow>(&sql);
|
||||
for name in chunk {
|
||||
query = query.bind(name);
|
||||
}
|
||||
if let Ok(rows) = query.fetch_all(pool).await {
|
||||
for row in rows {
|
||||
cached_map.insert(row.target_name.clone(), TargetInfo::from(row));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(Some(db_row)) = row {
|
||||
// 2. 遍历检查哪些命中缓存，哪些需要调用 Sesame
|
||||
for name in names {
|
||||
if let Some(info) = cached_map.remove(&name) {
|
||||
info!("天体 {} 命中本地缓存", name);
|
||||
results.push(TargetInfo::from(db_row));
|
||||
results.push(info);
|
||||
} else {
|
||||
// 本地缓存未命中，释放连接并外部查询 Sesame
|
||||
info!("天体 {} 本地缓存未命中，查询 CDS Sesame...", name);
|
||||
|
||||
@ -323,7 +323,10 @@ impl<'a> VizierCatalog<'a> {
|
||||
} else {
|
||||
sanitize_identifier(table)?
|
||||
};
|
||||
Ok(format!("SELECT TOP {} {} FROM {}", limit, columns, table_ref))
|
||||
Ok(format!(
|
||||
"SELECT TOP {} {} FROM {}",
|
||||
limit, columns, table_ref
|
||||
))
|
||||
}
|
||||
|
||||
/// 导出星表数据到 CSV 文件
|
||||
@ -357,7 +360,13 @@ impl<'a> VizierCatalog<'a> {
|
||||
let catalog_name = table.unwrap_or("adql_query");
|
||||
let safe_name: String = catalog_name
|
||||
.chars()
|
||||
.map(|c| if c.is_alphanumeric() || c == '-' || c == '_' { c } else { '_' })
|
||||
.map(|c| {
|
||||
if c.is_alphanumeric() || c == '-' || c == '_' {
|
||||
c
|
||||
} else {
|
||||
'_'
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let ts = chrono::Utc::now().format("%Y%m%d_%H%M%S");
|
||||
@ -367,10 +376,12 @@ impl<'a> VizierCatalog<'a> {
|
||||
.join(format!("{}.csv", ts));
|
||||
|
||||
if let Some(parent) = output_path.parent() {
|
||||
tokio::fs::create_dir_all(parent).await
|
||||
tokio::fs::create_dir_all(parent)
|
||||
.await
|
||||
.map_err(|e| anyhow!("创建目录失败: {}", e))?;
|
||||
}
|
||||
tokio::fs::write(&output_path, &export.csv).await
|
||||
tokio::fs::write(&output_path, &export.csv)
|
||||
.await
|
||||
.map_err(|e| anyhow!("写入文件失败: {}", e))?;
|
||||
|
||||
Ok(ExportToFileResult {
|
||||
|
||||
@ -152,83 +152,74 @@ impl Downloader {
|
||||
doi: Option<&str>,
|
||||
library_dir: &Path,
|
||||
) -> (Result<PathBuf, String>, Result<PathBuf, String>) {
|
||||
let base = "https://ui.adsabs.harvard.edu/link_gateway";
|
||||
let pdf_dest = library_dir.join("PDF").join(format!("{}.pdf", bibcode));
|
||||
let html_dest = library_dir.join("HTML").join(format!("{}.html", bibcode));
|
||||
|
||||
let mut pdf_res = Err("未尝试任何下载通道".to_string());
|
||||
let mut html_res = Err("未尝试任何下载通道".to_string());
|
||||
let pdf_res = self.try_download_pdf(bibcode, doi, &pdf_dest).await;
|
||||
let html_res = self.try_download_html(bibcode, &html_dest).await;
|
||||
|
||||
let mut pdf_errors = Vec::new();
|
||||
let mut html_errors = Vec::new();
|
||||
(pdf_res, html_res)
|
||||
}
|
||||
|
||||
/// PDF 多级回退下载：PUB_PDF → ADS_PDF → EPRINT_PDF → CrossRef → SCAN
|
||||
async fn try_download_pdf(
|
||||
&self,
|
||||
bibcode: &str,
|
||||
doi: Option<&str>,
|
||||
dest: &Path,
|
||||
) -> Result<PathBuf, String> {
|
||||
let base = "https://ui.adsabs.harvard.edu/link_gateway";
|
||||
let mut errors = Vec::new();
|
||||
|
||||
// ── PDF 下载 ───────────────────────────────────────────
|
||||
info!("[下载] 开始 PDF 下载: {}", bibcode);
|
||||
|
||||
'pdf: {
|
||||
// 1a. ADS PUB_PDF 网关
|
||||
let gw = format!("{}/{}/PUB_PDF", base, bibcode);
|
||||
match self.resolve_ads_gateway(&gw).await {
|
||||
Ok(resolved) => {
|
||||
let result = if resolved.contains("iopscience.iop.org") {
|
||||
// 提取 DOI 路径部分，走 IOP 专属策略
|
||||
let doi = resolved
|
||||
.trim_start_matches("https://iopscience.iop.org/article/")
|
||||
.trim_end_matches("/pdf")
|
||||
.trim_end_matches('/');
|
||||
self.download_iop_pdf(doi, &pdf_dest).await
|
||||
} else if resolved.contains("link.springer.com")
|
||||
|| resolved.contains("nature.com")
|
||||
self.download_iop_pdf(doi, dest).await
|
||||
} else if resolved.contains("link.springer.com") || resolved.contains("nature.com")
|
||||
{
|
||||
// Springer/Nature：HTML 更可靠，PDF 用通用策略
|
||||
self.download_pdf_direct(&resolved, &pdf_dest, "Springer")
|
||||
.await
|
||||
self.download_pdf_direct(&resolved, dest, "Springer").await
|
||||
} else {
|
||||
self.download_pdf_direct(&resolved, &pdf_dest, "PUB_PDF")
|
||||
.await
|
||||
self.download_pdf_direct(&resolved, dest, "PUB_PDF").await
|
||||
};
|
||||
match result {
|
||||
Ok(_) => {
|
||||
pdf_res = Ok(pdf_dest.clone());
|
||||
break 'pdf;
|
||||
}
|
||||
Ok(_) => return Ok(dest.to_path_buf()),
|
||||
Err(e) => {
|
||||
let msg = format!("PUB_PDF下载失败: {}", e);
|
||||
warn!("{}", msg);
|
||||
pdf_errors.push(msg);
|
||||
errors.push(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!("PUB_PDF网关解析失败: {}", e);
|
||||
warn!("{}", msg);
|
||||
pdf_errors.push(msg);
|
||||
errors.push(msg);
|
||||
}
|
||||
}
|
||||
|
||||
// 1b. ADS_PDF 网关 (经典 ADS 整合 PDF 直接通道)
|
||||
// 1b. ADS_PDF 网关
|
||||
let gw = format!("{}/{}/ADS_PDF", base, bibcode);
|
||||
match self.resolve_ads_gateway(&gw).await {
|
||||
Ok(resolved) => {
|
||||
match self
|
||||
.download_pdf_direct(&resolved, &pdf_dest, "ADS_PDF")
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
pdf_res = Ok(pdf_dest.clone());
|
||||
break 'pdf;
|
||||
}
|
||||
Ok(resolved) => match self.download_pdf_direct(&resolved, dest, "ADS_PDF").await {
|
||||
Ok(_) => return Ok(dest.to_path_buf()),
|
||||
Err(e) => {
|
||||
let msg = format!("ADS_PDF下载失败: {}", e);
|
||||
warn!("{}", msg);
|
||||
pdf_errors.push(msg);
|
||||
}
|
||||
}
|
||||
errors.push(msg);
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
let msg = format!("ADS_PDF网关解析失败: {}", e);
|
||||
warn!("{}", msg);
|
||||
pdf_errors.push(msg);
|
||||
errors.push(msg);
|
||||
}
|
||||
}
|
||||
|
||||
@ -237,142 +228,129 @@ impl Downloader {
|
||||
match self.resolve_ads_gateway(&gw).await {
|
||||
Ok(resolved) => {
|
||||
match self
|
||||
.download_pdf_direct(&resolved, &pdf_dest, "EPRINT_PDF")
|
||||
.download_pdf_direct(&resolved, dest, "EPRINT_PDF")
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
pdf_res = Ok(pdf_dest.clone());
|
||||
break 'pdf;
|
||||
}
|
||||
Ok(_) => return Ok(dest.to_path_buf()),
|
||||
Err(e) => {
|
||||
let msg = format!("EPRINT_PDF下载失败: {}", e);
|
||||
warn!("{}", msg);
|
||||
pdf_errors.push(msg);
|
||||
errors.push(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!("EPRINT_PDF网关解析失败: {}", e);
|
||||
warn!("{}", msg);
|
||||
pdf_errors.push(msg);
|
||||
errors.push(msg);
|
||||
}
|
||||
}
|
||||
|
||||
// 1d. CrossRef API 回退（需要 DOI）
|
||||
// 1d. CrossRef API 回退
|
||||
if let Some(doi_str) = doi {
|
||||
match self.download_crossref_pdf(doi_str, &pdf_dest).await {
|
||||
Ok(_) => {
|
||||
pdf_res = Ok(pdf_dest.clone());
|
||||
break 'pdf;
|
||||
}
|
||||
match self.download_crossref_pdf(doi_str, dest).await {
|
||||
Ok(_) => return Ok(dest.to_path_buf()),
|
||||
Err(e) => {
|
||||
let msg = format!("CrossRef下载失败: {}", e);
|
||||
warn!("{}", msg);
|
||||
pdf_errors.push(msg);
|
||||
errors.push(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1e. ADS SCAN 扫描版文献直接合并下载 PDF（主要针对早期/不可下载直接 PDF 的文献）
|
||||
let scan_url = format!("https://articles.adsabs.harvard.edu/cgi-bin/nph-iarticle_query?bibcode={}&db_key=AST&data_type=PDF_HIGH", bibcode);
|
||||
match self
|
||||
.download_pdf_direct(&scan_url, &pdf_dest, "ADS_SCAN")
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
pdf_res = Ok(pdf_dest.clone());
|
||||
}
|
||||
// 1e. ADS SCAN 扫描版
|
||||
let scan_url = format!(
|
||||
"https://articles.adsabs.harvard.edu/cgi-bin/nph-iarticle_query?bibcode={}&db_key=AST&data_type=PDF_HIGH",
|
||||
bibcode
|
||||
);
|
||||
match self.download_pdf_direct(&scan_url, dest, "ADS_SCAN").await {
|
||||
Ok(_) => return Ok(dest.to_path_buf()),
|
||||
Err(e) => {
|
||||
let msg = format!("ADS_SCAN下载失败: {}", e);
|
||||
warn!("{}", msg);
|
||||
pdf_errors.push(msg);
|
||||
}
|
||||
errors.push(msg);
|
||||
}
|
||||
}
|
||||
|
||||
if pdf_res.is_err() && !pdf_errors.is_empty() {
|
||||
pdf_res = Err(pdf_errors.join("; "));
|
||||
if errors.is_empty() {
|
||||
Err("未尝试任何下载通道".to_string())
|
||||
} else {
|
||||
Err(errors.join("; "))
|
||||
}
|
||||
}
|
||||
|
||||
// ── HTML 下载 ──────────────────────────────────────────
|
||||
/// HTML 多级回退下载：PUB_HTML → EPRINT_HTML
|
||||
async fn try_download_html(&self, bibcode: &str, dest: &Path) -> Result<PathBuf, String> {
|
||||
let base = "https://ui.adsabs.harvard.edu/link_gateway";
|
||||
let mut errors = Vec::new();
|
||||
|
||||
info!("[下载] 开始 HTML 下载: {}", bibcode);
|
||||
|
||||
'html: {
|
||||
// 2a. ADS PUB_HTML 网关
|
||||
let gw = format!("{}/{}/PUB_HTML", base, bibcode);
|
||||
match self.resolve_ads_gateway(&gw).await {
|
||||
Ok(resolved) => {
|
||||
let result = if resolved.contains("link.springer.com")
|
||||
|| resolved.contains("nature.com")
|
||||
{
|
||||
// Springer/Nature 专属 HTML 策略
|
||||
let result =
|
||||
if resolved.contains("link.springer.com") || resolved.contains("nature.com") {
|
||||
let doi_part = resolved
|
||||
.trim_start_matches("https://link.springer.com/article/")
|
||||
.trim_start_matches("https://www.nature.com/articles/")
|
||||
.trim_end_matches('/');
|
||||
self.download_springer_html(doi_part, &html_dest).await
|
||||
self.download_springer_html(doi_part, dest).await
|
||||
} else if let Some(arxiv_id) = extract_arxiv_id_from_url(&resolved) {
|
||||
// ADS 网关指向 arXiv abs 页面 → 优先官方 HTML，ar5iv 兜底
|
||||
self.download_arxiv_html_with_fallback(&arxiv_id, &html_dest)
|
||||
self.download_arxiv_html_with_fallback(&arxiv_id, dest)
|
||||
.await
|
||||
} else {
|
||||
self.download_html_direct(&resolved, &html_dest, "PUB_HTML")
|
||||
.await
|
||||
self.download_html_direct(&resolved, dest, "PUB_HTML").await
|
||||
};
|
||||
match result {
|
||||
Ok(_) => {
|
||||
html_res = Ok(html_dest.clone());
|
||||
break 'html;
|
||||
}
|
||||
Ok(_) => return Ok(dest.to_path_buf()),
|
||||
Err(e) => {
|
||||
let msg = format!("PUB_HTML下载失败: {}", e);
|
||||
warn!("{}", msg);
|
||||
html_errors.push(msg);
|
||||
errors.push(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!("PUB_HTML网关解析失败: {}", e);
|
||||
warn!("{}", msg);
|
||||
html_errors.push(msg);
|
||||
errors.push(msg);
|
||||
}
|
||||
}
|
||||
|
||||
// 2b. ADS EPRINT_HTML 网关（大多数天文论文有 arXiv eprint）
|
||||
// 2b. ADS EPRINT_HTML 网关
|
||||
let gw = format!("{}/{}/EPRINT_HTML", base, bibcode);
|
||||
match self.resolve_ads_gateway(&gw).await {
|
||||
Ok(resolved) => {
|
||||
let result = if let Some(arxiv_id) = extract_arxiv_id_from_url(&resolved) {
|
||||
self.download_arxiv_html_with_fallback(&arxiv_id, &html_dest)
|
||||
self.download_arxiv_html_with_fallback(&arxiv_id, dest)
|
||||
.await
|
||||
} else {
|
||||
self.download_html_direct(&resolved, &html_dest, "EPRINT_HTML")
|
||||
self.download_html_direct(&resolved, dest, "EPRINT_HTML")
|
||||
.await
|
||||
};
|
||||
match result {
|
||||
Ok(_) => {
|
||||
html_res = Ok(html_dest.clone());
|
||||
}
|
||||
Ok(_) => return Ok(dest.to_path_buf()),
|
||||
Err(e) => {
|
||||
let msg = format!("EPRINT_HTML下载失败: {}", e);
|
||||
warn!("{}", msg);
|
||||
html_errors.push(msg);
|
||||
errors.push(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!("EPRINT_HTML网关解析失败: {}", e);
|
||||
warn!("{}", msg);
|
||||
html_errors.push(msg);
|
||||
}
|
||||
errors.push(msg);
|
||||
}
|
||||
}
|
||||
|
||||
if html_res.is_err() && !html_errors.is_empty() {
|
||||
html_res = Err(html_errors.join("; "));
|
||||
if errors.is_empty() {
|
||||
Err("未尝试任何下载通道".to_string())
|
||||
} else {
|
||||
Err(errors.join("; "))
|
||||
}
|
||||
|
||||
(pdf_res, html_res)
|
||||
}
|
||||
|
||||
/// 高层文献下载服务，处理数据库记录、多级回退下载、数据库路径写回
|
||||
|
||||
@ -27,7 +27,7 @@ impl Downloader {
|
||||
info!("[Obscura 后备通道] 启动下载: {}", url);
|
||||
|
||||
if let Some(parent) = dest_path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
tokio::fs::create_dir_all(parent).await?;
|
||||
}
|
||||
|
||||
#[cfg(feature = "obscura-inprocess")]
|
||||
@ -190,7 +190,7 @@ impl Downloader {
|
||||
use futures_util::StreamExt;
|
||||
|
||||
if let Some(parent) = target_path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
tokio::fs::create_dir_all(parent).await?;
|
||||
}
|
||||
|
||||
let mut file = tokio::fs::File::create(target_path)
|
||||
|
||||
@ -56,7 +56,11 @@ pub fn init_logging() -> anyhow::Result<Vec<WorkerGuard>> {
|
||||
let log_dir = env::var("LOG_DIR").unwrap_or_else(|_| "logs".to_string());
|
||||
fs::create_dir_all(&log_dir).unwrap_or(());
|
||||
|
||||
let file_appender = rolling::daily(log_dir, "astro_research.log");
|
||||
let file_appender = rolling::RollingFileAppender::builder()
|
||||
.rotation(rolling::Rotation::DAILY)
|
||||
.filename_prefix("astro_research")
|
||||
.filename_suffix("log")
|
||||
.build(log_dir)?;
|
||||
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
|
||||
guards.push(guard);
|
||||
|
||||
|
||||
@ -4,12 +4,11 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// 保存研究笔记为本地 Markdown 文件
|
||||
pub fn save_note_service(
|
||||
pub async fn save_note_service(
|
||||
library_dir: &Path,
|
||||
title: &str,
|
||||
content: &str,
|
||||
) -> anyhow::Result<(String, PathBuf, usize)> {
|
||||
// 文件名安全化
|
||||
let safe_title: String = title
|
||||
.chars()
|
||||
.map(|c| {
|
||||
@ -23,7 +22,7 @@ pub fn save_note_service(
|
||||
.replace(' ', "_");
|
||||
|
||||
let notes_dir = library_dir.join("notes");
|
||||
std::fs::create_dir_all(¬es_dir)?;
|
||||
tokio::fs::create_dir_all(¬es_dir).await?;
|
||||
|
||||
let filename = format!("{}.md", safe_title);
|
||||
let filepath = notes_dir.join(&filename);
|
||||
@ -35,7 +34,7 @@ pub fn save_note_service(
|
||||
content
|
||||
);
|
||||
|
||||
std::fs::write(&filepath, &full_content)?;
|
||||
tokio::fs::write(&filepath, &full_content).await?;
|
||||
|
||||
Ok((filename, filepath, full_content.len()))
|
||||
}
|
||||
@ -103,13 +102,13 @@ pub async fn delete_highlight_note(db: &sqlx::SqlitePool, id: i64) -> Result<boo
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_save_note_service() {
|
||||
#[tokio::test]
|
||||
async fn test_save_note_service() {
|
||||
let tmp_dir = std::env::temp_dir().join("astro_note_test");
|
||||
let title = "Test Note Title 123!";
|
||||
let content = "Hello world from note service test";
|
||||
|
||||
let res = save_note_service(&tmp_dir, title, content);
|
||||
let res = save_note_service(&tmp_dir, title, content).await;
|
||||
assert!(res.is_ok());
|
||||
let (filename, filepath, size) = res.unwrap();
|
||||
assert_eq!(filename, "Test_Note_Title_123_.md");
|
||||
|
||||
@ -198,26 +198,39 @@ pub struct CachedPage {
|
||||
}
|
||||
|
||||
/// 列出已缓存产物（服务端分页 + 筛选 + 排序）
|
||||
///
|
||||
/// 所有筛选值通过 SQL 参数绑定（`?`）传入，杜绝 SQL 注入与 LIKE 通配符（`%`/`_`）误匹配。
|
||||
/// 排序方向用白名单（只接受 created/source/product 三个字面量）。
|
||||
pub async fn list_cached_paged(pool: &SqlitePool, params: &ListCachedParams) -> Result<CachedPage> {
|
||||
let limit = params.limit.unwrap_or(50).clamp(1, 200);
|
||||
let offset = params.offset.unwrap_or(0).max(0);
|
||||
|
||||
// 动态构造 SQL（参数绑定防注入；列名/排序方向用白名单）
|
||||
let mut where_clauses: Vec<String> = Vec::new();
|
||||
// 动态构造 WHERE 子句（用 ? 占位符）+ 同步收集绑定值
|
||||
// bind_values 顺序必须与占位符出现顺序一致：source → product → q → limit → offset
|
||||
let mut where_parts: Vec<&'static str> = Vec::new();
|
||||
let mut bind_source: Option<String> = None;
|
||||
let mut bind_product: Option<String> = None;
|
||||
let mut bind_q: Option<String> = None; // 已包成 '%q%'，且 LIKE 通配符已转义
|
||||
|
||||
if let Some(s) = ¶ms.source {
|
||||
where_clauses.push(format!("source = '{}'", s.replace('\'', "''")));
|
||||
where_parts.push("source = ?");
|
||||
bind_source = Some(s.clone());
|
||||
}
|
||||
if let Some(p) = ¶ms.product {
|
||||
where_clauses.push(format!("product = '{}'", p.replace('\'', "''")));
|
||||
where_parts.push("product = ?");
|
||||
bind_product = Some(p.clone());
|
||||
}
|
||||
if let Some(q) = ¶ms.q {
|
||||
let q = q.replace('\'', "''");
|
||||
where_clauses.push(format!("source_id LIKE '%{}%'", q));
|
||||
// 转义 LIKE 通配符，避免用户输入的 %/_ 被解释为模式
|
||||
let escaped = q.replace('%', "\\%").replace('_', "\\_");
|
||||
where_parts.push("source_id LIKE ? ESCAPE '\\'");
|
||||
bind_q = Some(format!("%{}%", escaped));
|
||||
}
|
||||
let where_sql = if where_clauses.is_empty() {
|
||||
|
||||
let where_sql = if where_parts.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(" WHERE {}", where_clauses.join(" AND "))
|
||||
format!(" WHERE {}", where_parts.join(" AND "))
|
||||
};
|
||||
|
||||
let order_sql = match params.sort.as_deref() {
|
||||
@ -226,22 +239,41 @@ pub async fn list_cached_paged(pool: &SqlitePool, params: &ListCachedParams) ->
|
||||
_ => " ORDER BY created_at DESC",
|
||||
};
|
||||
|
||||
// count
|
||||
let count_sql = format!("SELECT COUNT(*) as cnt FROM observation_cache{}", where_sql);
|
||||
let total: i64 = sqlx::query_scalar(&count_sql)
|
||||
// count（绑定 source/product/q，不绑 limit/offset）
|
||||
let count_sql = format!("SELECT COUNT(*) FROM observation_cache{}", where_sql);
|
||||
let mut count_q = sqlx::query_scalar::<_, i64>(&count_sql);
|
||||
if let Some(v) = &bind_source {
|
||||
count_q = count_q.bind(v);
|
||||
}
|
||||
if let Some(v) = &bind_product {
|
||||
count_q = count_q.bind(v);
|
||||
}
|
||||
if let Some(v) = &bind_q {
|
||||
count_q = count_q.bind(v);
|
||||
}
|
||||
let total = count_q
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|e| anyhow!("统计 observation_cache 总数失败: {}", e))?;
|
||||
|
||||
// items
|
||||
// items（绑定 source/product/q/limit/offset）
|
||||
let items_sql = format!(
|
||||
"SELECT source, product, source_id, ra, dec, artifacts_json, meta_json, created_at \
|
||||
FROM observation_cache{}{} LIMIT ? OFFSET ?",
|
||||
where_sql, order_sql
|
||||
);
|
||||
let items = sqlx::query_as::<_, ObservationCacheRow>(&items_sql)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
let mut items_q = sqlx::query_as::<_, ObservationCacheRow>(&items_sql);
|
||||
if let Some(v) = &bind_source {
|
||||
items_q = items_q.bind(v);
|
||||
}
|
||||
if let Some(v) = &bind_product {
|
||||
items_q = items_q.bind(v);
|
||||
}
|
||||
if let Some(v) = &bind_q {
|
||||
items_q = items_q.bind(v);
|
||||
}
|
||||
items_q = items_q.bind(limit).bind(offset);
|
||||
let items = items_q
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| anyhow!("查询 observation_cache 分页列表失败: {}", e))?;
|
||||
@ -259,26 +291,27 @@ pub fn file_url_from_path(rel_path: &str) -> String {
|
||||
}
|
||||
|
||||
/// 落盘字节到 library_dir/<rel_path>，返回字节数
|
||||
pub fn persist_bytes(library_dir: &Path, rel_path: &str, bytes: &[u8]) -> Result<usize> {
|
||||
pub async fn persist_bytes(library_dir: &Path, rel_path: &str, bytes: &[u8]) -> Result<usize> {
|
||||
let abs_path = library_dir.join(rel_path);
|
||||
if let Some(parent) = abs_path.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
tokio::fs::create_dir_all(parent)
|
||||
.await
|
||||
.map_err(|e| anyhow!("创建观测数据目录失败 {:?}: {}", parent, e))?;
|
||||
}
|
||||
std::fs::write(&abs_path, bytes)
|
||||
tokio::fs::write(&abs_path, bytes)
|
||||
.await
|
||||
.map_err(|e| anyhow!("写入观测数据文件失败 {:?}: {}", abs_path, e))?;
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
/// 校验缓存产物中至少有一个文件真实存在；存在则返回其总大小
|
||||
pub fn cached_files_total_size(library_dir: &Path, artifacts: &[Artifact]) -> Option<usize> {
|
||||
pub async fn cached_files_total_size(library_dir: &Path, artifacts: &[Artifact]) -> Option<usize> {
|
||||
let mut total = 0;
|
||||
for a in artifacts {
|
||||
let abs_path = library_dir.join(&a.file_path);
|
||||
match std::fs::metadata(&abs_path) {
|
||||
match tokio::fs::metadata(&abs_path).await {
|
||||
Ok(m) => total += m.len() as usize,
|
||||
Err(_) => {
|
||||
// 任一文件缺失即视为缓存不完整
|
||||
return None;
|
||||
}
|
||||
}
|
||||
@ -292,7 +325,10 @@ pub fn cached_files_total_size(library_dir: &Path, artifacts: &[Artifact]) -> Op
|
||||
|
||||
/// 记录缓存写入失败但不中断流程
|
||||
pub fn log_write_failure(source: &str, product: &str, e: anyhow::Error) {
|
||||
error!("[Observation] 写入 observation_cache 失败 ({}/{}): {}", source, product, e);
|
||||
error!(
|
||||
"[Observation] 写入 observation_cache 失败 ({}/{}): {}",
|
||||
source, product, e
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@ -22,11 +22,11 @@ use tracing::{info, warn};
|
||||
|
||||
// ── 版本枚举 ──
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum DesiRelease { Edr, Dr1 }
|
||||
|
||||
impl Default for DesiRelease {
|
||||
fn default() -> Self { Self::Dr1 }
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum DesiRelease {
|
||||
Edr,
|
||||
#[default]
|
||||
Dr1,
|
||||
}
|
||||
|
||||
impl DesiRelease {
|
||||
@ -34,16 +34,28 @@ impl DesiRelease {
|
||||
&["edr", "dr1"]
|
||||
}
|
||||
pub fn dr_segment(&self) -> &'static str {
|
||||
match self { Self::Edr => "edr", Self::Dr1 => "dr1" }
|
||||
match self {
|
||||
Self::Edr => "edr",
|
||||
Self::Dr1 => "dr1",
|
||||
}
|
||||
}
|
||||
pub fn specredux_ver(&self) -> &'static str {
|
||||
match self { Self::Edr => "fuji", Self::Dr1 => "iron" }
|
||||
match self {
|
||||
Self::Edr => "fuji",
|
||||
Self::Dr1 => "iron",
|
||||
}
|
||||
}
|
||||
pub fn datalab_table(&self) -> &'static str {
|
||||
match self { Self::Edr => "desi_edr.zpix", Self::Dr1 => "desi_dr1.zpix" }
|
||||
match self {
|
||||
Self::Edr => "desi_edr.zpix",
|
||||
Self::Dr1 => "desi_dr1.zpix",
|
||||
}
|
||||
}
|
||||
pub fn display(&self) -> &'static str {
|
||||
match self { Self::Edr => "EDR", Self::Dr1 => "DR1" }
|
||||
match self {
|
||||
Self::Edr => "EDR",
|
||||
Self::Dr1 => "DR1",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -52,7 +64,13 @@ pub fn parse_desi_release(s: Option<&str>) -> Result<DesiRelease> {
|
||||
None => DesiRelease::Dr1,
|
||||
Some("edr") => DesiRelease::Edr,
|
||||
Some("dr1") => DesiRelease::Dr1,
|
||||
Some(other) => return Err(anyhow!("不支持的 DESI release '{}'，可选: {:?}", other, DesiRelease::valid_values())),
|
||||
Some(other) => {
|
||||
return Err(anyhow!(
|
||||
"不支持的 DESI release '{}'，可选: {:?}",
|
||||
other,
|
||||
DesiRelease::valid_values()
|
||||
))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@ -114,8 +132,12 @@ impl ObservationFetcher for DesiSpectrumFetcher {
|
||||
async fn cone_search_raw(
|
||||
&self,
|
||||
state: &AppState,
|
||||
ra: f64, dec: f64, radius_deg: f64,
|
||||
release: Option<&str>, _subtype: Option<&str>,
|
||||
ra: f64,
|
||||
dec: f64,
|
||||
radius_deg: f64,
|
||||
release: Option<&str>,
|
||||
_subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
) -> Result<Vec<Candidate>> {
|
||||
validate_cone_params(ra, dec, radius_deg)?;
|
||||
let rel = parse_desi_release(release)?;
|
||||
@ -128,6 +150,7 @@ impl ObservationFetcher for DesiSpectrumFetcher {
|
||||
identifier: &str,
|
||||
_release: Option<&str>,
|
||||
_subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
) -> Result<Candidate> {
|
||||
// 格式 "{survey}-{program}-{healpix}"
|
||||
let parts: Vec<&str> = identifier.split('-').collect();
|
||||
@ -137,7 +160,28 @@ impl ObservationFetcher for DesiSpectrumFetcher {
|
||||
identifier
|
||||
));
|
||||
}
|
||||
let healpix: i64 = parts[2].parse()
|
||||
let survey = parts[0];
|
||||
let program = parts[1];
|
||||
// 白名单校验：survey/program 会拼进文件系统路径和下载 URL，
|
||||
// 必须防止路径穿越（如 "../etc"）。仅允许 DESI 实际使用的取值。
|
||||
const VALID_SURVEYS: &[&str] = &["main", "sv1", "sv2", "sv3", "sv", "backup", "cumulative"];
|
||||
const VALID_PROGRAMS: &[&str] = &["dark", "bright", "backup", "other"];
|
||||
if !VALID_SURVEYS.contains(&survey) {
|
||||
return Err(anyhow!(
|
||||
"DESI survey '{}' 不在允许列表 {:?} 中",
|
||||
survey,
|
||||
VALID_SURVEYS
|
||||
));
|
||||
}
|
||||
if !VALID_PROGRAMS.contains(&program) {
|
||||
return Err(anyhow!(
|
||||
"DESI program '{}' 不在允许列表 {:?} 中",
|
||||
program,
|
||||
VALID_PROGRAMS
|
||||
));
|
||||
}
|
||||
let healpix: i64 = parts[2]
|
||||
.parse()
|
||||
.map_err(|_| anyhow!("healpix 非数字: '{}'", parts[2]))?;
|
||||
if healpix < 0 {
|
||||
return Err(anyhow!("healpix 应为非负整数，当前: {}", healpix));
|
||||
@ -145,10 +189,12 @@ impl ObservationFetcher for DesiSpectrumFetcher {
|
||||
Ok(Candidate {
|
||||
source: Source::Desi,
|
||||
source_id: identifier.to_string(),
|
||||
label: format!("{} {} healpix {}", parts[0], parts[1], healpix),
|
||||
ra: None, dec: None, distance: None,
|
||||
label: format!("{} {} healpix {}", survey, program, healpix),
|
||||
ra: None,
|
||||
dec: None,
|
||||
distance: None,
|
||||
raw: Some(serde_json::json!({
|
||||
"survey": parts[0], "program": parts[1], "healpix": healpix
|
||||
"survey": survey, "program": program, "healpix": healpix
|
||||
})),
|
||||
})
|
||||
}
|
||||
@ -159,13 +205,26 @@ impl ObservationFetcher for DesiSpectrumFetcher {
|
||||
candidate: &Candidate,
|
||||
release: Option<&str>,
|
||||
_subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
force: bool,
|
||||
) -> Result<ObservationProduct> {
|
||||
let survey = candidate.raw.as_ref().and_then(|v| v.get("survey")).and_then(|v| v.as_str())
|
||||
let survey = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("survey"))
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow!("DESI Candidate 缺 survey"))?;
|
||||
let program = candidate.raw.as_ref().and_then(|v| v.get("program")).and_then(|v| v.as_str())
|
||||
let program = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("program"))
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow!("DESI Candidate 缺 program"))?;
|
||||
let healpix = candidate.raw.as_ref().and_then(|v| v.get("healpix")).and_then(|v| v.as_i64())
|
||||
let healpix = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("healpix"))
|
||||
.and_then(|v| v.as_i64())
|
||||
.ok_or_else(|| anyhow!("DESI Candidate 缺 healpix"))?;
|
||||
let rel = parse_desi_release(release)?;
|
||||
let product = ProductSpec::with_subtype(ProductType::Spectrum, "coadd");
|
||||
@ -175,12 +234,21 @@ impl ObservationFetcher for DesiSpectrumFetcher {
|
||||
|
||||
// 1) 缓存命中
|
||||
if !force {
|
||||
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Desi, &product, &cache_key).await? {
|
||||
if cached_files_total_size(&state.config.library_dir, &artifacts).is_some() {
|
||||
if let Some((artifacts, meta)) =
|
||||
fetch_observation_cache(&state.db, Source::Desi, &product, &cache_key).await?
|
||||
{
|
||||
if cached_files_total_size(&state.config.library_dir, &artifacts)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
info!("[DESI] coadd 缓存命中 ({})", cache_key);
|
||||
return Ok(ObservationProduct {
|
||||
source: Source::Desi, product,
|
||||
source_id: cache_key, source_label, artifacts, source_meta: meta,
|
||||
source: Source::Desi,
|
||||
product,
|
||||
source_id: cache_key,
|
||||
source_label,
|
||||
artifacts,
|
||||
source_meta: meta,
|
||||
});
|
||||
}
|
||||
warn!("[DESI] 缓存记录存在但文件缺失，重新下载 ({})", cache_key);
|
||||
@ -189,34 +257,57 @@ impl ObservationFetcher for DesiSpectrumFetcher {
|
||||
|
||||
// 2) 下载 coadd FITS（无压缩）
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let fits_bytes = state.desi.download_coadd(survey, program, healpix, rel).await?;
|
||||
let fits_bytes = state
|
||||
.desi
|
||||
.download_coadd(survey, program, healpix, rel)
|
||||
.await?;
|
||||
let rel_path = format!(
|
||||
"Telescope/desi/{dr}/{survey}-{program}/{healpix}.fits",
|
||||
dr = dr, survey = survey, program = program, healpix = healpix
|
||||
dr = dr,
|
||||
survey = survey,
|
||||
program = program,
|
||||
healpix = healpix
|
||||
);
|
||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes).await?;
|
||||
info!(
|
||||
"[DESI] coadd 已保存 {} → {} ({}B)",
|
||||
cache_key, rel_path, size
|
||||
);
|
||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes)?;
|
||||
info!("[DESI] coadd 已保存 {} → {} ({}B)", cache_key, rel_path, size);
|
||||
|
||||
let meta = serde_json::json!({"survey": survey, "program": program, "healpix": healpix, "dr": dr});
|
||||
let meta =
|
||||
serde_json::json!({"survey": survey, "program": program, "healpix": healpix, "dr": dr});
|
||||
let artifact = Artifact {
|
||||
band: None, original_name: None,
|
||||
band: None,
|
||||
original_name: None,
|
||||
file_path: rel_path.clone(),
|
||||
file_url: file_url_from_path(&rel_path),
|
||||
file_format: "fits".to_string(),
|
||||
size_bytes: size, cached: false,
|
||||
size_bytes: size,
|
||||
cached: false,
|
||||
};
|
||||
let meta_str = meta.to_string();
|
||||
if let Err(e) = write_observation_cache(
|
||||
&state.db, Source::Desi, &product, &cache_key, None, None,
|
||||
std::slice::from_ref(&artifact), Some(&meta_str),
|
||||
).await {
|
||||
&state.db,
|
||||
Source::Desi,
|
||||
&product,
|
||||
&cache_key,
|
||||
None,
|
||||
None,
|
||||
std::slice::from_ref(&artifact),
|
||||
Some(&meta_str),
|
||||
)
|
||||
.await
|
||||
{
|
||||
log_write_failure(Source::Desi.as_str(), "spectrum", e);
|
||||
}
|
||||
|
||||
Ok(ObservationProduct {
|
||||
source: Source::Desi, product,
|
||||
source_id: cache_key, source_label,
|
||||
artifacts: vec![artifact], source_meta: Some(meta),
|
||||
source: Source::Desi,
|
||||
product,
|
||||
source_id: cache_key,
|
||||
source_label,
|
||||
artifacts: vec![artifact],
|
||||
source_meta: Some(meta),
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -226,7 +317,9 @@ fn row_to_candidate(row: &DesiSpectrumRow) -> Candidate {
|
||||
source: Source::Desi,
|
||||
source_id: format!("{}-{}-{}", row.survey, row.program, row.healpix),
|
||||
label: format!("{} {} healpix {}", row.survey, row.program, row.healpix),
|
||||
ra: row.ra, dec: row.dec, distance: None,
|
||||
ra: row.ra,
|
||||
dec: row.dec,
|
||||
distance: None,
|
||||
raw: Some(serde_json::json!({
|
||||
"survey": row.survey, "program": row.program, "healpix": row.healpix
|
||||
})),
|
||||
|
||||
@ -30,6 +30,7 @@ pub async fn search_observation(
|
||||
dec: f64,
|
||||
radius_deg: f64,
|
||||
release: Option<&str>,
|
||||
version: Option<&str>,
|
||||
) -> Result<Vec<Candidate>> {
|
||||
let subtype = product.subtype.as_deref();
|
||||
let fetcher = registry.get(source, product.product, subtype)?;
|
||||
@ -38,7 +39,7 @@ pub async fn search_observation(
|
||||
source, product.product, ra, dec, radius_deg
|
||||
);
|
||||
let matched = fetcher
|
||||
.cone_search_cached(state, ra, dec, radius_deg, release, subtype)
|
||||
.cone_search_cached(state, ra, dec, radius_deg, release, subtype, version)
|
||||
.await?;
|
||||
Ok(matched)
|
||||
}
|
||||
@ -54,19 +55,48 @@ pub async fn download_observation(
|
||||
) -> Result<ObservationBatch> {
|
||||
match request {
|
||||
ObservationRequest::ByCoordinates {
|
||||
source, product, ra, dec, radius_deg, strategy, release,
|
||||
source,
|
||||
product,
|
||||
ra,
|
||||
dec,
|
||||
radius_deg,
|
||||
strategy,
|
||||
release,
|
||||
version,
|
||||
} => {
|
||||
download_by_coordinates(
|
||||
state, registry, *source, product, *ra, *dec, *radius_deg,
|
||||
*strategy, release.as_deref(), force,
|
||||
).await
|
||||
state,
|
||||
registry,
|
||||
*source,
|
||||
product,
|
||||
*ra,
|
||||
*dec,
|
||||
*radius_deg,
|
||||
*strategy,
|
||||
release.as_deref(),
|
||||
version.as_deref(),
|
||||
force,
|
||||
)
|
||||
.await
|
||||
}
|
||||
ObservationRequest::ByIdentifier {
|
||||
source, product, identifiers, release,
|
||||
source,
|
||||
product,
|
||||
identifiers,
|
||||
release,
|
||||
version,
|
||||
} => {
|
||||
download_by_identifiers(
|
||||
state, registry, *source, product, identifiers, release.as_deref(), force,
|
||||
).await
|
||||
state,
|
||||
registry,
|
||||
*source,
|
||||
product,
|
||||
identifiers,
|
||||
release.as_deref(),
|
||||
version.as_deref(),
|
||||
force,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -77,9 +107,12 @@ async fn download_by_coordinates(
|
||||
registry: &ObservationRegistry,
|
||||
source: crate::services::observation::types::Source,
|
||||
product: &ProductSpec,
|
||||
ra: f64, dec: f64, radius_deg: f64,
|
||||
ra: f64,
|
||||
dec: f64,
|
||||
radius_deg: f64,
|
||||
strategy: FindStrategy,
|
||||
release: Option<&str>,
|
||||
version: Option<&str>,
|
||||
force: bool,
|
||||
) -> Result<ObservationBatch> {
|
||||
let subtype = product.subtype.as_deref();
|
||||
@ -89,14 +122,21 @@ async fn download_by_coordinates(
|
||||
source, product.product, ra, dec, radius_deg, strategy
|
||||
);
|
||||
|
||||
let matched = fetcher.cone_search_cached(state, ra, dec, radius_deg, release, subtype).await?;
|
||||
let matched = fetcher
|
||||
.cone_search_cached(state, ra, dec, radius_deg, release, subtype, version)
|
||||
.await?;
|
||||
let matched_count = matched.len();
|
||||
|
||||
if matched_count == 0 {
|
||||
return Ok(ObservationBatch {
|
||||
source, product: product.clone(),
|
||||
ra: Some(ra), dec: Some(dec), radius_deg: Some(radius_deg),
|
||||
matched_count: 0, products: Vec::new(), failures: Vec::new(),
|
||||
source,
|
||||
product: product.clone(),
|
||||
ra: Some(ra),
|
||||
dec: Some(dec),
|
||||
radius_deg: Some(radius_deg),
|
||||
matched_count: 0,
|
||||
products: Vec::new(),
|
||||
failures: Vec::new(),
|
||||
});
|
||||
}
|
||||
|
||||
@ -109,7 +149,10 @@ async fn download_by_coordinates(
|
||||
let mut failures = Vec::new();
|
||||
for idx in targets {
|
||||
let candidate = &matched[idx];
|
||||
match fetcher.fetch(state, candidate, release, subtype, force).await {
|
||||
match fetcher
|
||||
.fetch(state, candidate, release, subtype, version, force)
|
||||
.await
|
||||
{
|
||||
Ok(p) => products.push(p),
|
||||
Err(e) => {
|
||||
let label = candidate.label.clone();
|
||||
@ -123,9 +166,14 @@ async fn download_by_coordinates(
|
||||
}
|
||||
|
||||
Ok(ObservationBatch {
|
||||
source, product: product.clone(),
|
||||
ra: Some(ra), dec: Some(dec), radius_deg: Some(radius_deg),
|
||||
matched_count, products, failures,
|
||||
source,
|
||||
product: product.clone(),
|
||||
ra: Some(ra),
|
||||
dec: Some(dec),
|
||||
radius_deg: Some(radius_deg),
|
||||
matched_count,
|
||||
products,
|
||||
failures,
|
||||
})
|
||||
}
|
||||
|
||||
@ -137,20 +185,34 @@ async fn download_by_identifiers(
|
||||
product: &ProductSpec,
|
||||
identifiers: &[String],
|
||||
release: Option<&str>,
|
||||
version: Option<&str>,
|
||||
force: bool,
|
||||
) -> Result<ObservationBatch> {
|
||||
let subtype = product.subtype.as_deref();
|
||||
let fetcher = registry.get(source, product.product, subtype)?;
|
||||
if identifiers.is_empty() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"标识符列表不能为空（source={:?} product={:?}）",
|
||||
source,
|
||||
product.product
|
||||
));
|
||||
}
|
||||
info!(
|
||||
"[Observation] by_id source={:?} product={:?} count={} release={:?}",
|
||||
source, product.product, identifiers.len(), release
|
||||
source,
|
||||
product.product,
|
||||
identifiers.len(),
|
||||
release
|
||||
);
|
||||
|
||||
let mut products = Vec::with_capacity(identifiers.len());
|
||||
let mut failures = Vec::new();
|
||||
for id in identifiers {
|
||||
// 先把标识符解析为 Candidate（验证格式 + 提取字段）
|
||||
let candidate = match fetcher.resolve_identifier(id, release, subtype).await {
|
||||
let candidate = match fetcher
|
||||
.resolve_identifier(id, release, subtype, version)
|
||||
.await
|
||||
{
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
warn!("[Observation] 标识符解析失败 {}: {}", id, e);
|
||||
@ -161,7 +223,10 @@ async fn download_by_identifiers(
|
||||
continue;
|
||||
}
|
||||
};
|
||||
match fetcher.fetch(state, &candidate, release, subtype, force).await {
|
||||
match fetcher
|
||||
.fetch(state, &candidate, release, subtype, version, force)
|
||||
.await
|
||||
{
|
||||
Ok(p) => products.push(p),
|
||||
Err(e) => {
|
||||
warn!("[Observation] 下载失败 {}: {}", id, e);
|
||||
@ -174,10 +239,14 @@ async fn download_by_identifiers(
|
||||
}
|
||||
|
||||
Ok(ObservationBatch {
|
||||
source, product: product.clone(),
|
||||
ra: None, dec: None, radius_deg: None,
|
||||
source,
|
||||
product: product.clone(),
|
||||
ra: None,
|
||||
dec: None,
|
||||
radius_deg: None,
|
||||
matched_count: products.len() + failures.len(),
|
||||
products, failures,
|
||||
products,
|
||||
failures,
|
||||
})
|
||||
}
|
||||
|
||||
@ -189,10 +258,15 @@ pub async fn download_one(
|
||||
product: ProductSpec,
|
||||
identifier: &str,
|
||||
release: Option<&str>,
|
||||
version: Option<&str>,
|
||||
force: bool,
|
||||
) -> Result<ObservationProduct> {
|
||||
let subtype = product.subtype.as_deref();
|
||||
let fetcher = registry.get(source, product.product, subtype)?;
|
||||
let candidate = fetcher.resolve_identifier(identifier, release, subtype).await?;
|
||||
fetcher.fetch(state, &candidate, release, subtype, force).await
|
||||
let candidate = fetcher
|
||||
.resolve_identifier(identifier, release, subtype, version)
|
||||
.await?;
|
||||
fetcher
|
||||
.fetch(state, &candidate, release, subtype, version, force)
|
||||
.await
|
||||
}
|
||||
|
||||
@ -76,6 +76,33 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
||||
None
|
||||
}
|
||||
|
||||
/// 该 release 支持的子版本列表（仅 LAMOST 有意义），返回空表示不区分子版本。
|
||||
/// `release` 为 None 时返回默认 release 的子版本列表。
|
||||
fn versions_for_release(&self, _release: Option<&str>) -> Vec<String> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// 该 release 的默认子版本（None 表示无子版本概念或用默认 release）
|
||||
fn default_version_for_release(&self, _release: Option<&str>) -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
/// 需要登录认证才能访问的 release 列表（如 LAMOST Internal DR），空表示全部公开。
|
||||
///
|
||||
/// 前端据此对这些 release 做禁用/灰显处理。
|
||||
fn releases_requiring_auth(&self) -> Vec<String> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// 每个 release 实际支持的 subtype 列表（用于表达 release×subtype 交叉约束）。
|
||||
///
|
||||
/// 默认返回空 HashMap，表示所有 release 都支持 `subtypes()` 的全部值。
|
||||
/// 仅当某些 release 不支持某些 subtype 时才需覆写（如 LAMOST DR5/6 不支持 MRS）。
|
||||
/// key=release 字符串，value=该 release 支持的 subtype 列表。
|
||||
fn subtypes_per_release(&self) -> std::collections::HashMap<String, Vec<String>> {
|
||||
std::collections::HashMap::new()
|
||||
}
|
||||
|
||||
/// 坐标模式检索半径的建议上限（度），用于前端表单提示与服务端警告
|
||||
///
|
||||
/// 各源主表规模差异极大（Gaia/SDSS/DESI 主表上亿行，大范围查询易超时），
|
||||
@ -109,6 +136,8 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
||||
// ── 检索（各源实现） ──
|
||||
|
||||
/// 纯 cone 检索（不查缓存）—— 各源实现自己的端点调用与行映射
|
||||
///
|
||||
/// `version` 为该 release 的子版本（仅 LAMOST 有意义，其他源可忽略）。
|
||||
async fn cone_search_raw(
|
||||
&self,
|
||||
state: &AppState,
|
||||
@ -117,6 +146,7 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
||||
radius_deg: f64,
|
||||
release: Option<&str>,
|
||||
subtype: Option<&str>,
|
||||
version: Option<&str>,
|
||||
) -> Result<Vec<Candidate>>;
|
||||
|
||||
/// 标识符模式：验证并归一化为 Candidate（不发起下载）
|
||||
@ -125,6 +155,7 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
||||
identifier: &str,
|
||||
release: Option<&str>,
|
||||
subtype: Option<&str>,
|
||||
version: Option<&str>,
|
||||
) -> Result<Candidate>;
|
||||
|
||||
// ── 下载（各源实现） ──
|
||||
@ -136,6 +167,7 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
||||
candidate: &Candidate,
|
||||
release: Option<&str>,
|
||||
subtype: Option<&str>,
|
||||
version: Option<&str>,
|
||||
force: bool,
|
||||
) -> Result<ObservationProduct>;
|
||||
|
||||
@ -159,8 +191,9 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
||||
radius_deg: f64,
|
||||
release: Option<&str>,
|
||||
subtype: Option<&str>,
|
||||
version: Option<&str>,
|
||||
) -> Result<Vec<Candidate>> {
|
||||
let query_hash = self.cone_cache_hash(ra, dec, radius_deg, release, subtype);
|
||||
let query_hash = self.cone_cache_hash(ra, dec, radius_deg, release, subtype, version);
|
||||
|
||||
if let Some(cached) = fetch_cone_cache(&state.db, &query_hash).await? {
|
||||
tracing::info!(
|
||||
@ -174,10 +207,12 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
let result = self
|
||||
.cone_search_raw(state, ra, dec, radius_deg, release, subtype)
|
||||
.cone_search_raw(state, ra, dec, radius_deg, release, subtype, version)
|
||||
.await?;
|
||||
|
||||
if let Err(e) = write_cone_cache(&state.db, &query_hash, &self.cone_cache_prefix(), &result).await {
|
||||
if let Err(e) =
|
||||
write_cone_cache(&state.db, &query_hash, &self.cone_cache_prefix(), &result).await
|
||||
{
|
||||
error!(
|
||||
"[Observation] 写入 cone 缓存失败 ({:?}/{:?}): {}",
|
||||
self.key().0,
|
||||
@ -188,7 +223,7 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// 计算 cone 缓存的 sha1 hash（prefix + 坐标 + release + subtype）
|
||||
/// 计算 cone 缓存的 sha1 hash（prefix + 坐标 + release + subtype + version）
|
||||
fn cone_cache_hash(
|
||||
&self,
|
||||
ra: f64,
|
||||
@ -196,18 +231,42 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
||||
radius_deg: f64,
|
||||
release: Option<&str>,
|
||||
subtype: Option<&str>,
|
||||
version: Option<&str>,
|
||||
) -> String {
|
||||
// 各字段加长度前缀 + 字段标记字节，避免不同组合拼接后产生相同字节流。
|
||||
// 例如：release="dr1"+subtype=None 与 release="dr"+subtype=Some("1")
|
||||
// 不加界定会哈希到相同字节序列，导致缓存错误命中。
|
||||
let mut h = Sha1::new();
|
||||
h.update(self.cone_cache_prefix().as_bytes());
|
||||
let prefix = self.cone_cache_prefix();
|
||||
h.update((prefix.len() as u64).to_le_bytes());
|
||||
h.update(prefix.as_bytes());
|
||||
h.update(ra.to_le_bytes());
|
||||
h.update(dec.to_le_bytes());
|
||||
h.update(radius_deg.to_le_bytes());
|
||||
if let Some(r) = release {
|
||||
match release {
|
||||
Some(r) => {
|
||||
h.update([1u8]); // presence marker
|
||||
h.update((r.len() as u64).to_le_bytes());
|
||||
h.update(r.as_bytes());
|
||||
}
|
||||
if let Some(st) = subtype {
|
||||
None => h.update([0u8]),
|
||||
}
|
||||
match subtype {
|
||||
Some(st) => {
|
||||
h.update([1u8]); // presence marker
|
||||
h.update((st.len() as u64).to_le_bytes());
|
||||
h.update(st.as_bytes());
|
||||
}
|
||||
None => h.update([0u8]),
|
||||
}
|
||||
match version {
|
||||
Some(v) => {
|
||||
h.update([1u8]); // presence marker
|
||||
h.update((v.len() as u64).to_le_bytes());
|
||||
h.update(v.as_bytes());
|
||||
}
|
||||
None => h.update([0u8]),
|
||||
}
|
||||
format!("{:x}", h.finalize())
|
||||
}
|
||||
}
|
||||
@ -224,10 +283,7 @@ struct ConeCacheRow {
|
||||
expires_at: Option<chrono::DateTime<chrono::Utc>>,
|
||||
}
|
||||
|
||||
async fn fetch_cone_cache(
|
||||
pool: &SqlitePool,
|
||||
query_hash: &str,
|
||||
) -> Result<Option<Vec<Candidate>>> {
|
||||
async fn fetch_cone_cache(pool: &SqlitePool, query_hash: &str) -> Result<Option<Vec<Candidate>>> {
|
||||
let row = sqlx::query_as::<_, ConeCacheRow>(
|
||||
"SELECT result_json, expires_at FROM vizier_query_cache WHERE query_hash = ? LIMIT 1",
|
||||
)
|
||||
@ -335,12 +391,22 @@ mod tests {
|
||||
fn test_pick_nearest_without_distance_uses_coords() {
|
||||
let rows = vec![
|
||||
Candidate {
|
||||
source: Source::Lamost, source_id: String::new(), label: String::new(),
|
||||
ra: Some(10.5), dec: Some(41.0), distance: None, raw: None,
|
||||
source: Source::Lamost,
|
||||
source_id: String::new(),
|
||||
label: String::new(),
|
||||
ra: Some(10.5),
|
||||
dec: Some(41.0),
|
||||
distance: None,
|
||||
raw: None,
|
||||
},
|
||||
Candidate {
|
||||
source: Source::Lamost, source_id: String::new(), label: String::new(),
|
||||
ra: Some(10.51), dec: Some(41.01), distance: None, raw: None,
|
||||
source: Source::Lamost,
|
||||
source_id: String::new(),
|
||||
label: String::new(),
|
||||
ra: Some(10.51),
|
||||
dec: Some(41.01),
|
||||
distance: None,
|
||||
raw: None,
|
||||
},
|
||||
];
|
||||
assert_eq!(pick_nearest(&rows, 10.5, 41.0), 0);
|
||||
|
||||
@ -28,21 +28,34 @@ use tracing::{info, warn};
|
||||
// ── 版本与解析辅助 ──
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum GaiaRelease { #[default] Dr3 }
|
||||
pub enum GaiaRelease {
|
||||
#[default]
|
||||
Dr3,
|
||||
}
|
||||
|
||||
impl GaiaRelease {
|
||||
pub fn valid_values() -> &'static [&'static str] {
|
||||
&["dr3"]
|
||||
}
|
||||
pub fn dr_str(&self) -> &'static str { "dr3" }
|
||||
pub fn tap_table(&self) -> &'static str { "gaiadr3.gaia_source" }
|
||||
pub fn display(&self) -> &'static str { "DR3" }
|
||||
pub fn dr_str(&self) -> &'static str {
|
||||
"dr3"
|
||||
}
|
||||
pub fn tap_table(&self) -> &'static str {
|
||||
"gaiadr3.gaia_source"
|
||||
}
|
||||
pub fn display(&self) -> &'static str {
|
||||
"DR3"
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_gaia_release(s: Option<&str>) -> Result<GaiaRelease> {
|
||||
match s {
|
||||
None | Some("dr3") => Ok(GaiaRelease::Dr3),
|
||||
Some(other) => Err(anyhow!("不支持的 Gaia release '{}'，可选: {:?}", other, GaiaRelease::valid_values())),
|
||||
Some(other) => Err(anyhow!(
|
||||
"不支持的 Gaia release '{}'，可选: {:?}",
|
||||
other,
|
||||
GaiaRelease::valid_values()
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
@ -53,7 +66,11 @@ pub fn parse_gaia_retrieval_type_str(s: Option<&str>) -> Result<GaiaRetrievalTyp
|
||||
Some("XP_SAMPLED") => Ok(GaiaRetrievalType::XpSampled),
|
||||
Some("EPOCH_PHOTOMETRY") => Ok(GaiaRetrievalType::EpochPhotometry),
|
||||
Some("RVS") => Ok(GaiaRetrievalType::Rvs),
|
||||
Some(other) => Err(anyhow!("不支持的 Gaia subtype '{}'，可选: {:?}", other, GaiaRetrievalType::valid_values())),
|
||||
Some(other) => Err(anyhow!(
|
||||
"不支持的 Gaia subtype '{}'，可选: {:?}",
|
||||
other,
|
||||
GaiaRetrievalType::valid_values()
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
@ -64,7 +81,13 @@ pub fn parse_gaia_retrieval_type(s: &str) -> Result<GaiaRetrievalType> {
|
||||
"XP_SAMPLED" => XpSampled,
|
||||
"EPOCH_PHOTOMETRY" => EpochPhotometry,
|
||||
"RVS" => Rvs,
|
||||
other => return Err(anyhow!("不支持的 Gaia retrieval_type '{}'，可选: {:?}", other, GaiaRetrievalType::valid_values())),
|
||||
other => {
|
||||
return Err(anyhow!(
|
||||
"不支持的 Gaia retrieval_type '{}'，可选: {:?}",
|
||||
other,
|
||||
GaiaRetrievalType::valid_values()
|
||||
))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@ -120,29 +143,52 @@ async fn download_gaia_product(
|
||||
|
||||
// 1) 缓存命中
|
||||
if !force {
|
||||
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Gaia, &product, &cache_key).await? {
|
||||
if cached_files_total_size(&state.config.library_dir, &artifacts).is_some() {
|
||||
info!("[Gaia] 缓存命中 (source_id={}, type={})", source_id, rt_param);
|
||||
let artifacts = artifacts.into_iter().map(|mut a| {
|
||||
if let Some((artifacts, meta)) =
|
||||
fetch_observation_cache(&state.db, Source::Gaia, &product, &cache_key).await?
|
||||
{
|
||||
if cached_files_total_size(&state.config.library_dir, &artifacts)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
info!(
|
||||
"[Gaia] 缓存命中 (source_id={}, type={})",
|
||||
source_id, rt_param
|
||||
);
|
||||
let artifacts = artifacts
|
||||
.into_iter()
|
||||
.map(|mut a| {
|
||||
if a.size_bytes == 0 {
|
||||
if let Ok(m) = std::fs::metadata(state.config.library_dir.join(&a.file_path)) {
|
||||
if let Ok(m) =
|
||||
std::fs::metadata(state.config.library_dir.join(&a.file_path))
|
||||
{
|
||||
a.size_bytes = m.len() as usize;
|
||||
}
|
||||
}
|
||||
a
|
||||
}).collect();
|
||||
})
|
||||
.collect();
|
||||
return Ok(ObservationProduct {
|
||||
source: Source::Gaia, product,
|
||||
source_id: cache_key, source_label, artifacts, source_meta: meta,
|
||||
source: Source::Gaia,
|
||||
product,
|
||||
source_id: cache_key,
|
||||
source_label,
|
||||
artifacts,
|
||||
source_meta: meta,
|
||||
});
|
||||
}
|
||||
warn!("[Gaia] 缓存记录存在但文件缺失，重新下载 (source_id={})", source_id);
|
||||
warn!(
|
||||
"[Gaia] 缓存记录存在但文件缺失，重新下载 (source_id={})",
|
||||
source_id
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 2) DataLink 下载 ZIP + 解包取**全部**文件（修复 EPOCH_PHOTOMETRY 多文件丢失）
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let zip_bytes = state.gaia.download_products(&[source_id.to_string()], retrieval_type, "fits").await?;
|
||||
let zip_bytes = state
|
||||
.gaia
|
||||
.download_products(&[source_id.to_string()], retrieval_type, "fits")
|
||||
.await?;
|
||||
let extracted = extract_all_zip_entries(&zip_bytes, source_id)?;
|
||||
|
||||
// 3) 落盘
|
||||
@ -152,13 +198,21 @@ async fn download_gaia_product(
|
||||
for ext_file in extracted {
|
||||
let rel_path = if product_type == ProductType::LightCurve {
|
||||
let safe_name = ext_file.name.replace('/', "_");
|
||||
format!("Telescope/gaia/lightcurve/{rt}/{sid}/{name}",
|
||||
rt = rt_lower, sid = source_id, name = safe_name)
|
||||
format!(
|
||||
"Telescope/gaia/lightcurve/{rt}/{sid}/{name}",
|
||||
rt = rt_lower,
|
||||
sid = source_id,
|
||||
name = safe_name
|
||||
)
|
||||
} else {
|
||||
format!("Telescope/gaia/spectrum/{rt}/{sid}.{ext}",
|
||||
rt = rt_lower, sid = source_id, ext = ext_file.ext)
|
||||
format!(
|
||||
"Telescope/gaia/spectrum/{rt}/{sid}.{ext}",
|
||||
rt = rt_lower,
|
||||
sid = source_id,
|
||||
ext = ext_file.ext
|
||||
)
|
||||
};
|
||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &ext_file.bytes)?;
|
||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &ext_file.bytes).await?;
|
||||
info!("[Gaia] 已保存 {} ({}B) → {}", ext_file.name, size, rel_path);
|
||||
artifacts.push(Artifact {
|
||||
band: parse_band_from_name(&ext_file.name),
|
||||
@ -174,22 +228,37 @@ async fn download_gaia_product(
|
||||
a.file_url = file_url_from_path(&a.file_path);
|
||||
}
|
||||
if artifacts.is_empty() {
|
||||
return Err(anyhow!("Gaia DataLink ZIP 内无可下载文件条目 (source_id={}, type={})", source_id, rt_param));
|
||||
return Err(anyhow!(
|
||||
"Gaia DataLink ZIP 内无可下载文件条目 (source_id={}, type={})",
|
||||
source_id,
|
||||
rt_param
|
||||
));
|
||||
}
|
||||
|
||||
let meta = serde_json::json!({"source_id": source_id, "retrieval_type": rt_param, "dr": dr});
|
||||
let meta_str = meta.to_string();
|
||||
if let Err(e) = write_observation_cache(
|
||||
&state.db, Source::Gaia, &product, &cache_key, None, None,
|
||||
&artifacts, Some(&meta_str),
|
||||
).await {
|
||||
&state.db,
|
||||
Source::Gaia,
|
||||
&product,
|
||||
&cache_key,
|
||||
None,
|
||||
None,
|
||||
&artifacts,
|
||||
Some(&meta_str),
|
||||
)
|
||||
.await
|
||||
{
|
||||
log_write_failure(Source::Gaia.as_str(), product.product.as_str(), e);
|
||||
}
|
||||
|
||||
Ok(ObservationProduct {
|
||||
source: Source::Gaia, product,
|
||||
source_id: cache_key, source_label,
|
||||
artifacts, source_meta: Some(meta),
|
||||
source: Source::Gaia,
|
||||
product,
|
||||
source_id: cache_key,
|
||||
source_label,
|
||||
artifacts,
|
||||
source_meta: Some(meta),
|
||||
})
|
||||
}
|
||||
|
||||
@ -208,30 +277,48 @@ struct ExtractedFile {
|
||||
/// 旧实现 `extract_first_zip_entry` 只取首个文件，丢失了 BP/RP 波段——本函数修复之。
|
||||
fn extract_all_zip_entries(zip_bytes: &[u8], source_id: &str) -> Result<Vec<ExtractedFile>> {
|
||||
let cursor = Cursor::new(zip_bytes);
|
||||
let mut archive = zip::ZipArchive::new(cursor).map_err(|e| anyhow!("Gaia ZIP 解析失败: {}", e))?;
|
||||
let mut archive =
|
||||
zip::ZipArchive::new(cursor).map_err(|e| anyhow!("Gaia ZIP 解析失败: {}", e))?;
|
||||
|
||||
let mut out = Vec::new();
|
||||
for i in 0..archive.len() {
|
||||
let mut file = archive.by_index(i)
|
||||
let mut file = archive
|
||||
.by_index(i)
|
||||
.map_err(|e| anyhow!("读取 Gaia ZIP 条目 {} 失败: {}", i, e))?;
|
||||
if file.is_dir() { continue; }
|
||||
if file.is_dir() {
|
||||
continue;
|
||||
}
|
||||
let name = file.name().to_string();
|
||||
let mut buf = Vec::new();
|
||||
file.read_to_end(&mut buf)
|
||||
.map_err(|e| anyhow!("解压 Gaia ZIP 条目失败: {}", e))?;
|
||||
if buf.is_empty() { continue; }
|
||||
if buf.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let lower = name.to_lowercase();
|
||||
let ext = if lower.ends_with(".fits") { "fits" }
|
||||
else if lower.ends_with(".vot") || lower.ends_with(".xml") { "vot" }
|
||||
else if lower.ends_with(".csv") { "csv" }
|
||||
else { "fits" };
|
||||
let ext = if lower.ends_with(".fits") {
|
||||
"fits"
|
||||
} else if lower.ends_with(".vot") || lower.ends_with(".xml") {
|
||||
"vot"
|
||||
} else if lower.ends_with(".csv") {
|
||||
"csv"
|
||||
} else {
|
||||
"fits"
|
||||
};
|
||||
|
||||
out.push(ExtractedFile { name, ext, bytes: buf });
|
||||
out.push(ExtractedFile {
|
||||
name,
|
||||
ext,
|
||||
bytes: buf,
|
||||
});
|
||||
}
|
||||
|
||||
if out.is_empty() {
|
||||
return Err(anyhow!("Gaia ZIP 内无可下载文件条目 (source_id={})", source_id));
|
||||
return Err(anyhow!(
|
||||
"Gaia ZIP 内无可下载文件条目 (source_id={})",
|
||||
source_id
|
||||
));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
@ -284,12 +371,19 @@ impl ObservationFetcher for GaiaSpectrumFetcher {
|
||||
async fn cone_search_raw(
|
||||
&self,
|
||||
state: &AppState,
|
||||
ra: f64, dec: f64, radius_deg: f64,
|
||||
_release: Option<&str>, _subtype: Option<&str>,
|
||||
ra: f64,
|
||||
dec: f64,
|
||||
radius_deg: f64,
|
||||
_release: Option<&str>,
|
||||
_subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
) -> Result<Vec<Candidate>> {
|
||||
validate_cone_params(ra, dec, radius_deg)?;
|
||||
let rel = parse_gaia_release(_release)?;
|
||||
let result = state.gaia.cone_search(ra, dec, radius_deg, 50, true, rel).await?;
|
||||
let result = state
|
||||
.gaia
|
||||
.cone_search(ra, dec, radius_deg, 50, true, rel)
|
||||
.await?;
|
||||
Ok(result.rows.iter().map(row_to_candidate).collect())
|
||||
}
|
||||
|
||||
@ -298,6 +392,7 @@ impl ObservationFetcher for GaiaSpectrumFetcher {
|
||||
identifier: &str,
|
||||
_release: Option<&str>,
|
||||
subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
) -> Result<Candidate> {
|
||||
let (rt_param, gaia_sid) = match identifier.split_once('|') {
|
||||
Some((rt, s)) => (rt, s),
|
||||
@ -315,7 +410,9 @@ impl ObservationFetcher for GaiaSpectrumFetcher {
|
||||
source: Source::Gaia,
|
||||
source_id: gaia_sid.to_string(),
|
||||
label: format!("Gaia {}", gaia_sid),
|
||||
ra: None, dec: None, distance: None,
|
||||
ra: None,
|
||||
dec: None,
|
||||
distance: None,
|
||||
raw: Some(serde_json::json!({"source_id": gaia_sid, "retrieval_type": rt.as_param()})),
|
||||
})
|
||||
}
|
||||
@ -326,13 +423,18 @@ impl ObservationFetcher for GaiaSpectrumFetcher {
|
||||
candidate: &Candidate,
|
||||
_release: Option<&str>,
|
||||
subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
force: bool,
|
||||
) -> Result<ObservationProduct> {
|
||||
let source_id = candidate.raw.as_ref()
|
||||
let source_id = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("source_id"))
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or(&candidate.source_id);
|
||||
let rt = if let Some(rt_str) = candidate.raw.as_ref()
|
||||
let rt = if let Some(rt_str) = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("retrieval_type"))
|
||||
.and_then(|v| v.as_str())
|
||||
{
|
||||
@ -381,13 +483,20 @@ impl ObservationFetcher for GaiaLightCurveFetcher {
|
||||
async fn cone_search_raw(
|
||||
&self,
|
||||
state: &AppState,
|
||||
ra: f64, dec: f64, radius_deg: f64,
|
||||
_release: Option<&str>, _subtype: Option<&str>,
|
||||
ra: f64,
|
||||
dec: f64,
|
||||
radius_deg: f64,
|
||||
_release: Option<&str>,
|
||||
_subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
) -> Result<Vec<Candidate>> {
|
||||
// 光变 cone 复用 Gaia 主表检索（不强制 xp_only，因为光变对所有源都有）
|
||||
validate_cone_params(ra, dec, radius_deg)?;
|
||||
let rel = parse_gaia_release(_release)?;
|
||||
let result = state.gaia.cone_search(ra, dec, radius_deg, 50, false, rel).await?;
|
||||
let result = state
|
||||
.gaia
|
||||
.cone_search(ra, dec, radius_deg, 50, false, rel)
|
||||
.await?;
|
||||
Ok(result.rows.iter().map(row_to_candidate).collect())
|
||||
}
|
||||
|
||||
@ -396,6 +505,7 @@ impl ObservationFetcher for GaiaLightCurveFetcher {
|
||||
identifier: &str,
|
||||
_release: Option<&str>,
|
||||
_subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
) -> Result<Candidate> {
|
||||
let sid = identifier.trim();
|
||||
if sid.is_empty() {
|
||||
@ -405,7 +515,9 @@ impl ObservationFetcher for GaiaLightCurveFetcher {
|
||||
source: Source::Gaia,
|
||||
source_id: sid.to_string(),
|
||||
label: format!("Gaia {}", sid),
|
||||
ra: None, dec: None, distance: None,
|
||||
ra: None,
|
||||
dec: None,
|
||||
distance: None,
|
||||
raw: Some(serde_json::json!({"source_id": sid, "retrieval_type": "EPOCH_PHOTOMETRY"})),
|
||||
})
|
||||
}
|
||||
@ -416,13 +528,23 @@ impl ObservationFetcher for GaiaLightCurveFetcher {
|
||||
candidate: &Candidate,
|
||||
_release: Option<&str>,
|
||||
_subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
force: bool,
|
||||
) -> Result<ObservationProduct> {
|
||||
let source_id = candidate.raw.as_ref()
|
||||
let source_id = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("source_id"))
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or(&candidate.source_id);
|
||||
download_gaia_product(state, source_id, GaiaRetrievalType::EpochPhotometry, ProductType::LightCurve, force).await
|
||||
download_gaia_product(
|
||||
state,
|
||||
source_id,
|
||||
GaiaRetrievalType::EpochPhotometry,
|
||||
ProductType::LightCurve,
|
||||
force,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@ -442,14 +564,26 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_parse_band_from_name_epoch() {
|
||||
assert_eq!(parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_G.fits").as_deref(), Some("G"));
|
||||
assert_eq!(parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_BP.fits").as_deref(), Some("BP"));
|
||||
assert_eq!(parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_RP.fits").as_deref(), Some("RP"));
|
||||
assert_eq!(
|
||||
parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_G.fits").as_deref(),
|
||||
Some("G")
|
||||
);
|
||||
assert_eq!(
|
||||
parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_BP.fits").as_deref(),
|
||||
Some("BP")
|
||||
);
|
||||
assert_eq!(
|
||||
parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_RP.fits").as_deref(),
|
||||
Some("RP")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_band_from_name_xp() {
|
||||
assert_eq!(parse_band_from_name("65214031805717376_XP_CONTINUOUS.fits"), None);
|
||||
assert_eq!(
|
||||
parse_band_from_name("65214031805717376_XP_CONTINUOUS.fits"),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@ -474,17 +608,26 @@ mod tests {
|
||||
{
|
||||
let mut writer = zip::ZipWriter::new(std::io::Cursor::new(&mut buf));
|
||||
let opts = zip::write::SimpleFileOptions::default();
|
||||
writer.start_file("123_EPOCH_PHOTOMETRY_G.fits", opts).unwrap();
|
||||
writer
|
||||
.start_file("123_EPOCH_PHOTOMETRY_G.fits", opts)
|
||||
.unwrap();
|
||||
writer.write_all(b"G_DATA").unwrap();
|
||||
writer.start_file("123_EPOCH_PHOTOMETRY_BP.fits", opts).unwrap();
|
||||
writer
|
||||
.start_file("123_EPOCH_PHOTOMETRY_BP.fits", opts)
|
||||
.unwrap();
|
||||
writer.write_all(b"BP_DATA").unwrap();
|
||||
writer.start_file("123_EPOCH_PHOTOMETRY_RP.fits", opts).unwrap();
|
||||
writer
|
||||
.start_file("123_EPOCH_PHOTOMETRY_RP.fits", opts)
|
||||
.unwrap();
|
||||
writer.write_all(b"RP_DATA").unwrap();
|
||||
writer.finish().unwrap();
|
||||
}
|
||||
let files = extract_all_zip_entries(&buf, "123").unwrap();
|
||||
assert_eq!(files.len(), 3, "EPOCH_PHOTOMETRY 应返回全部 3 个波段文件");
|
||||
let bands: Vec<_> = files.iter().map(|f| parse_band_from_name(&f.name)).collect();
|
||||
let bands: Vec<_> = files
|
||||
.iter()
|
||||
.map(|f| parse_band_from_name(&f.name))
|
||||
.collect();
|
||||
assert!(bands.contains(&Some("G".into())));
|
||||
assert!(bands.contains(&Some("BP".into())));
|
||||
assert!(bands.contains(&Some("RP".into())));
|
||||
|
||||
@ -23,55 +23,116 @@ use tracing::{info, warn};
|
||||
|
||||
// ── 版本与分辨率枚举（领域知识） ──
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum LamostRelease {
|
||||
Dr5, Dr6, Dr7, Dr8, Dr9, Dr10, Dr11,
|
||||
}
|
||||
|
||||
impl Default for LamostRelease {
|
||||
fn default() -> Self { Self::Dr10 }
|
||||
Dr5,
|
||||
Dr6,
|
||||
Dr7,
|
||||
Dr8,
|
||||
Dr9,
|
||||
#[default]
|
||||
Dr10,
|
||||
Dr11,
|
||||
Dr12,
|
||||
Dr13,
|
||||
Dr14,
|
||||
}
|
||||
|
||||
impl LamostRelease {
|
||||
pub fn valid_values() -> &'static [&'static str] {
|
||||
&["dr5", "dr6", "dr7", "dr8", "dr9", "dr10", "dr11"]
|
||||
&[
|
||||
"dr5", "dr6", "dr7", "dr8", "dr9", "dr10", "dr11", "dr12", "dr13", "dr14",
|
||||
]
|
||||
}
|
||||
pub fn path_segment(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Dr5 => "dr5", Self::Dr6 => "dr6", Self::Dr7 => "dr7",
|
||||
Self::Dr8 => "dr8", Self::Dr9 => "dr9", Self::Dr10 => "dr10", Self::Dr11 => "dr11",
|
||||
Self::Dr5 => "dr5",
|
||||
Self::Dr6 => "dr6",
|
||||
Self::Dr7 => "dr7",
|
||||
Self::Dr8 => "dr8",
|
||||
Self::Dr9 => "dr9",
|
||||
Self::Dr10 => "dr10",
|
||||
Self::Dr11 => "dr11",
|
||||
Self::Dr12 => "dr12",
|
||||
Self::Dr13 => "dr13",
|
||||
Self::Dr14 => "dr14",
|
||||
}
|
||||
}
|
||||
pub fn version_segment(&self) -> &'static str {
|
||||
/// 该 DR 支持的全部子版本（URL 路径段），按时间从早到晚排序
|
||||
///
|
||||
/// 数据来源：LAMOST 官网 lmusers 页面的发布版本表格 + 端点实测。
|
||||
/// v0 通常是早期/内部预览版（数据量少或为空），v2.0 为最终公开版。
|
||||
pub fn versions(&self) -> &'static [&'static str] {
|
||||
match self {
|
||||
Self::Dr5 | Self::Dr6 => &["v0", "v2"],
|
||||
Self::Dr7 => &["v0", "v1.2"],
|
||||
Self::Dr8 | Self::Dr9 => &["v0", "v1.0", "v2.0"],
|
||||
Self::Dr10 => &["v0", "v1.0", "v2.0"],
|
||||
Self::Dr11 | Self::Dr12 | Self::Dr13 | Self::Dr14 => &["v0", "v1.0", "v1.1", "v2.0"],
|
||||
}
|
||||
}
|
||||
/// 该 DR 的默认（最新公开）子版本
|
||||
pub fn default_version(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Dr5 | Self::Dr6 => "v2",
|
||||
Self::Dr7 => "v1.2",
|
||||
Self::Dr8 | Self::Dr9 | Self::Dr10 | Self::Dr11 => "v2.0",
|
||||
// v2.0 为各 DR 的最终/最新公开版
|
||||
Self::Dr8
|
||||
| Self::Dr9
|
||||
| Self::Dr10
|
||||
| Self::Dr11
|
||||
| Self::Dr12
|
||||
| Self::Dr13
|
||||
| Self::Dr14 => "v2.0",
|
||||
}
|
||||
}
|
||||
pub fn display(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Dr5 => "DR5", Self::Dr6 => "DR6", Self::Dr7 => "DR7",
|
||||
Self::Dr8 => "DR8", Self::Dr9 => "DR9", Self::Dr10 => "DR10", Self::Dr11 => "DR11",
|
||||
Self::Dr5 => "DR5",
|
||||
Self::Dr6 => "DR6",
|
||||
Self::Dr7 => "DR7",
|
||||
Self::Dr8 => "DR8",
|
||||
Self::Dr9 => "DR9",
|
||||
Self::Dr10 => "DR10",
|
||||
Self::Dr11 => "DR11",
|
||||
Self::Dr12 => "DR12",
|
||||
Self::Dr13 => "DR13",
|
||||
Self::Dr14 => "DR14",
|
||||
}
|
||||
}
|
||||
pub fn supports_mrs(&self) -> bool {
|
||||
!matches!(self, Self::Dr5 | Self::Dr6)
|
||||
}
|
||||
/// 是否为公开发布（无需登录即可访问 cone search / FITS 下载）。
|
||||
///
|
||||
/// LAMOST 的 Internal 数据发布需经 oauth.china-vo.org 完成统一身份认证后才能访问。
|
||||
/// DR12-14 当前为 Internal，公开 cone search 端点会被 302 重定向到登录页。
|
||||
pub fn is_public(&self) -> bool {
|
||||
!matches!(self, Self::Dr12 | Self::Dr13 | Self::Dr14)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum LamostResolution { Lrs, Mrs }
|
||||
pub enum LamostResolution {
|
||||
Lrs,
|
||||
Mrs,
|
||||
}
|
||||
|
||||
impl LamostResolution {
|
||||
pub fn valid_values() -> &'static [&'static str] {
|
||||
&["lrs", "mrs"]
|
||||
}
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self { Self::Lrs => "lrs", Self::Mrs => "mrs" }
|
||||
match self {
|
||||
Self::Lrs => "lrs",
|
||||
Self::Mrs => "mrs",
|
||||
}
|
||||
}
|
||||
pub fn display(&self) -> &'static str {
|
||||
match self { Self::Lrs => "LRS", Self::Mrs => "MRS" }
|
||||
match self {
|
||||
Self::Lrs => "LRS",
|
||||
Self::Mrs => "MRS",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -85,16 +146,56 @@ pub fn parse_lamost_release(s: Option<&str>) -> Result<LamostRelease> {
|
||||
Some("dr9") => LamostRelease::Dr9,
|
||||
Some("dr10") => LamostRelease::Dr10,
|
||||
Some("dr11") => LamostRelease::Dr11,
|
||||
Some(other) => return Err(anyhow!("不支持的 LAMOST release '{}'，可选: {:?}", other, LamostRelease::valid_values())),
|
||||
Some("dr12") => LamostRelease::Dr12,
|
||||
Some("dr13") => LamostRelease::Dr13,
|
||||
Some("dr14") => LamostRelease::Dr14,
|
||||
Some(other) => {
|
||||
return Err(anyhow!(
|
||||
"不支持的 LAMOST release '{}'，可选: {:?}",
|
||||
other,
|
||||
LamostRelease::valid_values()
|
||||
))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// 解析 LAMOST 子版本参数。None 时返回该 DR 的默认（最新公开）子版本。
|
||||
pub fn parse_lamost_version(release: LamostRelease, version: Option<&str>) -> Result<String> {
|
||||
let valid = release.versions();
|
||||
match version.map(|x| x.trim().to_lowercase()) {
|
||||
None => Ok(release.default_version().to_string()),
|
||||
Some(v) => {
|
||||
if valid.iter().any(|x| x.eq_ignore_ascii_case(&v)) {
|
||||
// 规范化为官网写法（保留原大小写约定，如 v1.0/v2.0）
|
||||
Ok(valid
|
||||
.iter()
|
||||
.find(|x| x.eq_ignore_ascii_case(&v))
|
||||
.unwrap()
|
||||
.to_string())
|
||||
} else {
|
||||
Err(anyhow!(
|
||||
"不支持的 LAMOST 子版本 '{}'（{:?} 的可选版本: {:?}）",
|
||||
v,
|
||||
release,
|
||||
valid
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_lamost_resolution(subtype: Option<&str>) -> Result<LamostResolution> {
|
||||
Ok(match subtype.map(|x| x.to_lowercase()).as_deref() {
|
||||
None => LamostResolution::Lrs,
|
||||
Some("lrs") | Some("low") => LamostResolution::Lrs,
|
||||
Some("mrs") | Some("medium") => LamostResolution::Mrs,
|
||||
Some(other) => return Err(anyhow!("不支持的 LAMOST subtype '{}'，可选: {:?}", other, LamostResolution::valid_values())),
|
||||
Some(other) => {
|
||||
return Err(anyhow!(
|
||||
"不支持的 LAMOST subtype '{}'，可选: {:?}",
|
||||
other,
|
||||
LamostResolution::valid_values()
|
||||
))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@ -143,6 +244,49 @@ impl ObservationFetcher for LamostSpectrumFetcher {
|
||||
Some("dr10")
|
||||
}
|
||||
|
||||
fn versions_for_release(&self, release: Option<&str>) -> Vec<String> {
|
||||
match parse_lamost_release(release) {
|
||||
Ok(rel) => rel.versions().iter().map(|s| s.to_string()).collect(),
|
||||
Err(_) => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn default_version_for_release(&self, release: Option<&str>) -> Option<String> {
|
||||
parse_lamost_release(release)
|
||||
.ok()
|
||||
.map(|rel| rel.default_version().to_string())
|
||||
}
|
||||
|
||||
/// Internal 数据发布（DR12-14）需经 oauth.china-vo.org 登录，公开 cone search 不可用
|
||||
fn releases_requiring_auth(&self) -> Vec<String> {
|
||||
LamostRelease::valid_values()
|
||||
.iter()
|
||||
.filter_map(|s| {
|
||||
parse_lamost_release(Some(s))
|
||||
.ok()
|
||||
.filter(|rel| !rel.is_public())
|
||||
.map(|_| s.to_string())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 表达 release×subtype 交叉约束：DR5/DR6 仅有 LRS（无 MRS），DR7+ 有 LRS+MRS。
|
||||
/// 前端据此在选 DR5/DR6 时隐藏/禁用 MRS 子类型选项。
|
||||
fn subtypes_per_release(&self) -> std::collections::HashMap<String, Vec<String>> {
|
||||
LamostRelease::valid_values()
|
||||
.iter()
|
||||
.map(|s| {
|
||||
let rel = parse_lamost_release(Some(s)).unwrap();
|
||||
let subs: Vec<String> = if rel.supports_mrs() {
|
||||
vec!["lrs".to_string(), "mrs".to_string()]
|
||||
} else {
|
||||
vec!["lrs".to_string()]
|
||||
};
|
||||
(s.to_string(), subs)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn suggested_max_radius_deg(&self) -> f64 {
|
||||
// 与 validate_cone_params 的建议范围一致（≤5° 仅警告不拒绝）
|
||||
5.0
|
||||
@ -155,31 +299,70 @@ impl ObservationFetcher for LamostSpectrumFetcher {
|
||||
async fn cone_search_raw(
|
||||
&self,
|
||||
state: &AppState,
|
||||
ra: f64, dec: f64, radius_deg: f64,
|
||||
release: Option<&str>, subtype: Option<&str>,
|
||||
ra: f64,
|
||||
dec: f64,
|
||||
radius_deg: f64,
|
||||
release: Option<&str>,
|
||||
subtype: Option<&str>,
|
||||
version: Option<&str>,
|
||||
) -> Result<Vec<Candidate>> {
|
||||
validate_cone_params(ra, dec, radius_deg)?;
|
||||
let rel = parse_lamost_release(release)?;
|
||||
let res = parse_lamost_resolution(subtype)?;
|
||||
if res == LamostResolution::Mrs && !rel.supports_mrs() {
|
||||
return Err(anyhow!("MRS（中分辨率）从 DR7 起才支持，{:?} 无 MRS 数据", rel));
|
||||
let ver = parse_lamost_version(rel, version)?;
|
||||
if !rel.is_public() {
|
||||
return Err(anyhow!(
|
||||
"{:?} 当前为 Internal 数据发布，公开 Cone Search 不可用（需经 oauth.china-vo.org 登录认证）",
|
||||
rel
|
||||
));
|
||||
}
|
||||
let result = state.lamost.cone_search(ra, dec, radius_deg, rel, res).await?;
|
||||
Ok(result.rows.iter().map(|r| row_to_candidate(r, res.as_str())).collect())
|
||||
if res == LamostResolution::Mrs && !rel.supports_mrs() {
|
||||
return Err(anyhow!(
|
||||
"MRS（中分辨率）从 DR7 起才支持，{:?} 无 MRS 数据",
|
||||
rel
|
||||
));
|
||||
}
|
||||
let result = state
|
||||
.lamost
|
||||
.cone_search(ra, dec, radius_deg, rel, &ver, res)
|
||||
.await?;
|
||||
Ok(result
|
||||
.rows
|
||||
.iter()
|
||||
.map(|r| row_to_candidate(r, res.as_str()))
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn resolve_identifier(
|
||||
&self,
|
||||
identifier: &str,
|
||||
_release: Option<&str>,
|
||||
release: Option<&str>,
|
||||
subtype: Option<&str>,
|
||||
version: Option<&str>,
|
||||
) -> Result<Candidate> {
|
||||
let obsid: i64 = identifier.trim().parse()
|
||||
let obsid: i64 = identifier
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| anyhow!("LAMOST source_id 应为纯数字 obsid，得到 '{}'", identifier))?;
|
||||
if obsid <= 0 {
|
||||
return Err(anyhow!("obsid 应为正整数，当前: {}", obsid));
|
||||
}
|
||||
let res = parse_lamost_resolution(subtype)?.as_str().to_string();
|
||||
let rel = parse_lamost_release(release)?;
|
||||
let res = parse_lamost_resolution(subtype)?;
|
||||
let ver = parse_lamost_version(rel, version)?;
|
||||
if !rel.is_public() {
|
||||
return Err(anyhow!(
|
||||
"{:?} 当前为 Internal 数据发布，公开访问不可用（需经 oauth.china-vo.org 登录认证）",
|
||||
rel
|
||||
));
|
||||
}
|
||||
// 与 cone_search_raw 一致的交叉约束：MRS 仅从 DR7 起支持
|
||||
if res == LamostResolution::Mrs && !rel.supports_mrs() {
|
||||
return Err(anyhow!(
|
||||
"MRS（中分辨率）从 DR7 起才支持，{:?} 无 MRS 数据",
|
||||
rel
|
||||
));
|
||||
}
|
||||
Ok(Candidate {
|
||||
source: Source::Lamost,
|
||||
source_id: obsid.to_string(),
|
||||
@ -187,7 +370,9 @@ impl ObservationFetcher for LamostSpectrumFetcher {
|
||||
ra: None,
|
||||
dec: None,
|
||||
distance: None,
|
||||
raw: Some(serde_json::json!({"obsid": obsid, "resolution": res})),
|
||||
raw: Some(
|
||||
serde_json::json!({"obsid": obsid, "resolution": res.as_str(), "version": ver}),
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
@ -197,68 +382,116 @@ impl ObservationFetcher for LamostSpectrumFetcher {
|
||||
candidate: &Candidate,
|
||||
release: Option<&str>,
|
||||
subtype: Option<&str>,
|
||||
version: Option<&str>,
|
||||
force: bool,
|
||||
) -> Result<ObservationProduct> {
|
||||
let obsid = candidate.raw.as_ref()
|
||||
let obsid = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("obsid"))
|
||||
.and_then(|v| v.as_i64())
|
||||
.ok_or_else(|| anyhow!("LAMOST Candidate 缺 obsid 字段"))?;
|
||||
let rel = parse_lamost_release(release)?;
|
||||
let res = parse_lamost_resolution(subtype)?;
|
||||
let ver = parse_lamost_version(rel, version)?;
|
||||
if !rel.is_public() {
|
||||
return Err(anyhow!(
|
||||
"{:?} 当前为 Internal 数据发布，公开访问不可用（需经 oauth.china-vo.org 登录认证）",
|
||||
rel
|
||||
));
|
||||
}
|
||||
// 防御性交叉约束（resolve_identifier 已查过；直接调 fetch 时再兜一次）
|
||||
if res == LamostResolution::Mrs && !rel.supports_mrs() {
|
||||
return Err(anyhow!(
|
||||
"MRS（中分辨率）从 DR7 起才支持，{:?} 无 MRS 数据",
|
||||
rel
|
||||
));
|
||||
}
|
||||
let product = ProductSpec::with_subtype(ProductType::Spectrum, res.as_str());
|
||||
let dr = rel.path_segment();
|
||||
let ver = rel.version_segment();
|
||||
let res_str = res.as_str();
|
||||
let cache_key = format!("{}|{}|{}|{}", dr, ver, res_str, obsid);
|
||||
let source_label = format!("obsid {}", obsid);
|
||||
|
||||
// 1) 缓存命中
|
||||
if !force {
|
||||
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Lamost, &product, &cache_key).await? {
|
||||
if let Some(_) = cached_files_total_size(&state.config.library_dir, &artifacts) {
|
||||
if let Some((artifacts, meta)) =
|
||||
fetch_observation_cache(&state.db, Source::Lamost, &product, &cache_key).await?
|
||||
{
|
||||
if cached_files_total_size(&state.config.library_dir, &artifacts)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
info!("[LAMOST] 光谱缓存命中 (obsid={})", obsid);
|
||||
return Ok(ObservationProduct {
|
||||
source: Source::Lamost, product,
|
||||
source_id: cache_key, source_label, artifacts, source_meta: meta,
|
||||
source: Source::Lamost,
|
||||
product,
|
||||
source_id: cache_key,
|
||||
source_label,
|
||||
artifacts,
|
||||
source_meta: meta,
|
||||
});
|
||||
}
|
||||
warn!("[LAMOST] 缓存记录存在但文件缺失，重新下载 (obsid={})", obsid);
|
||||
warn!(
|
||||
"[LAMOST] 缓存记录存在但文件缺失，重新下载 (obsid={})",
|
||||
obsid
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 2) 下载 + 解压
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let gz_bytes = state.lamost.download_fits(obsid, rel).await?;
|
||||
let gz_bytes = state.lamost.download_fits(obsid, rel, &ver).await?;
|
||||
let fits_bytes = decompress_if_gzip(&gz_bytes)?;
|
||||
|
||||
// 3) 落盘
|
||||
let rel_path = format!(
|
||||
"Telescope/lamost/spectrum/{res}/{dr}/{ver}/{obsid}.fits",
|
||||
res = res_str, dr = dr, ver = ver, obsid = obsid
|
||||
res = res_str,
|
||||
dr = dr,
|
||||
ver = ver,
|
||||
obsid = obsid
|
||||
);
|
||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes).await?;
|
||||
info!(
|
||||
"[LAMOST] FITS 已保存 obsid={} → {} ({}B)",
|
||||
obsid, rel_path, size
|
||||
);
|
||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes)?;
|
||||
info!("[LAMOST] FITS 已保存 obsid={} → {} ({}B)", obsid, rel_path, size);
|
||||
|
||||
let meta = serde_json::json!({"obsid": obsid, "dr": dr, "version": ver, "resolution": res_str});
|
||||
let meta =
|
||||
serde_json::json!({"obsid": obsid, "dr": dr, "version": ver, "resolution": res_str});
|
||||
let artifact = Artifact {
|
||||
band: None, original_name: None,
|
||||
band: None,
|
||||
original_name: None,
|
||||
file_path: rel_path.clone(),
|
||||
file_url: file_url_from_path(&rel_path),
|
||||
file_format: "fits".to_string(),
|
||||
size_bytes: size, cached: false,
|
||||
size_bytes: size,
|
||||
cached: false,
|
||||
};
|
||||
let meta_str = meta.to_string();
|
||||
if let Err(e) = write_observation_cache(
|
||||
&state.db, Source::Lamost, &product, &cache_key, None, None,
|
||||
std::slice::from_ref(&artifact), Some(&meta_str),
|
||||
).await {
|
||||
&state.db,
|
||||
Source::Lamost,
|
||||
&product,
|
||||
&cache_key,
|
||||
None,
|
||||
None,
|
||||
std::slice::from_ref(&artifact),
|
||||
Some(&meta_str),
|
||||
)
|
||||
.await
|
||||
{
|
||||
log_write_failure(Source::Lamost.as_str(), "spectrum", e);
|
||||
}
|
||||
|
||||
Ok(ObservationProduct {
|
||||
source: Source::Lamost, product,
|
||||
source_id: cache_key, source_label,
|
||||
artifacts: vec![artifact], source_meta: Some(meta),
|
||||
source: Source::Lamost,
|
||||
product,
|
||||
source_id: cache_key,
|
||||
source_label,
|
||||
artifacts: vec![artifact],
|
||||
source_meta: Some(meta),
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -280,7 +513,9 @@ fn decompress_if_gzip(bytes: &[u8]) -> Result<Vec<u8>> {
|
||||
if bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b {
|
||||
let mut decoder = flate2::read::GzDecoder::new(bytes);
|
||||
let mut out = Vec::new();
|
||||
decoder.read_to_end(&mut out).map_err(|e| anyhow!("gzip 解压失败: {}", e))?;
|
||||
decoder
|
||||
.read_to_end(&mut out)
|
||||
.map_err(|e| anyhow!("gzip 解压失败: {}", e))?;
|
||||
Ok(out)
|
||||
} else {
|
||||
Ok(bytes.to_vec())
|
||||
@ -323,9 +558,18 @@ mod tests {
|
||||
#[test]
|
||||
fn test_row_to_candidate() {
|
||||
let row = LamostSpectrumRow {
|
||||
obsid: 438809089, designation: None, ra_obs: Some(10.5), dec_obs: Some(41.0),
|
||||
z: None, class: None, subclass: None,
|
||||
sn_u: None, sn_g: None, sn_r: None, sn_i: None, sn_z: None,
|
||||
obsid: 438809089,
|
||||
designation: None,
|
||||
ra_obs: Some(10.5),
|
||||
dec_obs: Some(41.0),
|
||||
z: None,
|
||||
class: None,
|
||||
subclass: None,
|
||||
sn_u: None,
|
||||
sn_g: None,
|
||||
sn_r: None,
|
||||
sn_i: None,
|
||||
sn_z: None,
|
||||
};
|
||||
let c = row_to_candidate(&row, "lrs");
|
||||
assert_eq!(c.source_id, "438809089");
|
||||
|
||||
@ -64,32 +64,38 @@ impl ObservationRegistry {
|
||||
product: ProductType,
|
||||
subtype: Option<&str>,
|
||||
) -> Result<Arc<dyn ObservationFetcher>> {
|
||||
let candidates = self
|
||||
.fetchers
|
||||
.get(&(source, product))
|
||||
.ok_or_else(|| anyhow!(
|
||||
let candidates = self.fetchers.get(&(source, product)).ok_or_else(|| {
|
||||
anyhow!(
|
||||
"{:?} 暂不支持 {:?} 产品。支持的组合：{}",
|
||||
source,
|
||||
product,
|
||||
self.list_supported_combinations()
|
||||
))?;
|
||||
)
|
||||
})?;
|
||||
|
||||
// subtype 精确匹配
|
||||
if let Some(st) = subtype {
|
||||
let st_lower = st.to_lowercase();
|
||||
for f in candidates {
|
||||
if f.subtypes().iter().any(|s| s.eq_ignore_ascii_case(&st_lower)) {
|
||||
if f.subtypes()
|
||||
.iter()
|
||||
.any(|s| s.eq_ignore_ascii_case(&st_lower))
|
||||
{
|
||||
return Ok(f.clone());
|
||||
}
|
||||
}
|
||||
// 未找到匹配的 subtype — 报错
|
||||
let valid: Vec<&str> = candidates.iter()
|
||||
let valid: Vec<&str> = candidates
|
||||
.iter()
|
||||
.flat_map(|f| f.subtypes())
|
||||
.copied()
|
||||
.collect();
|
||||
return Err(anyhow!(
|
||||
"{:?} 的 {:?} 不支持 subtype '{}'，可选: {:?}",
|
||||
source, product, st, valid
|
||||
source,
|
||||
product,
|
||||
st,
|
||||
valid
|
||||
));
|
||||
}
|
||||
|
||||
@ -134,6 +140,16 @@ impl ObservationRegistry {
|
||||
subtypes: f.subtypes().iter().map(|s| s.to_string()).collect(),
|
||||
releases: f.releases().iter().map(|s| s.to_string()).collect(),
|
||||
default_release: f.default_release().map(|s| s.to_string()),
|
||||
// 每个 release 对应的子版本列表（仅 LAMOST 有内容）
|
||||
release_versions: f
|
||||
.releases()
|
||||
.iter()
|
||||
.map(|r| (r.to_string(), f.versions_for_release(Some(r))))
|
||||
.collect(),
|
||||
// 需要登录认证的 release（LAMOST Internal DR12-14）
|
||||
releases_requiring_auth: f.releases_requiring_auth(),
|
||||
// 每个 release 实际支持的 subtype（表达交叉约束，如 LAMOST DR5/6 无 MRS）
|
||||
subtypes_per_release: f.subtypes_per_release().into_iter().collect(),
|
||||
suggested_max_radius_deg: f.suggested_max_radius_deg(),
|
||||
hard_max_radius_deg: f.hard_max_radius_deg(),
|
||||
identifier_format: f.identifier_format().map(|s| s.to_string()),
|
||||
@ -163,6 +179,15 @@ pub struct CapabilitySpec {
|
||||
pub releases: Vec<String>,
|
||||
/// 默认版本（release 参数为空时使用），None 表示无版本概念
|
||||
pub default_release: Option<String>,
|
||||
/// 每个 release 对应的子版本列表（仅 LAMOST 有内容；key=release 如 "dr11"，value=["v0","v1.0",...]）
|
||||
#[serde(default)]
|
||||
pub release_versions: std::collections::BTreeMap<String, Vec<String>>,
|
||||
/// 需要登录认证才能访问的 release 列表（如 LAMOST Internal DR12-14），空表示全部公开
|
||||
#[serde(default)]
|
||||
pub releases_requiring_auth: Vec<String>,
|
||||
/// 每个 release 实际支持的 subtype（表达交叉约束，如 LAMOST DR5/6 无 MRS）；空表示所有 release 都支持全部 subtypes
|
||||
#[serde(default)]
|
||||
pub subtypes_per_release: std::collections::BTreeMap<String, Vec<String>>,
|
||||
/// 坐标模式检索半径的建议上限（度），超出仅警告不拒绝
|
||||
pub suggested_max_radius_deg: f64,
|
||||
/// 坐标模式检索半径的硬性上限（度），超出会返回错误
|
||||
@ -186,8 +211,12 @@ mod tests {
|
||||
assert!(reg.get(Source::Gaia, ProductType::LightCurve, None).is_ok());
|
||||
// SDSS spectrum (specobj 默认 + apogee/aspcap subtype)
|
||||
assert!(reg.get(Source::Sdss, ProductType::Spectrum, None).is_ok());
|
||||
assert!(reg.get(Source::Sdss, ProductType::Spectrum, Some("apstar")).is_ok());
|
||||
assert!(reg.get(Source::Sdss, ProductType::Spectrum, Some("aspcap")).is_ok());
|
||||
assert!(reg
|
||||
.get(Source::Sdss, ProductType::Spectrum, Some("apstar"))
|
||||
.is_ok());
|
||||
assert!(reg
|
||||
.get(Source::Sdss, ProductType::Spectrum, Some("aspcap"))
|
||||
.is_ok());
|
||||
// DESI spectrum
|
||||
assert!(reg.get(Source::Desi, ProductType::Spectrum, None).is_ok());
|
||||
}
|
||||
@ -195,7 +224,9 @@ mod tests {
|
||||
#[test]
|
||||
fn test_unsupported_combination_error() {
|
||||
let reg = ObservationRegistry::default();
|
||||
let err = reg.get(Source::Lamost, ProductType::LightCurve, None).unwrap_err();
|
||||
let err = reg
|
||||
.get(Source::Lamost, ProductType::LightCurve, None)
|
||||
.unwrap_err();
|
||||
assert!(format!("{}", err).contains("暂不支持"));
|
||||
}
|
||||
|
||||
@ -203,7 +234,9 @@ mod tests {
|
||||
fn test_sdss_subtype_routing() {
|
||||
let reg = ObservationRegistry::default();
|
||||
// subtype=apstar 应路由到 SdssApogeeFetcher（其 subtypes 含 "apstar"）
|
||||
let apstar = reg.get(Source::Sdss, ProductType::Spectrum, Some("apstar")).unwrap();
|
||||
let apstar = reg
|
||||
.get(Source::Sdss, ProductType::Spectrum, Some("apstar"))
|
||||
.unwrap();
|
||||
assert!(apstar.subtypes().contains(&"apstar"));
|
||||
// 默认（无 subtype）应路由到 SdssSpectrumFetcher
|
||||
let specobj = reg.get(Source::Sdss, ProductType::Spectrum, None).unwrap();
|
||||
|
||||
@ -23,11 +23,13 @@ use tracing::{info, warn};
|
||||
|
||||
// ── 版本枚举 ──
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum SdssRelease { Dr16, Dr17, Dr18, Dr19 }
|
||||
|
||||
impl Default for SdssRelease {
|
||||
fn default() -> Self { Self::Dr17 }
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum SdssRelease {
|
||||
Dr16,
|
||||
#[default]
|
||||
Dr17,
|
||||
Dr18,
|
||||
Dr19,
|
||||
}
|
||||
|
||||
impl SdssRelease {
|
||||
@ -36,8 +38,10 @@ impl SdssRelease {
|
||||
}
|
||||
pub fn sas_dr_segment(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Dr16 => "dr16", Self::Dr17 => "dr17",
|
||||
Self::Dr18 => "dr18", Self::Dr19 => "dr19",
|
||||
Self::Dr16 => "dr16",
|
||||
Self::Dr17 => "dr17",
|
||||
Self::Dr18 => "dr18",
|
||||
Self::Dr19 => "dr19",
|
||||
}
|
||||
}
|
||||
pub fn datalab_table(&self) -> Option<&'static str> {
|
||||
@ -55,8 +59,10 @@ impl SdssRelease {
|
||||
}
|
||||
pub fn display(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Dr16 => "DR16", Self::Dr17 => "DR17",
|
||||
Self::Dr18 => "DR18", Self::Dr19 => "DR19",
|
||||
Self::Dr16 => "DR16",
|
||||
Self::Dr17 => "DR17",
|
||||
Self::Dr18 => "DR18",
|
||||
Self::Dr19 => "DR19",
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -68,7 +74,13 @@ pub fn parse_sdss_release(s: Option<&str>) -> Result<SdssRelease> {
|
||||
Some("dr17") => SdssRelease::Dr17,
|
||||
Some("dr18") => SdssRelease::Dr18,
|
||||
Some("dr19") => SdssRelease::Dr19,
|
||||
Some(other) => return Err(anyhow!("不支持的 SDSS release '{}'，可选: {:?}", other, SdssRelease::valid_values())),
|
||||
Some(other) => {
|
||||
return Err(anyhow!(
|
||||
"不支持的 SDSS release '{}'，可选: {:?}",
|
||||
other,
|
||||
SdssRelease::valid_values()
|
||||
))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@ -131,8 +143,12 @@ impl ObservationFetcher for SdssSpectrumFetcher {
|
||||
async fn cone_search_raw(
|
||||
&self,
|
||||
state: &AppState,
|
||||
ra: f64, dec: f64, radius_deg: f64,
|
||||
release: Option<&str>, _subtype: Option<&str>,
|
||||
ra: f64,
|
||||
dec: f64,
|
||||
radius_deg: f64,
|
||||
release: Option<&str>,
|
||||
_subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
) -> Result<Vec<Candidate>> {
|
||||
validate_cone_params(ra, dec, radius_deg)?;
|
||||
let rel = parse_sdss_release(release)?;
|
||||
@ -145,6 +161,7 @@ impl ObservationFetcher for SdssSpectrumFetcher {
|
||||
identifier: &str,
|
||||
_release: Option<&str>,
|
||||
_subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
) -> Result<Candidate> {
|
||||
let parts: Vec<&str> = identifier.split('-').collect();
|
||||
if parts.len() != 4 {
|
||||
@ -153,14 +170,22 @@ impl ObservationFetcher for SdssSpectrumFetcher {
|
||||
identifier
|
||||
));
|
||||
}
|
||||
let plate: i64 = parts[1].parse().map_err(|_| anyhow!("plate 非数字: '{}'", parts[1]))?;
|
||||
let mjd: i64 = parts[2].parse().map_err(|_| anyhow!("mjd 非数字: '{}'", parts[2]))?;
|
||||
let fiberid: i64 = parts[3].parse().map_err(|_| anyhow!("fiberid 非数字: '{}'", parts[3]))?;
|
||||
let plate: i64 = parts[1]
|
||||
.parse()
|
||||
.map_err(|_| anyhow!("plate 非数字: '{}'", parts[1]))?;
|
||||
let mjd: i64 = parts[2]
|
||||
.parse()
|
||||
.map_err(|_| anyhow!("mjd 非数字: '{}'", parts[2]))?;
|
||||
let fiberid: i64 = parts[3]
|
||||
.parse()
|
||||
.map_err(|_| anyhow!("fiberid 非数字: '{}'", parts[3]))?;
|
||||
Ok(Candidate {
|
||||
source: Source::Sdss,
|
||||
source_id: identifier.to_string(),
|
||||
label: format!("plate {} mjd {} fiber {}", plate, mjd, fiberid),
|
||||
ra: None, dec: None, distance: None,
|
||||
ra: None,
|
||||
dec: None,
|
||||
distance: None,
|
||||
raw: Some(serde_json::json!({
|
||||
"plate": plate, "mjd": mjd, "fiberid": fiberid, "run2d": parts[0]
|
||||
})),
|
||||
@ -173,15 +198,32 @@ impl ObservationFetcher for SdssSpectrumFetcher {
|
||||
candidate: &Candidate,
|
||||
release: Option<&str>,
|
||||
_subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
force: bool,
|
||||
) -> Result<ObservationProduct> {
|
||||
let plate = candidate.raw.as_ref().and_then(|v| v.get("plate")).and_then(|v| v.as_i64())
|
||||
let plate = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("plate"))
|
||||
.and_then(|v| v.as_i64())
|
||||
.ok_or_else(|| anyhow!("SDSS Candidate 缺 plate"))?;
|
||||
let mjd = candidate.raw.as_ref().and_then(|v| v.get("mjd")).and_then(|v| v.as_i64())
|
||||
let mjd = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("mjd"))
|
||||
.and_then(|v| v.as_i64())
|
||||
.ok_or_else(|| anyhow!("SDSS Candidate 缺 mjd"))?;
|
||||
let fiberid = candidate.raw.as_ref().and_then(|v| v.get("fiberid")).and_then(|v| v.as_i64())
|
||||
let fiberid = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("fiberid"))
|
||||
.and_then(|v| v.as_i64())
|
||||
.ok_or_else(|| anyhow!("SDSS Candidate 缺 fiberid"))?;
|
||||
let run2d = candidate.raw.as_ref().and_then(|v| v.get("run2d")).and_then(|v| v.as_str())
|
||||
let run2d = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("run2d"))
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow!("SDSS Candidate 缺 run2d"))?;
|
||||
let rel = parse_sdss_release(release)?;
|
||||
let product = ProductSpec::with_subtype(ProductType::Spectrum, "spec");
|
||||
@ -191,12 +233,21 @@ impl ObservationFetcher for SdssSpectrumFetcher {
|
||||
|
||||
// 1) 缓存命中
|
||||
if !force {
|
||||
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Sdss, &product, &cache_key).await? {
|
||||
if cached_files_total_size(&state.config.library_dir, &artifacts).is_some() {
|
||||
if let Some((artifacts, meta)) =
|
||||
fetch_observation_cache(&state.db, Source::Sdss, &product, &cache_key).await?
|
||||
{
|
||||
if cached_files_total_size(&state.config.library_dir, &artifacts)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
info!("[SDSS] 光谱缓存命中 ({})", cache_key);
|
||||
return Ok(ObservationProduct {
|
||||
source: Source::Sdss, product,
|
||||
source_id: cache_key, source_label, artifacts, source_meta: meta,
|
||||
source: Source::Sdss,
|
||||
product,
|
||||
source_id: cache_key,
|
||||
source_label,
|
||||
artifacts,
|
||||
source_meta: meta,
|
||||
});
|
||||
}
|
||||
warn!("[SDSS] 缓存记录存在但文件缺失，重新下载 ({})", cache_key);
|
||||
@ -205,34 +256,57 @@ impl ObservationFetcher for SdssSpectrumFetcher {
|
||||
|
||||
// 2) SAS 下载（未压缩 FITS）+ 落盘
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let fits_bytes = state.sdss.download_fits(plate, mjd, fiberid, run2d, rel).await?;
|
||||
let fits_bytes = state
|
||||
.sdss
|
||||
.download_fits(plate, mjd, fiberid, run2d, rel)
|
||||
.await?;
|
||||
let rel_path = format!(
|
||||
"Telescope/sdss/{dr}/spec/{run2d}-{plate}-{mjd}-{fiber:04}.fits",
|
||||
dr = dr, run2d = run2d, plate = plate, mjd = mjd, fiber = fiberid
|
||||
dr = dr,
|
||||
run2d = run2d,
|
||||
plate = plate,
|
||||
mjd = mjd,
|
||||
fiber = fiberid
|
||||
);
|
||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes).await?;
|
||||
info!(
|
||||
"[SDSS] FITS 已保存 {} → {} ({}B)",
|
||||
cache_key, rel_path, size
|
||||
);
|
||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes)?;
|
||||
info!("[SDSS] FITS 已保存 {} → {} ({}B)", cache_key, rel_path, size);
|
||||
|
||||
let meta = serde_json::json!({"plate": plate, "mjd": mjd, "fiberid": fiberid, "run2d": run2d, "dr": dr});
|
||||
let artifact = Artifact {
|
||||
band: None, original_name: None,
|
||||
band: None,
|
||||
original_name: None,
|
||||
file_path: rel_path.clone(),
|
||||
file_url: file_url_from_path(&rel_path),
|
||||
file_format: "fits".to_string(),
|
||||
size_bytes: size, cached: false,
|
||||
size_bytes: size,
|
||||
cached: false,
|
||||
};
|
||||
let meta_str = meta.to_string();
|
||||
if let Err(e) = write_observation_cache(
|
||||
&state.db, Source::Sdss, &product, &cache_key, None, None,
|
||||
std::slice::from_ref(&artifact), Some(&meta_str),
|
||||
).await {
|
||||
&state.db,
|
||||
Source::Sdss,
|
||||
&product,
|
||||
&cache_key,
|
||||
None,
|
||||
None,
|
||||
std::slice::from_ref(&artifact),
|
||||
Some(&meta_str),
|
||||
)
|
||||
.await
|
||||
{
|
||||
log_write_failure(Source::Sdss.as_str(), "spectrum", e);
|
||||
}
|
||||
|
||||
Ok(ObservationProduct {
|
||||
source: Source::Sdss, product,
|
||||
source_id: cache_key, source_label,
|
||||
artifacts: vec![artifact], source_meta: Some(meta),
|
||||
source: Source::Sdss,
|
||||
product,
|
||||
source_id: cache_key,
|
||||
source_label,
|
||||
artifacts: vec![artifact],
|
||||
source_meta: Some(meta),
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -242,7 +316,9 @@ fn row_to_candidate(row: &SdssSpectrumRow) -> Candidate {
|
||||
source: Source::Sdss,
|
||||
source_id: format!("{}-{}-{}-{}", row.run2d, row.plate, row.mjd, row.fiberid),
|
||||
label: format!("plate {} mjd {} fiber {}", row.plate, row.mjd, row.fiberid),
|
||||
ra: row.ra, dec: row.dec, distance: None,
|
||||
ra: row.ra,
|
||||
dec: row.dec,
|
||||
distance: None,
|
||||
raw: Some(serde_json::json!({
|
||||
"plate": row.plate, "mjd": row.mjd, "fiberid": row.fiberid, "run2d": row.run2d
|
||||
})),
|
||||
@ -287,12 +363,19 @@ impl ObservationFetcher for SdssApogeeFetcher {
|
||||
async fn cone_search_raw(
|
||||
&self,
|
||||
state: &AppState,
|
||||
ra: f64, dec: f64, radius_deg: f64,
|
||||
release: Option<&str>, _subtype: Option<&str>,
|
||||
ra: f64,
|
||||
dec: f64,
|
||||
radius_deg: f64,
|
||||
release: Option<&str>,
|
||||
_subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
) -> Result<Vec<Candidate>> {
|
||||
validate_cone_params(ra, dec, radius_deg)?;
|
||||
let rel = parse_sdss_release(release)?;
|
||||
let result = state.sdss.apogee_cone_search(ra, dec, radius_deg, 50, rel).await?;
|
||||
let result = state
|
||||
.sdss
|
||||
.apogee_cone_search(ra, dec, radius_deg, 50, rel)
|
||||
.await?;
|
||||
Ok(result.rows.iter().map(apogee_row_to_candidate).collect())
|
||||
}
|
||||
|
||||
@ -301,6 +384,7 @@ impl ObservationFetcher for SdssApogeeFetcher {
|
||||
identifier: &str,
|
||||
_release: Option<&str>,
|
||||
subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
) -> Result<Candidate> {
|
||||
// 格式 "{telescope}|{field}|{apogee_id}"
|
||||
let parts: Vec<&str> = identifier.splitn(3, '|').collect();
|
||||
@ -315,7 +399,9 @@ impl ObservationFetcher for SdssApogeeFetcher {
|
||||
source: Source::Sdss,
|
||||
source_id: identifier.to_string(),
|
||||
label: format!("{} {} {}", parts[0], dt, parts[2]),
|
||||
ra: None, dec: None, distance: None,
|
||||
ra: None,
|
||||
dec: None,
|
||||
distance: None,
|
||||
raw: Some(serde_json::json!({
|
||||
"apogee_id": parts[2], "telescope": parts[0], "field": parts[1], "data_type": dt
|
||||
})),
|
||||
@ -328,16 +414,34 @@ impl ObservationFetcher for SdssApogeeFetcher {
|
||||
candidate: &Candidate,
|
||||
release: Option<&str>,
|
||||
subtype: Option<&str>,
|
||||
_version: Option<&str>,
|
||||
force: bool,
|
||||
) -> Result<ObservationProduct> {
|
||||
let apogee_id = candidate.raw.as_ref().and_then(|v| v.get("apogee_id")).and_then(|v| v.as_str())
|
||||
let apogee_id = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("apogee_id"))
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow!("APOGEE Candidate 缺 apogee_id"))?;
|
||||
let telescope = candidate.raw.as_ref().and_then(|v| v.get("telescope")).and_then(|v| v.as_str())
|
||||
let telescope = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("telescope"))
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow!("APOGEE Candidate 缺 telescope"))?;
|
||||
let field = candidate.raw.as_ref().and_then(|v| v.get("field")).and_then(|v| v.as_str())
|
||||
let field = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("field"))
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow!("APOGEE Candidate 缺 field"))?;
|
||||
let dt = candidate.raw.as_ref().and_then(|v| v.get("data_type")).and_then(|v| v.as_str())
|
||||
.or(subtype).unwrap_or("apstar");
|
||||
let dt = candidate
|
||||
.raw
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("data_type"))
|
||||
.and_then(|v| v.as_str())
|
||||
.or(subtype)
|
||||
.unwrap_or("apstar");
|
||||
let rel = parse_sdss_release(release)?;
|
||||
let product = ProductSpec::with_subtype(ProductType::Spectrum, dt);
|
||||
let dr = rel.sas_dr_segment();
|
||||
@ -346,12 +450,21 @@ impl ObservationFetcher for SdssApogeeFetcher {
|
||||
|
||||
// 1) 缓存命中
|
||||
if !force {
|
||||
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Sdss, &product, &cache_key).await? {
|
||||
if cached_files_total_size(&state.config.library_dir, &artifacts).is_some() {
|
||||
if let Some((artifacts, meta)) =
|
||||
fetch_observation_cache(&state.db, Source::Sdss, &product, &cache_key).await?
|
||||
{
|
||||
if cached_files_total_size(&state.config.library_dir, &artifacts)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
info!("[APOGEE] 缓存命中 ({})", cache_key);
|
||||
return Ok(ObservationProduct {
|
||||
source: Source::Sdss, product,
|
||||
source_id: cache_key, source_label, artifacts, source_meta: meta,
|
||||
source: Source::Sdss,
|
||||
product,
|
||||
source_id: cache_key,
|
||||
source_label,
|
||||
artifacts,
|
||||
source_meta: meta,
|
||||
});
|
||||
}
|
||||
warn!("[APOGEE] 缓存记录存在但文件缺失，重新下载 ({})", cache_key);
|
||||
@ -360,10 +473,18 @@ impl ObservationFetcher for SdssApogeeFetcher {
|
||||
|
||||
// 2) 下载 FITS（无压缩）
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let fits_bytes = state.sdss.download_apogee_fits(apogee_id, telescope, field, rel, dt).await?;
|
||||
let safe_id = apogee_id.replace('+', "_").replace('/', "_");
|
||||
let rel_path = format!("Telescope/sdss/{dr}/{dt}/{safe_id}.fits", dr = dr, dt = dt, safe_id = safe_id);
|
||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes)?;
|
||||
let fits_bytes = state
|
||||
.sdss
|
||||
.download_apogee_fits(apogee_id, telescope, field, rel, dt)
|
||||
.await?;
|
||||
let safe_id = apogee_id.replace(['+', '/'], "_");
|
||||
let rel_path = format!(
|
||||
"Telescope/sdss/{dr}/{dt}/{safe_id}.fits",
|
||||
dr = dr,
|
||||
dt = dt,
|
||||
safe_id = safe_id
|
||||
);
|
||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes).await?;
|
||||
info!("[APOGEE] 已保存 {} → {} ({}B)", cache_key, rel_path, size);
|
||||
|
||||
let meta = serde_json::json!({
|
||||
@ -371,24 +492,37 @@ impl ObservationFetcher for SdssApogeeFetcher {
|
||||
"data_type": dt, "dr": dr
|
||||
});
|
||||
let artifact = Artifact {
|
||||
band: None, original_name: None,
|
||||
band: None,
|
||||
original_name: None,
|
||||
file_path: rel_path.clone(),
|
||||
file_url: file_url_from_path(&rel_path),
|
||||
file_format: "fits".to_string(),
|
||||
size_bytes: size, cached: false,
|
||||
size_bytes: size,
|
||||
cached: false,
|
||||
};
|
||||
let meta_str = meta.to_string();
|
||||
if let Err(e) = write_observation_cache(
|
||||
&state.db, Source::Sdss, &product, &cache_key, None, None,
|
||||
std::slice::from_ref(&artifact), Some(&meta_str),
|
||||
).await {
|
||||
&state.db,
|
||||
Source::Sdss,
|
||||
&product,
|
||||
&cache_key,
|
||||
None,
|
||||
None,
|
||||
std::slice::from_ref(&artifact),
|
||||
Some(&meta_str),
|
||||
)
|
||||
.await
|
||||
{
|
||||
log_write_failure(Source::Sdss.as_str(), "apogee", e);
|
||||
}
|
||||
|
||||
Ok(ObservationProduct {
|
||||
source: Source::Sdss, product,
|
||||
source_id: cache_key, source_label,
|
||||
artifacts: vec![artifact], source_meta: Some(meta),
|
||||
source: Source::Sdss,
|
||||
product,
|
||||
source_id: cache_key,
|
||||
source_label,
|
||||
artifacts: vec![artifact],
|
||||
source_meta: Some(meta),
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -398,7 +532,9 @@ fn apogee_row_to_candidate(row: &ApogeeStarRow) -> Candidate {
|
||||
source: Source::Sdss,
|
||||
source_id: format!("{}|{}|{}", row.telescope, row.field, row.apogee_id),
|
||||
label: format!("{} {}", row.telescope, row.apogee_id),
|
||||
ra: row.ra, dec: row.dec, distance: None,
|
||||
ra: row.ra,
|
||||
dec: row.dec,
|
||||
distance: None,
|
||||
raw: Some(serde_json::json!({
|
||||
"apogee_id": row.apogee_id, "telescope": row.telescope, "field": row.field
|
||||
})),
|
||||
|
||||
@ -39,6 +39,24 @@ impl Source {
|
||||
}
|
||||
}
|
||||
|
||||
/// 从字符串解析数据源（大小写不敏感）。新增源只需在此加一行 match arm，
|
||||
/// 所有调用方（API handler、Agent tool）统一用本方法，不再各自手写 match。
|
||||
pub fn from_str(s: &str) -> Result<Self, String> {
|
||||
Ok(match s.to_lowercase().as_str() {
|
||||
"lamost" => Self::Lamost,
|
||||
"gaia" => Self::Gaia,
|
||||
"sdss" => Self::Sdss,
|
||||
"desi" => Self::Desi,
|
||||
other => {
|
||||
return Err(format!(
|
||||
"不支持的 source '{}'，可选: {:?}",
|
||||
other,
|
||||
Self::valid_values()
|
||||
))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn display(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Lamost => "LAMOST",
|
||||
@ -81,6 +99,24 @@ impl ProductType {
|
||||
}
|
||||
}
|
||||
|
||||
/// 从字符串解析产品类型（大小写不敏感，接受常见别名）。
|
||||
/// 别名集中在此一处管理，所有调用方统一用本方法。
|
||||
pub fn from_str(s: &str) -> Result<Self, String> {
|
||||
Ok(match s.to_lowercase().as_str() {
|
||||
"spectrum" | "spec" => Self::Spectrum,
|
||||
"lightcurve" | "light_curve" | "lc" => Self::LightCurve,
|
||||
"photometry" => Self::Photometry,
|
||||
"image" => Self::Image,
|
||||
other => {
|
||||
return Err(format!(
|
||||
"不支持的 product '{}'，可选: {:?}",
|
||||
other,
|
||||
Self::valid_values()
|
||||
))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn display(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Spectrum => "光谱",
|
||||
@ -113,11 +149,17 @@ pub struct ProductSpec {
|
||||
|
||||
impl ProductSpec {
|
||||
pub fn new(product: ProductType) -> Self {
|
||||
Self { product, subtype: None }
|
||||
Self {
|
||||
product,
|
||||
subtype: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_subtype(product: ProductType, subtype: impl Into<String>) -> Self {
|
||||
Self { product, subtype: Some(subtype.into()) }
|
||||
Self {
|
||||
product,
|
||||
subtype: Some(subtype.into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// 落盘子目录：有 subtype 则拼 "product/subtype"，否则只用 product
|
||||
@ -227,6 +269,9 @@ pub enum ObservationRequest {
|
||||
strategy: FindStrategy,
|
||||
#[serde(default)]
|
||||
release: Option<String>,
|
||||
/// 数据发布的子版本（仅 LAMOST 有意义，如 "v2.0"/"v1.0"）；None 时用该 DR 的默认版本
|
||||
#[serde(default)]
|
||||
version: Option<String>,
|
||||
},
|
||||
ByIdentifier {
|
||||
source: Source,
|
||||
@ -234,6 +279,8 @@ pub enum ObservationRequest {
|
||||
identifiers: Vec<String>,
|
||||
#[serde(default)]
|
||||
release: Option<String>,
|
||||
#[serde(default)]
|
||||
version: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
@ -253,7 +300,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_product_spec_path() {
|
||||
assert_eq!(ProductSpec::new(ProductType::Spectrum).path_segment(), "spectrum");
|
||||
assert_eq!(
|
||||
ProductSpec::new(ProductType::Spectrum).path_segment(),
|
||||
"spectrum"
|
||||
);
|
||||
assert_eq!(
|
||||
ProductSpec::with_subtype(ProductType::Spectrum, "lrs").path_segment(),
|
||||
"spectrum/lrs"
|
||||
|
||||
@ -88,6 +88,12 @@ pub async fn upload_paper_file_service(
|
||||
.map_err(|e| anyhow::anyhow!("PDF 文件内容校验失败: {}", e))?;
|
||||
let pdf_filename = format!("{}.pdf", bibcode);
|
||||
let pdf_dest = library_dir.join("PDF").join(&pdf_filename);
|
||||
if pdf_dest
|
||||
.components()
|
||||
.any(|c| matches!(c, std::path::Component::ParentDir))
|
||||
{
|
||||
anyhow::bail!("非法的文件写入路径，bibcode 包含路径穿越字符");
|
||||
}
|
||||
if let Some(parent) = pdf_dest.parent() {
|
||||
tokio::fs::create_dir_all(parent)
|
||||
.await
|
||||
@ -104,6 +110,12 @@ pub async fn upload_paper_file_service(
|
||||
.map_err(|e| anyhow::anyhow!("HTML 文件内容校验失败: {}", e))?;
|
||||
let html_filename = format!("{}.html", bibcode);
|
||||
let html_dest = library_dir.join("HTML").join(&html_filename);
|
||||
if html_dest
|
||||
.components()
|
||||
.any(|c| matches!(c, std::path::Component::ParentDir))
|
||||
{
|
||||
anyhow::bail!("非法的文件写入路径，bibcode 包含路径穿越字符");
|
||||
}
|
||||
if let Some(parent) = html_dest.parent() {
|
||||
tokio::fs::create_dir_all(parent)
|
||||
.await
|
||||
|
||||
@ -85,10 +85,20 @@ pub async fn search_papers(
|
||||
let mut seen_arxiv: HashSet<String> = HashSet::new();
|
||||
|
||||
for r in results {
|
||||
let dup = (!r.bibcode.is_empty() && !seen_bibcodes.insert(r.bibcode.clone()))
|
||||
|| (!r.doi.is_empty() && !seen_dois.insert(r.doi.clone()))
|
||||
|| (!r.arxiv_id.is_empty() && !seen_arxiv.insert(r.arxiv_id.clone()));
|
||||
// 先用引用检查是否重复，仅非重复项才 clone 入 HashSet
|
||||
let dup = (!r.bibcode.is_empty() && seen_bibcodes.contains(&r.bibcode))
|
||||
|| (!r.doi.is_empty() && seen_dois.contains(&r.doi))
|
||||
|| (!r.arxiv_id.is_empty() && seen_arxiv.contains(&r.arxiv_id));
|
||||
if !dup {
|
||||
if !r.bibcode.is_empty() {
|
||||
seen_bibcodes.insert(r.bibcode.clone());
|
||||
}
|
||||
if !r.doi.is_empty() {
|
||||
seen_dois.insert(r.doi.clone());
|
||||
}
|
||||
if !r.arxiv_id.is_empty() {
|
||||
seen_arxiv.insert(r.arxiv_id.clone());
|
||||
}
|
||||
unique_results.push(r);
|
||||
}
|
||||
}
|
||||
@ -199,13 +209,15 @@ async fn batch_fetch_existing_papers(
|
||||
/// 会被替换为空格，确保查询作为纯文本搜索而非布尔表达式执行。
|
||||
pub fn sanitize_fts5_query(query: &str) -> String {
|
||||
query
|
||||
.replace('"', "\"\"")
|
||||
.replace('\'', "''")
|
||||
.replace(['"', '\''], " ")
|
||||
.replace(['*', ':'], " ")
|
||||
.replace(" AND ", " ")
|
||||
.replace(" OR ", " ")
|
||||
.replace(" NOT ", " ")
|
||||
.replace(" NEAR ", " ")
|
||||
.split_whitespace()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
}
|
||||
|
||||
/// 本地文献库 FTS5 全文检索。
|
||||
|
||||
@ -74,23 +74,22 @@ impl Dictionary {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// 基础分词清理：保留字母数字及连接符，其余视为空格以进行精确段落划分
|
||||
let clean_text = text
|
||||
// 基础分词清理 + 预计算 lowercase，减少后续重复分配
|
||||
let clean_text: String = text
|
||||
.chars()
|
||||
.map(|c| {
|
||||
if c.is_alphanumeric() || c == '-' || c == '\'' || c == ' ' {
|
||||
c
|
||||
c.to_ascii_lowercase()
|
||||
} else {
|
||||
' '
|
||||
}
|
||||
})
|
||||
.collect::<String>();
|
||||
.collect();
|
||||
|
||||
let words: Vec<&str> = clean_text.split_whitespace().collect();
|
||||
let mut matched = HashSet::new();
|
||||
let mut results = Vec::new();
|
||||
|
||||
// 天文学词条跨度最大限制（一般多词短语不超过 6 个英文单词）
|
||||
let max_span = 6;
|
||||
let n = words.len();
|
||||
let mut next_valid_index = 0;
|
||||
@ -99,22 +98,19 @@ impl Dictionary {
|
||||
if i < next_valid_index {
|
||||
continue;
|
||||
}
|
||||
// 快速过滤：如果当前单词的 lowercase 不在 first_words 集合中，不可能匹配任何词条
|
||||
let first_word_lower = words[i].to_lowercase();
|
||||
if !self.first_words.contains(&first_word_lower) {
|
||||
// 已 lowercase 的首词直接查 first_words
|
||||
if !self.first_words.contains(words[i]) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for len in (1..=max_span).rev() {
|
||||
if i + len <= n {
|
||||
let phrase_slice = &words[i..i + len];
|
||||
let phrase = phrase_slice.join(" ").to_lowercase();
|
||||
let phrase = words[i..i + len].join(" ");
|
||||
|
||||
if self.terms.contains_key(&phrase) {
|
||||
// 避免重复匹配更长名词的子词 (如已匹配 'active galactic nucleus' 就不重复提取其中的 'nucleus')
|
||||
if !matched.contains(&phrase) {
|
||||
let chinese = self.terms.get(&phrase).unwrap().clone();
|
||||
let original_phrase = phrase_slice.join(" ");
|
||||
let original_phrase = words[i..i + len].join(" ");
|
||||
|
||||
results.push((original_phrase, chinese));
|
||||
matched.insert(phrase);
|
||||
@ -248,7 +244,7 @@ pub async fn translate_markdown(
|
||||
|
||||
info!(
|
||||
"正在请求大模型开展中英翻译。所选大模型: {}",
|
||||
llm_client.model()
|
||||
llm_client.model().await
|
||||
);
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
@ -260,7 +256,7 @@ pub async fn translate_markdown(
|
||||
let duration = start_time.elapsed();
|
||||
info!(
|
||||
"LLM 翻译成功。所选大模型: {}, 耗时: {:?}, 译文字符数: {}",
|
||||
llm_client.model(),
|
||||
llm_client.model().await,
|
||||
duration,
|
||||
translated.len()
|
||||
);
|
||||
|
||||
@ -30,7 +30,7 @@ pub async fn analyze_image_stream_service(
|
||||
// 流式输出：先发送状态头部（作为第一条 text_delta，保证持久化到工具结果中）
|
||||
let header = format!(
|
||||
"> 正在使用视觉模型（{}）分析图片: {}\n\n",
|
||||
vision_llm.model(),
|
||||
vision_llm.model().await,
|
||||
image_path
|
||||
);
|
||||
if let Some(tx) = sse_tx {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user