feat: LAMOST DR12-14 与子版本体系接入、观测层安全加固与并发异步化
- LAMOST 新增 DR12/13/14 及子版本(v0/v1.0/v1.1/v2.0)维度,Internal 发布标记需登录认证并前端灰显,release×subtype 交叉约束下沉至 capabilities 统一声明 - ObservationFetcher trait 扩展版本/认证/交叉约束能力声明,version 参数贯穿 client→service→API→Agent tool→前端全链路 - 安全:observation cache SQL 全参数绑定 + LIKE 转义、cone_cache_hash 加长度前缀防碰撞、DESI survey/program 白名单防穿越 - 异步化:persist/cached_files_total_size/maybe_persist_tool_result迁移到 tokio::fs;cancelled_runs 与 session_permission_checkers改用 DashMap;auth 读锁优先 + 60s 节流 - Gaia 去 native-tls 改禁用连接池规避 UnexpectedEof,reqwest 移除 native-tls feature - 重构:Source/ProductType from_str 集中解析、download 模块拆分为 try_download_pdf/html、AgentRuntime::init 抽取共享逻辑 - 部署:新增 deploy.sh 一键打包推送脚本、catch-panic 启用
This commit is contained in:
parent
2f1fd19d74
commit
2c8d0b8f8b
131
CLAUDE.md
131
CLAUDE.md
@ -42,6 +42,9 @@ cargo test --lib
|
|||||||
# Run a specific test
|
# Run a specific test
|
||||||
cargo test test_name
|
cargo test test_name
|
||||||
|
|
||||||
|
# Coverage (requires cargo-llvm-cov)
|
||||||
|
cargo llvm-cov --fail-under-lines 80
|
||||||
|
|
||||||
# Frontend (cd dashboard first)
|
# Frontend (cd dashboard first)
|
||||||
npm run dev # HMR dev server on :5173, proxies /api to :8000
|
npm run dev # HMR dev server on :5173, proxies /api to :8000
|
||||||
npm run build # TypeScript check + Vite production build → dashboard/dist/
|
npm run build # TypeScript check + Vite production build → dashboard/dist/
|
||||||
@ -62,44 +65,101 @@ src/
|
|||||||
├── main.rs # Axum server entry: logging, DB pool, migrations, vec0 table,
|
├── main.rs # Axum server entry: logging, DB pool, migrations, vec0 table,
|
||||||
│ # client/service construction, route registration, AppState assembly
|
│ # client/service construction, route registration, AppState assembly
|
||||||
├── lib.rs # Config struct + from_env() loading from .env
|
├── lib.rs # Config struct + from_env() loading from .env
|
||||||
├── api/ # HTTP handlers, AppState, StandardPaper type
|
├── api/ # HTTP handlers, AppState, re-exports via handlers namespace
|
||||||
│ ├── mod.rs # AppState (shared state), StandardPaper, handlers re-exports
|
│ ├── mod.rs # AppState (shared state), handler re-exports, permission/event types
|
||||||
│ ├── agent.rs # SSE chat_agent endpoint, session CRUD, metrics, audit log
|
│ ├── agent.rs # SSE chat_agent endpoint, session CRUD, metrics, audit log
|
||||||
│ ├── papers.rs # Search, download, parse, translate, embed, citations, library, export
|
│ ├── papers.rs # Search, download, parse, translate, embed, citations, library, export
|
||||||
|
│ ├── catalog.rs # VizieR TAP catalog queries + cone search endpoints
|
||||||
|
│ ├── observation.rs # Observation data search/download (LAMOST/Gaia/SDSS/DESI)
|
||||||
│ ├── notes.rs # Highlight/note CRUD
|
│ ├── notes.rs # Highlight/note CRUD
|
||||||
│ ├── sync.rs # Meta-sync and asset-batch endpoints
|
│ ├── sync.rs # Meta-sync and asset-batch endpoints
|
||||||
│ ├── targets.rs # Target query/associate/extract, RAG chat, figure chat
|
│ ├── targets.rs # Target query/associate/extract, RAG chat, figure chat
|
||||||
│ └── helpers.rs # Shared DB helpers, format conversion, path validation
|
│ ├── auth.rs # Login/logout/auth-check (token-based, session memory)
|
||||||
|
│ ├── permissions.rs # Agent tool permission rule management
|
||||||
|
│ ├── search.rs # Search history endpoint
|
||||||
|
│ ├── error.rs # Unified API error type (status code + user-safe message)
|
||||||
|
│ └── helpers.rs # Shared DB helpers, format conversion, path validation (private)
|
||||||
├── agent/ # ReAct-based research agent (LLM-driven tool-use loop)
|
├── agent/ # ReAct-based research agent (LLM-driven tool-use loop)
|
||||||
│ ├── tools/ # AgentTool trait, ToolRegistry, 25+ tool implementations per domain file
|
│ ├── tools/ # AgentTool trait, ToolRegistry, tool implementations per domain
|
||||||
│ ├── runtime/ # ReAct loop core: session, context, streaming, executor, token_budget,
|
│ │ ├── mod.rs # AgentTool trait, ToolContext (session_id, sse_tx, silent, etc.)
|
||||||
│ │ # permission, permission_profile, checkpoint, circuit_breaker, error_recovery,
|
│ │ ├── filesystem/ # read_file, grep_files, write, edit
|
||||||
│ │ # hardline, partitioner, file_cache, system_prompt, finalize, denial_tracker
|
│ │ ├── astro/ # search_papers, download_paper, parse_paper, rag_search,
|
||||||
|
│ │ │ # query_target, save_note, analyze_image, research/*
|
||||||
|
│ │ ├── ask_user.rs # Permission-gated interactive user question
|
||||||
|
│ │ ├── background.rs# bg_task_run, bg_task_check (async slow tasks)
|
||||||
|
│ │ ├── compress.rs # Context compression trigger
|
||||||
|
│ │ ├── memory.rs # save_memory, load_memory
|
||||||
|
│ │ ├── persist.rs # Data persistence (used by memory/compaction)
|
||||||
|
│ │ ├── skill.rs # load_skill
|
||||||
|
│ │ ├── subagent.rs # Context-isolated sub-agent spawner
|
||||||
|
│ │ ├── team.rs # spawn_teammate, send_teammate_message, etc.
|
||||||
|
│ │ └── todo.rs # todo_write (task tracking)
|
||||||
|
│ ├── runtime/ # ReAct loop core
|
||||||
|
│ │ ├── mod.rs # AgentRuntime: session create/resume → context → ReAct → finalize
|
||||||
|
│ │ ├── executor/ # Step executor + helpers (tool dispatch, parallel tool calls)
|
||||||
|
│ │ ├── streaming.rs # SSE AgentStreamEvent types
|
||||||
|
│ │ ├── streaming_executor.rs # SSE-emitting executor wrapper
|
||||||
|
│ │ ├── session.rs # Session CRUD, DB persist, agent_config
|
||||||
|
│ │ ├── token_budget.rs # Soft/hard token limits, tiktoken counting
|
||||||
|
│ │ ├── permission.rs # Deny > Allow > Ask rule chain, pattern matching
|
||||||
|
│ │ ├── permission_profile.rs # Named profiles (research, readonly)
|
||||||
|
│ │ ├── checkpoint.rs # Rewind/retry state snapshots
|
||||||
|
│ │ ├── circuit_breaker.rs# CompactionCircuitBreaker (cooldown rate limiting)
|
||||||
|
│ │ ├── error_recovery.rs # Error classification + retry strategies
|
||||||
|
│ │ ├── hardline.rs # Terminal conditions (too many denials, loops, etc.)
|
||||||
|
│ │ ├── partitioner.rs # Message window partitioning for compaction
|
||||||
|
│ │ ├── file_cache.rs # ReadFileState — file content dedup cache
|
||||||
|
│ │ ├── system_prompt.rs # System prompt assembly (mode, skills, memory, tools)
|
||||||
|
│ │ ├── finalize.rs # Post-loop cleanup: memory extraction, audit log
|
||||||
|
│ │ └── denial_tracker.rs # Consecutive denial detection
|
||||||
│ ├── compact/ # Context compression (micro/auto/manual layers + collapse)
|
│ ├── compact/ # Context compression (micro/auto/manual layers + collapse)
|
||||||
│ ├── memory/ # Persistent memory manager: extraction, dedup, decay, age, guardrails
|
│ ├── memory/ # Persistent memory manager: extraction, dedup, decay, age, guardrails
|
||||||
│ ├── hooks/ # Lifecycle hooks: registry, dispatch, builtins, matcher, traits (PreToolUse/PostToolUse/Stop/etc.)
|
│ ├── hooks/ # Lifecycle hooks: registry, dispatch, builtins, matcher, traits
|
||||||
│ ├── modes/ # Agent session modes: default, deep-research, literature-reader (identity/tools/config presets)
|
│ ├── modes/ # Agent session modes (see Agent Session Modes section below)
|
||||||
│ ├── skills.rs # SkillRegistry: hot-loads SKILL.md files from skills/ directory
|
│ ├── skills.rs # SkillRegistry: hot-loads SKILL.md files from skills/ directory
|
||||||
│ ├── subagent.rs # Context-isolated sub-agent runner (subagent tool)
|
│ ├── subagent.rs # Context-isolated sub-agent runner
|
||||||
│ ├── team/ # Multi-agent team: file-based inbox, lead/teammate coordination
|
│ ├── team/ # Multi-agent team: file-based inbox, lead/teammate coordination
|
||||||
│ ├── background.rs# BgNotificationQueue for async slow-task (download/parse) notifications
|
│ ├── background.rs# BgNotificationQueue for async slow-task notifications
|
||||||
│ ├── task_board.rs# Persistent task board (agent_tasks table)
|
│ ├── task_board.rs# Persistent task board (agent_tasks table)
|
||||||
│ ├── trajectory.rs# Session trajectory recording for audit/debug
|
│ ├── trajectory.rs# Session trajectory recording for audit/debug
|
||||||
│ └── terminal.rs # Escape sequence filter for ANSI-heavy tool outputs
|
│ └── terminal.rs # Escape sequence filter for ANSI-heavy tool outputs
|
||||||
├── clients/ # External API wrappers
|
├── clients/ # External API wrappers (pure communication, no business logic)
|
||||||
│ ├── llm.rs # LlmClient (OpenAI-compatible chat + streaming), EmbeddingClient
|
│ ├── llm/ # LlmClient (OpenAI-compatible chat + streaming), EmbeddingClient
|
||||||
│ ├── ads.rs # NASA ADS API
|
│ ├── ads.rs # NASA ADS API
|
||||||
│ ├── arxiv.rs # arXiv Atom XML API
|
│ ├── arxiv.rs # arXiv Atom XML API
|
||||||
|
│ ├── cds/ # CDS services (sesame.rs — Sesame name resolver; vizier.rs — TAP queries)
|
||||||
|
│ ├── gaia/ # Gaia archive (TAP + DataLink)
|
||||||
|
│ ├── lamost/ # LAMOST archive (ConeSearch + FITS.gz download)
|
||||||
|
│ ├── sdss/ # SDSS archive (Data Lab TAP + SAS bulk download)
|
||||||
|
│ ├── desi/ # DESI archive (Data Lab TAP + HEALPix coadd SAS)
|
||||||
|
│ ├── vo/ # Virtual Observatory generic client
|
||||||
│ └── qiniu.rs # Qiniu cloud storage
|
│ └── qiniu.rs # Qiniu cloud storage
|
||||||
├── services/ # Business logic
|
├── services/ # Business logic
|
||||||
│ ├── search.rs # Unified cross-source search (ADS + arXiv dedup)
|
│ ├── search.rs # Unified cross-source search (ADS + arXiv dedup)
|
||||||
│ ├── download.rs # PDF/HTML download with anti-bot measures and fallback chain
|
│ ├── download.rs # PDF/HTML download with anti-bot measures and fallback chain
|
||||||
│ ├── parser/ # HTML/PDF → Markdown parsers (A&A, IOP, ar5iv, generic, PDF via MinerU)
|
│ ├── parser/ # HTML/PDF → Markdown parsers (A&A, IOP, ar5iv, generic, PDF via MinerU)
|
||||||
|
│ ├── paper/ # Paper lifecycle: model, db (CRUD), ingest (ADS/arXiv→DB), reader (markdown)
|
||||||
│ ├── translation.rs # LLM bilingual translation with Trie-based astronomy glossary
|
│ ├── translation.rs # LLM bilingual translation with Trie-based astronomy glossary
|
||||||
│ ├── rag.rs # Embedding ingest + vector similarity retrieval + LLM answer generation
|
│ ├── rag.rs # Embedding ingest + vector similarity retrieval + LLM answer generation
|
||||||
│ ├── target.rs # Celestial target extraction (IAU name regex) + CDS Sesame lookup
|
│ ├── cds/ # CDS business layer: target.rs (Sesame name resolver), vizier.rs (TAP + cache)
|
||||||
│ ├── chunker.rs # Markdown text chunking for embedding
|
│ ├── observation/ # Cross-source observation data (spectra, light curves, photometry, images)
|
||||||
|
│ │ ├── mod.rs # 双轴正交架构: Source × ProductType → ObservationFetcher → Registry
|
||||||
|
│ │ ├── types.rs # Source/ProductType/ProductSpec/Artifact/ObservationProduct/Candidate
|
||||||
|
│ │ ├── fetcher.rs # ObservationFetcher trait + cone search cache template methods
|
||||||
|
│ │ ├── registry.rs # ObservationRegistry (fetcher registration, capability discovery)
|
||||||
|
│ │ ├── dispatch.rs # search_observation + download_observation (unified entry points)
|
||||||
|
│ │ ├── cache.rs # observation_cache table read/write + file staging
|
||||||
|
│ │ ├── lamost.rs # LamostSpectrumFetcher
|
||||||
|
│ │ ├── gaia.rs # GaiaSpectrumFetcher + GaiaLightCurveFetcher
|
||||||
|
│ │ ├── sdss.rs # SdssSpectrumFetcher + SdssApogeeFetcher
|
||||||
|
│ │ └── desi.rs # DesiSpectrumFetcher
|
||||||
│ ├── batch/ # Meta-sync (ADS bulk harvest) and asset-batch processing engines
|
│ ├── batch/ # Meta-sync (ADS bulk harvest) and asset-batch processing engines
|
||||||
|
│ ├── citation.rs # Citation network graph data (references + citations)
|
||||||
|
│ ├── session.rs # Agent session listing, metrics, audit log (for API consumption)
|
||||||
|
│ ├── vision.rs # Image analysis via dedicated vision model
|
||||||
|
│ ├── note.rs # Highlight/note storage and retrieval
|
||||||
|
│ ├── pipeline.rs # Synchronous paper processing pipeline (download→parse→translate→embed)
|
||||||
|
│ ├── chunker.rs # Markdown text chunking for embedding
|
||||||
│ ├── query_parser.rs # Advanced search query syntax parser
|
│ ├── query_parser.rs # Advanced search query syntax parser
|
||||||
│ └── logging.rs # Pretty console + rolling file logger
|
│ └── logging.rs # Pretty console + rolling file logger
|
||||||
└── bin/
|
└── bin/
|
||||||
@ -113,11 +173,18 @@ src/
|
|||||||
All handlers access state via `Arc<AppState>`. Key fields:
|
All handlers access state via `Arc<AppState>`. Key fields:
|
||||||
|
|
||||||
- `db: SqlitePool` — SQLite connection pool (5 max connections, foreign keys enforced)
|
- `db: SqlitePool` — SQLite connection pool (5 max connections, foreign keys enforced)
|
||||||
- `llm: LlmClient` / `embedding: EmbeddingClient` — OpenAI-compatible LLM clients
|
- `llm: LlmClient` / `medium_llm` / `fast_llm` — OpenAI-compatible LLM clients (multi-tier)
|
||||||
|
- `vision_llm: Option<LlmClient>` — dedicated vision model client
|
||||||
|
- `embedding: EmbeddingClient` — embedding model client
|
||||||
- `ads: AdsClient` / `arxiv: ArxivClient` — academic search clients
|
- `ads: AdsClient` / `arxiv: ArxivClient` — academic search clients
|
||||||
|
- `vizier: VizierClient` — CDS VizieR TAP catalog query client
|
||||||
|
- `lamost` / `gaia` / `sdss` / `desi` — observation data source clients
|
||||||
|
- `observation_registry: Arc<ObservationRegistry>` — cross-source observation fetcher registry
|
||||||
- `skill_registry: Arc<RwLock<SkillRegistry>>` — hot-reloaded agent skills
|
- `skill_registry: Arc<RwLock<SkillRegistry>>` — hot-reloaded agent skills
|
||||||
- `memory_manager: Arc<MemoryManager>` — persistent agent memory (MEMORY.md + decay)
|
- `memory_manager: Arc<MemoryManager>` — persistent agent memory (MEMORY.md + decay)
|
||||||
- `cancelled_runs: Arc<Mutex<HashSet<String>>>` — agent cancellation tokens
|
- `cancelled_runs: Arc<DashMap<String, ()>>` — agent cancellation tokens
|
||||||
|
- `session_permission_checkers: Arc<DashMap<String, PermissionChecker>>` — per-session permission rules
|
||||||
|
- `pending_questions` / `pending_permissions` — interactive agent-user handshake channels
|
||||||
- `harvest_status` / `batch_status` — async batch operation status tracking
|
- `harvest_status` / `batch_status` — async batch operation status tracking
|
||||||
|
|
||||||
### Agent System Design
|
### Agent System Design
|
||||||
@ -131,7 +198,7 @@ The agent (`src/agent/`) implements a **ReAct** (Thought → Action → Observat
|
|||||||
5. **Tool call ID tracking**: LLM may not return tool_call IDs — `LlmClient` generates UUID fallbacks. `ToolCall` and `ToolResult` SSE events carry matching IDs for precise frontend pairing.
|
5. **Tool call ID tracking**: LLM may not return tool_call IDs — `LlmClient` generates UUID fallbacks. `ToolCall` and `ToolResult` SSE events carry matching IDs for precise frontend pairing.
|
||||||
6. **ToolContext** (`tools/mod.rs`): Injected into every tool execution — holds `app_state`, `session_id`, `sse_tx` (for intermediate events), `enable_thinking`, `read_file_state` (file cache for dedup), `silent` (sub-agents skip permission prompts).
|
6. **ToolContext** (`tools/mod.rs`): Injected into every tool execution — holds `app_state`, `session_id`, `sse_tx` (for intermediate events), `enable_thinking`, `read_file_state` (file cache for dedup), `silent` (sub-agents skip permission prompts).
|
||||||
7. **Context compression**: Four layers — micro (placeholder replacement), snip (old-message truncation), auto (LLM summarization), aggro_micro (aggressive placeholder). Protected by `CompactionCircuitBreaker`. Transcripts persisted in `agent_messages` table, not filesystem snapshots.
|
7. **Context compression**: Four layers — micro (placeholder replacement), snip (old-message truncation), auto (LLM summarization), aggro_micro (aggressive placeholder). Protected by `CompactionCircuitBreaker`. Transcripts persisted in `agent_messages` table, not filesystem snapshots.
|
||||||
8. **Skills** (`skills.rs`): Two-layer loading — system-reminder lists names (~20 tokens each), LLM calls `load_skill` to inject full SKILL.md content
|
8. **Skills** (`skills.rs`): Two-layer loading — system-reminder lists names (~20 tokens each), LLM calls `load_skill` to inject full SKILL.md content. Skills are organized into subdirectories: `methodology/`, `plotting/`, `presentation/`.
|
||||||
9. **Sub-agents** (`subagent.rs`): `subagent` tool spawns a context-isolated sub-agent with its own ReAct loop. Sub-agent messages (system/user/assistant/tool) are persisted to `agent_messages` with `agent_name` identifier. Returns final summary + activity log. SSE progress forwarded to parent via ToolContext.
|
9. **Sub-agents** (`subagent.rs`): `subagent` tool spawns a context-isolated sub-agent with its own ReAct loop. Sub-agent messages (system/user/assistant/tool) are persisted to `agent_messages` with `agent_name` identifier. Returns final summary + activity log. SSE progress forwarded to parent via ToolContext.
|
||||||
10. **Memory** (`memory/`): File-based persistent memory (MEMORY.md). `MemoryManager` handles extraction from conversation, dedup, recency decay, age-based pruning, and guardrails. Tools: `save_memory`, `load_memory` (auto-injected in system prompt).
|
10. **Memory** (`memory/`): File-based persistent memory (MEMORY.md). `MemoryManager` handles extraction from conversation, dedup, recency decay, age-based pruning, and guardrails. Tools: `save_memory`, `load_memory` (auto-injected in system prompt).
|
||||||
11. **Teams** (`team/`): File-based inbox directory per session for lead/teammate message passing
|
11. **Teams** (`team/`): File-based inbox directory per session for lead/teammate message passing
|
||||||
@ -141,17 +208,29 @@ Environment variables for agent tuning: `AGENT_TOKEN_SOFT_LIMIT` (default 80000)
|
|||||||
|
|
||||||
### Database
|
### Database
|
||||||
|
|
||||||
SQLite via `sqlx::sqlite`. Migrations in `migrations/` are auto-run on startup (`sqlx::migrate!("./migrations")`). Key tables: `papers`, `citations_references`, `notes`, `agent_sessions`, `agent_messages`, `agent_tasks`, `agent_audit_log`, `paper_chunks_content`. Vector embeddings use `sqlite-vec` (`vec_paper_chunks` virtual table, auto-registered before any DB connection).
|
SQLite via `sqlx::sqlite`. Migrations in `migrations/` are auto-run on startup (`sqlx::migrate!("./migrations")`). Key tables: `papers`, `citations_references`, `notes`, `agent_sessions`, `agent_messages`, `agent_tasks`, `agent_audit_log`, `paper_chunks_content`, `vizier_cache`, `observation_cache`. Vector embeddings use `sqlite-vec` (`vec_paper_chunks` virtual table, auto-registered before any DB connection).
|
||||||
|
|
||||||
The embedding dimension is controlled by `EMBEDDING_DIM` env var (default 1536). On dimension mismatch, the vec table and chunk content are dropped and recreated.
|
The embedding dimension is controlled by `EMBEDDING_DIM` env var (default 1536). On dimension mismatch, the vec table and chunk content are dropped and recreated.
|
||||||
|
|
||||||
|
### Observation Data System (`src/services/observation/`)
|
||||||
|
|
||||||
|
A cross-source astronomical observation data subsystem with a **dual-axis orthogonal** architecture: **Source** (LAMOST/Gaia/SDSS/DESI) × **ProductType** (Spectrum/LightCurve/Photometry/Image). Each valid combination implements an `ObservationFetcher`, registered in `ObservationRegistry`.
|
||||||
|
|
||||||
|
New fetchers follow OCP (pure addition, no modification to existing code):
|
||||||
|
|
||||||
|
1. Create `{source}.rs`, implement `ObservationFetcher` trait
|
||||||
|
2. Add `register!(YourFetcher)` in `registry.rs::default()`
|
||||||
|
3. Add new variants to `Source`/`ProductType` enums in `types.rs` if needed
|
||||||
|
|
||||||
|
External callers use `dispatch::search_observation()` (metadata only) and `dispatch::download_observation()` (metadata + file download) via `ObservationRequest`. Results are cached in `observation_cache` table with file staging to `library/`.
|
||||||
|
|
||||||
### Frontend (dashboard/)
|
### Frontend (dashboard/)
|
||||||
|
|
||||||
React 19 + TypeScript + Vite + Tailwind CSS 4. Organized by technical layer (Type-Based):
|
React 19 + TypeScript + Vite + Tailwind CSS 4. Organized by technical layer (Type-Based):
|
||||||
|
|
||||||
- `pages/` — Page-level panel view components (SearchPanel, LibraryPanel, ReaderPanel, CitationPanel, SyncPanel, ResearchAgentPanel, SettingsPanel)
|
- `pages/` — Page-level panel view components (SearchPanel, LibraryPanel, ReaderPanel, CitationPanel, SyncPanel, ResearchAgentPanel, ObservationPanel, SettingsPanel)
|
||||||
- `components/` — Reusable and layout components (sub-folders: agent, reader, sync, layout, dialogs)
|
- `components/` — Reusable and layout components (sub-folders: agent, reader, sync, layout, dialogs)
|
||||||
- `hooks/` — Global and feature-specific custom stateful React Hooks (e.g., useLibrary, useSearch, useNotes)
|
- `hooks/` — Global and feature-specific custom stateful React Hooks (e.g., useLibrary, useSearch, useNotes, useObservation)
|
||||||
- `types/` — Global TypeScript type definitions (types/index.ts)
|
- `types/` — Global TypeScript type definitions (types/index.ts)
|
||||||
- `utils/` — Common utility helper functions
|
- `utils/` — Common utility helper functions
|
||||||
- `assets/` — Static assets and global stylesheet styles
|
- `assets/` — Static assets and global stylesheet styles
|
||||||
@ -160,7 +239,7 @@ Dependencies: `react-markdown` + `rehype-katex` + `remark-math` for Markdown/LaT
|
|||||||
|
|
||||||
### Obscura In-Process Browser
|
### Obscura In-Process Browser
|
||||||
|
|
||||||
The `obscura-inprocess` feature compiles `obscura-browser` and `obscura-net` directly into the binary, eliminating the need for external browser binaries. This is used for bypassing Cloudflare/WAF on PDF download. Enabled via `--features obscura-inprocess`.
|
The `obscura-inprocess` feature compiles `obscura-browser` and `obscura-net` from `libs/obscura/` directly into the binary, eliminating the need for external browser binaries. This is used for bypassing Cloudflare/WAF on PDF download. Enabled via `--features obscura-inprocess`.
|
||||||
|
|
||||||
### Astronomy Glossary (dictionary.txt)
|
### Astronomy Glossary (dictionary.txt)
|
||||||
|
|
||||||
@ -235,6 +314,10 @@ Controlled by env vars:
|
|||||||
- `EXTRACT_MEMORY_THROTTLE_TURNS` (default `3`) — extract every N turns
|
- `EXTRACT_MEMORY_THROTTLE_TURNS` (default `3`) — extract every N turns
|
||||||
- `EXTRACT_MEMORY_MAX_STEPS` (default `3`) — sub-agent max steps for extraction
|
- `EXTRACT_MEMORY_MAX_STEPS` (default `3`) — sub-agent max steps for extraction
|
||||||
|
|
||||||
|
### Deployment (`deploy.sh`)
|
||||||
|
|
||||||
|
The `deploy.sh` script builds the project locally (release-min profile, Composer-based) and deploys to a remote server via rsync + SSH. It handles: source compilation, frontend build, remote directory sync, service restart via docker compose, and health check verification.
|
||||||
|
|
||||||
## Code Conventions
|
## Code Conventions
|
||||||
|
|
||||||
- Use `anyhow` for application errors, `thiserror` for library-style typed errors
|
- Use `anyhow` for application errors, `thiserror` for library-style typed errors
|
||||||
@ -243,3 +326,7 @@ Controlled by env vars:
|
|||||||
- API handlers take `State(Arc<AppState>)` and return Axum-compatible responses
|
- API handlers take `State(Arc<AppState>)` and return Axum-compatible responses
|
||||||
- Agent tools implement `AgentTool` trait; new tools register in `ToolRegistry::new()`
|
- Agent tools implement `AgentTool` trait; new tools register in `ToolRegistry::new()`
|
||||||
- Front-end build is triggered by `build.rs` (auto npm install + build when `dashboard/src/` changes)
|
- Front-end build is triggered by `build.rs` (auto npm install + build when `dashboard/src/` changes)
|
||||||
|
- New observation fetchers follow OCP: add module + register — don't modify existing fetchers or dispatch
|
||||||
|
- External API clients (`src/clients/`) are pure communication; business logic lives in `src/services/`
|
||||||
|
- Route registration happens in `src/main.rs`; handler re-exports are maintained in `src/api/mod.rs` → `handlers` namespace
|
||||||
|
- Agent hooks (`src/agent/hooks/`) implement lifecycles: PreToolUse, PostToolUse, Stop, etc. — register in `builtins.rs`
|
||||||
|
|||||||
168
Cargo.lock
generated
168
Cargo.lock
generated
@ -138,6 +138,7 @@ dependencies = [
|
|||||||
"base64 0.22.1",
|
"base64 0.22.1",
|
||||||
"chrono",
|
"chrono",
|
||||||
"clap",
|
"clap",
|
||||||
|
"dashmap",
|
||||||
"dotenvy",
|
"dotenvy",
|
||||||
"flate2",
|
"flate2",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
@ -481,7 +482,7 @@ checksum = "2c5e60b8c8d282c86360cab651ded04ab0335a7b5390c8d34145cbeab8cacf5f"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"bitflags 2.13.0",
|
"bitflags 2.13.0",
|
||||||
"btls-sys",
|
"btls-sys",
|
||||||
"foreign-types 0.5.0",
|
"foreign-types",
|
||||||
"libc",
|
"libc",
|
||||||
"openssl-macros",
|
"openssl-macros",
|
||||||
]
|
]
|
||||||
@ -761,16 +762,6 @@ dependencies = [
|
|||||||
"url",
|
"url",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "core-foundation"
|
|
||||||
version = "0.10.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
|
|
||||||
dependencies = [
|
|
||||||
"core-foundation-sys",
|
|
||||||
"libc",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "core-foundation-sys"
|
name = "core-foundation-sys"
|
||||||
version = "0.8.7"
|
version = "0.8.7"
|
||||||
@ -900,6 +891,20 @@ dependencies = [
|
|||||||
"cmov",
|
"cmov",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "dashmap"
|
||||||
|
version = "6.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"crossbeam-utils",
|
||||||
|
"hashbrown 0.14.5",
|
||||||
|
"lock_api",
|
||||||
|
"once_cell",
|
||||||
|
"parking_lot_core",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "data-encoding"
|
name = "data-encoding"
|
||||||
version = "2.11.0"
|
version = "2.11.0"
|
||||||
@ -1242,15 +1247,6 @@ version = "0.2.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "foreign-types"
|
|
||||||
version = "0.3.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
|
|
||||||
dependencies = [
|
|
||||||
"foreign-types-shared 0.1.1",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "foreign-types"
|
name = "foreign-types"
|
||||||
version = "0.5.0"
|
version = "0.5.0"
|
||||||
@ -1258,7 +1254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965"
|
checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"foreign-types-macros",
|
"foreign-types-macros",
|
||||||
"foreign-types-shared 0.3.1",
|
"foreign-types-shared",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@ -1272,12 +1268,6 @@ dependencies = [
|
|||||||
"syn 2.0.117",
|
"syn 2.0.117",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "foreign-types-shared"
|
|
||||||
version = "0.1.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "foreign-types-shared"
|
name = "foreign-types-shared"
|
||||||
version = "0.3.1"
|
version = "0.3.1"
|
||||||
@ -1760,29 +1750,13 @@ dependencies = [
|
|||||||
"http",
|
"http",
|
||||||
"hyper",
|
"hyper",
|
||||||
"hyper-util",
|
"hyper-util",
|
||||||
"rustls 0.23.40",
|
"rustls 0.23.41",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-rustls",
|
"tokio-rustls",
|
||||||
"tower-service",
|
"tower-service",
|
||||||
"webpki-roots 1.0.7",
|
"webpki-roots 1.0.7",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "hyper-tls"
|
|
||||||
version = "0.6.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
|
|
||||||
dependencies = [
|
|
||||||
"bytes",
|
|
||||||
"http-body-util",
|
|
||||||
"hyper",
|
|
||||||
"hyper-util",
|
|
||||||
"native-tls",
|
|
||||||
"tokio",
|
|
||||||
"tokio-native-tls",
|
|
||||||
"tower-service",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hyper-util"
|
name = "hyper-util"
|
||||||
version = "0.1.20"
|
version = "0.1.20"
|
||||||
@ -2419,23 +2393,6 @@ dependencies = [
|
|||||||
"version_check",
|
"version_check",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "native-tls"
|
|
||||||
version = "0.2.18"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"
|
|
||||||
dependencies = [
|
|
||||||
"libc",
|
|
||||||
"log",
|
|
||||||
"openssl",
|
|
||||||
"openssl-probe",
|
|
||||||
"openssl-sys",
|
|
||||||
"schannel",
|
|
||||||
"security-framework",
|
|
||||||
"security-framework-sys",
|
|
||||||
"tempfile",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "new_debug_unreachable"
|
name = "new_debug_unreachable"
|
||||||
version = "1.0.6"
|
version = "1.0.6"
|
||||||
@ -2624,20 +2581,6 @@ version = "1.70.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "openssl"
|
|
||||||
version = "0.10.80"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags 2.13.0",
|
|
||||||
"cfg-if",
|
|
||||||
"foreign-types 0.3.2",
|
|
||||||
"libc",
|
|
||||||
"openssl-macros",
|
|
||||||
"openssl-sys",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "openssl-macros"
|
name = "openssl-macros"
|
||||||
version = "0.1.1"
|
version = "0.1.1"
|
||||||
@ -2649,24 +2592,6 @@ dependencies = [
|
|||||||
"syn 2.0.117",
|
"syn 2.0.117",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "openssl-probe"
|
|
||||||
version = "0.2.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "openssl-sys"
|
|
||||||
version = "0.9.117"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
|
|
||||||
dependencies = [
|
|
||||||
"cc",
|
|
||||||
"libc",
|
|
||||||
"pkg-config",
|
|
||||||
"vcpkg",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "outref"
|
name = "outref"
|
||||||
version = "0.5.2"
|
version = "0.5.2"
|
||||||
@ -2954,7 +2879,7 @@ dependencies = [
|
|||||||
"quinn-proto",
|
"quinn-proto",
|
||||||
"quinn-udp",
|
"quinn-udp",
|
||||||
"rustc-hash",
|
"rustc-hash",
|
||||||
"rustls 0.23.40",
|
"rustls 0.23.41",
|
||||||
"socket2",
|
"socket2",
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
"tokio",
|
"tokio",
|
||||||
@ -2974,7 +2899,7 @@ dependencies = [
|
|||||||
"rand 0.9.4",
|
"rand 0.9.4",
|
||||||
"ring",
|
"ring",
|
||||||
"rustc-hash",
|
"rustc-hash",
|
||||||
"rustls 0.23.40",
|
"rustls 0.23.41",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
"slab",
|
"slab",
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
@ -3147,23 +3072,20 @@ dependencies = [
|
|||||||
"http-body-util",
|
"http-body-util",
|
||||||
"hyper",
|
"hyper",
|
||||||
"hyper-rustls",
|
"hyper-rustls",
|
||||||
"hyper-tls",
|
|
||||||
"hyper-util",
|
"hyper-util",
|
||||||
"js-sys",
|
"js-sys",
|
||||||
"log",
|
"log",
|
||||||
"mime_guess",
|
"mime_guess",
|
||||||
"native-tls",
|
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"quinn",
|
"quinn",
|
||||||
"rustls 0.23.40",
|
"rustls 0.23.41",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_urlencoded",
|
"serde_urlencoded",
|
||||||
"sync_wrapper",
|
"sync_wrapper",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-native-tls",
|
|
||||||
"tokio-rustls",
|
"tokio-rustls",
|
||||||
"tokio-util",
|
"tokio-util",
|
||||||
"tower",
|
"tower",
|
||||||
@ -3256,9 +3178,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustls"
|
name = "rustls"
|
||||||
version = "0.23.40"
|
version = "0.23.41"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
|
checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"ring",
|
"ring",
|
||||||
@ -3329,15 +3251,6 @@ dependencies = [
|
|||||||
"winapi-util",
|
"winapi-util",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "schannel"
|
|
||||||
version = "0.1.29"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
|
|
||||||
dependencies = [
|
|
||||||
"windows-sys 0.61.2",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "scopeguard"
|
name = "scopeguard"
|
||||||
version = "1.2.0"
|
version = "1.2.0"
|
||||||
@ -3354,29 +3267,6 @@ dependencies = [
|
|||||||
"untrusted",
|
"untrusted",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "security-framework"
|
|
||||||
version = "3.7.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags 2.13.0",
|
|
||||||
"core-foundation",
|
|
||||||
"core-foundation-sys",
|
|
||||||
"libc",
|
|
||||||
"security-framework-sys",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "security-framework-sys"
|
|
||||||
version = "2.17.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
|
|
||||||
dependencies = [
|
|
||||||
"core-foundation-sys",
|
|
||||||
"libc",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "selectors"
|
name = "selectors"
|
||||||
version = "0.26.0"
|
version = "0.26.0"
|
||||||
@ -4210,23 +4100,13 @@ dependencies = [
|
|||||||
"syn 2.0.117",
|
"syn 2.0.117",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tokio-native-tls"
|
|
||||||
version = "0.3.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
|
|
||||||
dependencies = [
|
|
||||||
"native-tls",
|
|
||||||
"tokio",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio-rustls"
|
name = "tokio-rustls"
|
||||||
version = "0.26.4"
|
version = "0.26.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
|
checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"rustls 0.23.40",
|
"rustls 0.23.41",
|
||||||
"tokio",
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@ -21,11 +21,11 @@ path = "src/bin/cli.rs"
|
|||||||
[dependencies]
|
[dependencies]
|
||||||
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "signal", "time", "fs", "net", "sync", "process"] }
|
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "signal", "time", "fs", "net", "sync", "process"] }
|
||||||
axum = { version = "0.7", features = ["macros", "multipart"] }
|
axum = { version = "0.7", features = ["macros", "multipart"] }
|
||||||
tower-http = { version = "0.5", features = ["cors", "fs", "trace", "set-header"] }
|
tower-http = { version = "0.5", features = ["cors", "fs", "trace", "set-header", "catch-panic"] }
|
||||||
sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "sqlite", "chrono", "json"] }
|
sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "sqlite", "chrono", "json"] }
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "multipart", "cookies", "rustls-tls", "native-tls"] }
|
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "multipart", "cookies", "rustls-tls"] }
|
||||||
dotenvy = "0.15"
|
dotenvy = "0.15"
|
||||||
quick-xml = { version = "0.31", features = ["serialize"] }
|
quick-xml = { version = "0.31", features = ["serialize"] }
|
||||||
anyhow = "1.0"
|
anyhow = "1.0"
|
||||||
@ -62,6 +62,7 @@ glob = "0.3"
|
|||||||
walkdir = "2"
|
walkdir = "2"
|
||||||
lru = "0.12"
|
lru = "0.12"
|
||||||
git2 = { version = "0.18", default-features = false, features = ["vendored-libgit2"] }
|
git2 = { version = "0.18", default-features = false, features = ["vendored-libgit2"] }
|
||||||
|
dashmap = "6"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = []
|
default = []
|
||||||
|
|||||||
@ -2,6 +2,7 @@
|
|||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8" />
|
<meta charset="UTF-8" />
|
||||||
|
<meta name="referrer" content="no-referrer" />
|
||||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||||
<link rel="manifest" href="/manifest.json" />
|
<link rel="manifest" href="/manifest.json" />
|
||||||
<!-- iOS PWA 独立全屏支持 -->
|
<!-- iOS PWA 独立全屏支持 -->
|
||||||
|
|||||||
@ -61,7 +61,13 @@ export default function App() {
|
|||||||
|
|
||||||
// 2. 局部状态定义 (多组件共用或全局网络进度状态)
|
// 2. 局部状态定义 (多组件共用或全局网络进度状态)
|
||||||
const [activeTab, setActiveTab] = useState<
|
const [activeTab, setActiveTab] = useState<
|
||||||
'search' | 'library' | 'reader' | 'citation' | 'sync' | 'observation' | 'agent'
|
| 'search'
|
||||||
|
| 'library'
|
||||||
|
| 'reader'
|
||||||
|
| 'citation'
|
||||||
|
| 'sync'
|
||||||
|
| 'observation'
|
||||||
|
| 'agent'
|
||||||
>(() => {
|
>(() => {
|
||||||
const saved = localStorage.getItem('astro_active_tab');
|
const saved = localStorage.getItem('astro_active_tab');
|
||||||
const validTabs = [
|
const validTabs = [
|
||||||
|
|||||||
@ -642,7 +642,9 @@ export function VizierResultCard({ metadata }: VizierResultCardProps) {
|
|||||||
if (val === null || val === undefined) return '—';
|
if (val === null || val === undefined) return '—';
|
||||||
if (typeof val === 'number') {
|
if (typeof val === 'number') {
|
||||||
// 数字保留合理精度
|
// 数字保留合理精度
|
||||||
return Number.isInteger(val) ? String(val) : val.toFixed(6).replace(/\.?0+$/, '');
|
return Number.isInteger(val)
|
||||||
|
? String(val)
|
||||||
|
: val.toFixed(6).replace(/\.?0+$/, '');
|
||||||
}
|
}
|
||||||
if (typeof val === 'boolean') return val ? 'true' : 'false';
|
if (typeof val === 'boolean') return val ? 'true' : 'false';
|
||||||
return String(val);
|
return String(val);
|
||||||
@ -703,11 +705,17 @@ export function VizierResultCard({ metadata }: VizierResultCardProps) {
|
|||||||
<th
|
<th
|
||||||
key={i}
|
key={i}
|
||||||
className="px-2 py-1.5 text-left font-bold text-slate-600 border-b border-slate-200 whitespace-nowrap"
|
className="px-2 py-1.5 text-left font-bold text-slate-600 border-b border-slate-200 whitespace-nowrap"
|
||||||
title={f.description || f.unit ? `${f.description || ''} ${f.unit ? `[${f.unit}]` : ''}`.trim() : undefined}
|
title={
|
||||||
|
f.description || f.unit
|
||||||
|
? `${f.description || ''} ${f.unit ? `[${f.unit}]` : ''}`.trim()
|
||||||
|
: undefined
|
||||||
|
}
|
||||||
>
|
>
|
||||||
{f.name}
|
{f.name}
|
||||||
{f.unit && (
|
{f.unit && (
|
||||||
<span className="text-slate-400 font-normal ml-1">[{f.unit}]</span>
|
<span className="text-slate-400 font-normal ml-1">
|
||||||
|
[{f.unit}]
|
||||||
|
</span>
|
||||||
)}
|
)}
|
||||||
</th>
|
</th>
|
||||||
))}
|
))}
|
||||||
@ -744,7 +752,6 @@ export function VizierResultCard({ metadata }: VizierResultCardProps) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// ==========================================
|
// ==========================================
|
||||||
// 6. 统一观测数据下载结果卡片 (find_observation)
|
// 6. 统一观测数据下载结果卡片 (find_observation)
|
||||||
// 跨 (LAMOST/Gaia/SDSS/DESI) × (Spectrum/LightCurve/Photometry/Image)
|
// 跨 (LAMOST/Gaia/SDSS/DESI) × (Spectrum/LightCurve/Photometry/Image)
|
||||||
|
|||||||
@ -108,7 +108,12 @@ export function ToolCallCard({
|
|||||||
metadata={
|
metadata={
|
||||||
metadata as {
|
metadata as {
|
||||||
table_name?: string;
|
table_name?: string;
|
||||||
fields: { name: string; description?: string; unit?: string; datatype?: string }[];
|
fields: {
|
||||||
|
name: string;
|
||||||
|
description?: string;
|
||||||
|
unit?: string;
|
||||||
|
datatype?: string;
|
||||||
|
}[];
|
||||||
rows: unknown[][];
|
rows: unknown[][];
|
||||||
row_count: number;
|
row_count: number;
|
||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
|
|||||||
@ -7,12 +7,7 @@
|
|||||||
// 2. (可选)SpecialToolRenderers 的 FindObservationCard 可改为薄封装调用本组件
|
// 2. (可选)SpecialToolRenderers 的 FindObservationCard 可改为薄封装调用本组件
|
||||||
//
|
//
|
||||||
// 视觉与 SpecialToolRenderers::FindObservationCard 保持一致,统一数据源主题色。
|
// 视觉与 SpecialToolRenderers::FindObservationCard 保持一致,统一数据源主题色。
|
||||||
import {
|
import { Activity, CheckCircle2, AlertTriangle, Download } from 'lucide-react';
|
||||||
Activity,
|
|
||||||
CheckCircle2,
|
|
||||||
AlertTriangle,
|
|
||||||
Download,
|
|
||||||
} from 'lucide-react';
|
|
||||||
import {
|
import {
|
||||||
SOURCE_THEME,
|
SOURCE_THEME,
|
||||||
PRODUCT_LABEL,
|
PRODUCT_LABEL,
|
||||||
@ -119,8 +114,7 @@ export function ObservationResultCard({ result }: ObservationResultCardProps) {
|
|||||||
{a.band}
|
{a.band}
|
||||||
</span>
|
</span>
|
||||||
)}
|
)}
|
||||||
{a.file_format.toUpperCase()} ·{' '}
|
{a.file_format.toUpperCase()} · {formatFileSize(a.size_bytes)}
|
||||||
{formatFileSize(a.size_bytes)}
|
|
||||||
</span>
|
</span>
|
||||||
<a
|
<a
|
||||||
href={a.file_url}
|
href={a.file_url}
|
||||||
|
|||||||
@ -12,11 +12,7 @@
|
|||||||
import { useState, useEffect, useCallback, useRef, useMemo } from 'react';
|
import { useState, useEffect, useCallback, useRef, useMemo } from 'react';
|
||||||
import axios from 'axios';
|
import axios from 'axios';
|
||||||
import { extractErrorMessage } from '../utils/apiError';
|
import { extractErrorMessage } from '../utils/apiError';
|
||||||
import type {
|
import type { ObservationRecord, Candidate, CapabilitySpec } from '../types';
|
||||||
ObservationRecord,
|
|
||||||
Candidate,
|
|
||||||
CapabilitySpec,
|
|
||||||
} from '../types';
|
|
||||||
import type { ObservationBatchResult } from '../components/observation/constants';
|
import type { ObservationBatchResult } from '../components/observation/constants';
|
||||||
|
|
||||||
interface UseObservationProps {
|
interface UseObservationProps {
|
||||||
@ -32,6 +28,8 @@ export interface SearchForm {
|
|||||||
dec: string;
|
dec: string;
|
||||||
radius: string;
|
radius: string;
|
||||||
release: string;
|
release: string;
|
||||||
|
/// 数据发布的子版本(仅 LAMOST 有意义);空串表示用该 DR 的默认子版本
|
||||||
|
version: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const EMPTY_SEARCH_FORM: SearchForm = {
|
export const EMPTY_SEARCH_FORM: SearchForm = {
|
||||||
@ -42,6 +40,7 @@ export const EMPTY_SEARCH_FORM: SearchForm = {
|
|||||||
dec: '',
|
dec: '',
|
||||||
radius: '0.1',
|
radius: '0.1',
|
||||||
release: '',
|
release: '',
|
||||||
|
version: '',
|
||||||
};
|
};
|
||||||
|
|
||||||
export function useObservation({ isAuthenticated }: UseObservationProps) {
|
export function useObservation({ isAuthenticated }: UseObservationProps) {
|
||||||
@ -65,7 +64,9 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
|||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (isAuthenticated === true) fetchCapabilities();
|
if (isAuthenticated !== true) return;
|
||||||
|
// eslint-disable-next-line react-hooks/set-state-in-effect -- async fetch, setState in microtask
|
||||||
|
fetchCapabilities();
|
||||||
}, [isAuthenticated, fetchCapabilities]);
|
}, [isAuthenticated, fetchCapabilities]);
|
||||||
|
|
||||||
// ════════════════════════════════════════════════════
|
// ════════════════════════════════════════════════════
|
||||||
@ -88,6 +89,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
|||||||
const valid = currentSpec.releases;
|
const valid = currentSpec.releases;
|
||||||
const hardMax = currentSpec.hard_max_radius_deg;
|
const hardMax = currentSpec.hard_max_radius_deg;
|
||||||
const suggested = currentSpec.suggested_max_radius_deg;
|
const suggested = currentSpec.suggested_max_radius_deg;
|
||||||
|
// eslint-disable-next-line react-hooks/set-state-in-effect -- derived form sync on spec change
|
||||||
setSearchForm((f) => {
|
setSearchForm((f) => {
|
||||||
let next = f;
|
let next = f;
|
||||||
// release 回落(仅在该源区分版本时)
|
// release 回落(仅在该源区分版本时)
|
||||||
@ -103,6 +105,27 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
|||||||
});
|
});
|
||||||
}, [currentSpec]);
|
}, [currentSpec]);
|
||||||
|
|
||||||
|
// 当 release 切换后:若该 release 有子版本列表,则校验当前 version 是否在列表里,
|
||||||
|
// 不在则清空(清空后端用默认子版本)。无子版本概念的源(release_versions 为空)也清空。
|
||||||
|
useEffect(() => {
|
||||||
|
if (!currentSpec) return;
|
||||||
|
const versionsMap = currentSpec.release_versions ?? {};
|
||||||
|
const versions = searchForm.release
|
||||||
|
? versionsMap[searchForm.release]
|
||||||
|
: undefined;
|
||||||
|
// eslint-disable-next-line react-hooks/set-state-in-effect -- derived version reset on release change
|
||||||
|
setSearchForm((f) => {
|
||||||
|
if (!versions || versions.length === 0) {
|
||||||
|
// 该 release 无子版本概念
|
||||||
|
return f.version ? { ...f, version: '' } : f;
|
||||||
|
}
|
||||||
|
if (!f.version || !versions.includes(f.version)) {
|
||||||
|
return { ...f, version: '' }; // 清空 = 用默认(最新公开)子版本
|
||||||
|
}
|
||||||
|
return f;
|
||||||
|
});
|
||||||
|
}, [currentSpec, searchForm.release]);
|
||||||
|
|
||||||
const [searchResults, setSearchResults] = useState<Candidate[]>([]);
|
const [searchResults, setSearchResults] = useState<Candidate[]>([]);
|
||||||
const [searching, setSearching] = useState(false);
|
const [searching, setSearching] = useState(false);
|
||||||
const [searchError, setSearchError] = useState<string | null>(null);
|
const [searchError, setSearchError] = useState<string | null>(null);
|
||||||
@ -131,6 +154,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
|||||||
dec,
|
dec,
|
||||||
radius: Math.max(0.0001, parseFloat(searchForm.radius) || 0.1),
|
radius: Math.max(0.0001, parseFloat(searchForm.radius) || 0.1),
|
||||||
release: searchForm.release || undefined,
|
release: searchForm.release || undefined,
|
||||||
|
version: searchForm.version || undefined,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
setSearchResults(res.data ?? []);
|
setSearchResults(res.data ?? []);
|
||||||
@ -169,13 +193,6 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
|||||||
useState<ObservationBatchResult | null>(null);
|
useState<ObservationBatchResult | null>(null);
|
||||||
const [downloadError, setDownloadError] = useState<string | null>(null);
|
const [downloadError, setDownloadError] = useState<string | null>(null);
|
||||||
|
|
||||||
/** 下载选中的候选源(标识符模式) */
|
|
||||||
const downloadSelected = useCallback(async () => {
|
|
||||||
const ids = Array.from(selectedSourceIds);
|
|
||||||
if (ids.length === 0) return;
|
|
||||||
await downloadByIds(ids);
|
|
||||||
}, [selectedSourceIds]); // eslint-disable-line react-hooks/exhaustive-deps
|
|
||||||
|
|
||||||
/** 按标识符列表下载 */
|
/** 按标识符列表下载 */
|
||||||
const downloadByIds = useCallback(
|
const downloadByIds = useCallback(
|
||||||
async (sourceIds: string[]) => {
|
async (sourceIds: string[]) => {
|
||||||
@ -190,6 +207,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
|||||||
product: searchForm.product,
|
product: searchForm.product,
|
||||||
subtype: searchForm.subtype || undefined,
|
subtype: searchForm.subtype || undefined,
|
||||||
release: searchForm.release || undefined,
|
release: searchForm.release || undefined,
|
||||||
|
version: searchForm.version || undefined,
|
||||||
force: false,
|
force: false,
|
||||||
mode: 'identifiers',
|
mode: 'identifiers',
|
||||||
source_ids: sourceIds,
|
source_ids: sourceIds,
|
||||||
@ -205,6 +223,13 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
|||||||
[searchForm]
|
[searchForm]
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/** 下载选中的候选源(标识符模式) */
|
||||||
|
const downloadSelected = useCallback(async () => {
|
||||||
|
const ids = Array.from(selectedSourceIds);
|
||||||
|
if (ids.length === 0) return;
|
||||||
|
await downloadByIds(ids);
|
||||||
|
}, [selectedSourceIds, downloadByIds]);
|
||||||
|
|
||||||
/** 按坐标下载(cone 检索 + 全部下载) */
|
/** 按坐标下载(cone 检索 + 全部下载) */
|
||||||
const downloadByCoordinates = useCallback(
|
const downloadByCoordinates = useCallback(
|
||||||
async (strategy: 'nearest' | 'all' = 'nearest') => {
|
async (strategy: 'nearest' | 'all' = 'nearest') => {
|
||||||
@ -224,6 +249,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
|||||||
product: searchForm.product,
|
product: searchForm.product,
|
||||||
subtype: searchForm.subtype || undefined,
|
subtype: searchForm.subtype || undefined,
|
||||||
release: searchForm.release || undefined,
|
release: searchForm.release || undefined,
|
||||||
|
version: searchForm.version || undefined,
|
||||||
force: false,
|
force: false,
|
||||||
mode: 'coordinates',
|
mode: 'coordinates',
|
||||||
ra,
|
ra,
|
||||||
@ -265,9 +291,10 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
|||||||
setLibraryLoading(true);
|
setLibraryLoading(true);
|
||||||
setLibraryError(null);
|
setLibraryError(null);
|
||||||
try {
|
try {
|
||||||
const res = await axios.get<{ items: ObservationRecord[]; total: number }>(
|
const res = await axios.get<{
|
||||||
'/api/observation/list',
|
items: ObservationRecord[];
|
||||||
{
|
total: number;
|
||||||
|
}>('/api/observation/list', {
|
||||||
params: {
|
params: {
|
||||||
source: libSource !== 'all' ? libSource : undefined,
|
source: libSource !== 'all' ? libSource : undefined,
|
||||||
product: libProduct !== 'all' ? libProduct : undefined,
|
product: libProduct !== 'all' ? libProduct : undefined,
|
||||||
@ -276,14 +303,11 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
|||||||
limit: libPageSize,
|
limit: libPageSize,
|
||||||
offset: (libPage - 1) * libPageSize,
|
offset: (libPage - 1) * libPageSize,
|
||||||
},
|
},
|
||||||
}
|
});
|
||||||
);
|
|
||||||
setLibraryItems(res.data.items ?? []);
|
setLibraryItems(res.data.items ?? []);
|
||||||
setLibraryTotal(res.data.total ?? 0);
|
setLibraryTotal(res.data.total ?? 0);
|
||||||
} catch (e: unknown) {
|
} catch (e: unknown) {
|
||||||
setLibraryError(
|
setLibraryError(extractErrorMessage(e, '加载缓存库失败,请检查后端连接'));
|
||||||
extractErrorMessage(e, '加载缓存库失败,请检查后端连接')
|
|
||||||
);
|
|
||||||
setLibraryItems([]);
|
setLibraryItems([]);
|
||||||
} finally {
|
} finally {
|
||||||
setLibraryLoading(false);
|
setLibraryLoading(false);
|
||||||
@ -300,6 +324,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
|
|||||||
|
|
||||||
// 任意筛选/分页变更 → 重新请求后端
|
// 任意筛选/分页变更 → 重新请求后端
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
// eslint-disable-next-line react-hooks/set-state-in-effect -- async fetch, setState in microtask
|
||||||
fetchLibrary();
|
fetchLibrary();
|
||||||
}, [fetchLibrary]);
|
}, [fetchLibrary]);
|
||||||
|
|
||||||
|
|||||||
@ -25,6 +25,7 @@ import {
|
|||||||
Database,
|
Database,
|
||||||
Send,
|
Send,
|
||||||
RotateCw,
|
RotateCw,
|
||||||
|
Lock,
|
||||||
} from 'lucide-react';
|
} from 'lucide-react';
|
||||||
import { CustomSelect } from '../components/CustomSelect';
|
import { CustomSelect } from '../components/CustomSelect';
|
||||||
import { ObservationResultCard } from '../components/observation/ObservationResultCard';
|
import { ObservationResultCard } from '../components/observation/ObservationResultCard';
|
||||||
@ -56,11 +57,13 @@ interface ObservationPanelProps {
|
|||||||
selectNone: () => void;
|
selectNone: () => void;
|
||||||
// 下载
|
// 下载
|
||||||
downloading: boolean;
|
downloading: boolean;
|
||||||
downloadResult: import('../components/observation/constants').ObservationBatchResult | null;
|
downloadResult:
|
||||||
|
import('../components/observation/constants').ObservationBatchResult | null;
|
||||||
downloadError: string | null;
|
downloadError: string | null;
|
||||||
setDownloadResult: React.Dispatch<
|
setDownloadResult: React.Dispatch<
|
||||||
React.SetStateAction<
|
React.SetStateAction<
|
||||||
import('../components/observation/constants').ObservationBatchResult | null
|
| import('../components/observation/constants').ObservationBatchResult
|
||||||
|
| null
|
||||||
>
|
>
|
||||||
>;
|
>;
|
||||||
downloadSelected: () => Promise<void>;
|
downloadSelected: () => Promise<void>;
|
||||||
@ -195,8 +198,7 @@ function SearchView({
|
|||||||
|
|
||||||
const subtypeOptions = useMemo(() => {
|
const subtypeOptions = useMemo(() => {
|
||||||
const cap = capabilities.find(
|
const cap = capabilities.find(
|
||||||
(c) =>
|
(c) => c.source === searchForm.source && c.product === searchForm.product
|
||||||
c.source === searchForm.source && c.product === searchForm.product
|
|
||||||
);
|
);
|
||||||
return (cap?.subtypes ?? []).map((s) => ({ value: s, label: s }));
|
return (cap?.subtypes ?? []).map((s) => ({ value: s, label: s }));
|
||||||
}, [capabilities, searchForm.source, searchForm.product]);
|
}, [capabilities, searchForm.source, searchForm.product]);
|
||||||
@ -208,22 +210,42 @@ function SearchView({
|
|||||||
|
|
||||||
// 当前源+产品支持的版本列表(供下拉)
|
// 当前源+产品支持的版本列表(供下拉)
|
||||||
const releaseOptions = useMemo(() => {
|
const releaseOptions = useMemo(() => {
|
||||||
|
const authRequired = new Set(currentSpec?.releases_requiring_auth ?? []);
|
||||||
return (currentSpec?.releases ?? []).map((r) => ({
|
return (currentSpec?.releases ?? []).map((r) => ({
|
||||||
value: r,
|
value: r,
|
||||||
label: r.toUpperCase(),
|
label: authRequired.has(r)
|
||||||
|
? `${r.toUpperCase()} · 需登录`
|
||||||
|
: r.toUpperCase(),
|
||||||
}));
|
}));
|
||||||
}, [currentSpec]);
|
}, [currentSpec]);
|
||||||
|
|
||||||
// LAMOST MRS 联动约束:MRS(中分辨率)从 DR7 起才支持,DR5/DR6 无 MRS 数据
|
// 当前选中的 release 是否需要登录(Internal DR)
|
||||||
const lamostMrsBlocked = useMemo(() => {
|
const currentReleaseNeedsAuth = useMemo(() => {
|
||||||
if (searchForm.source !== 'lamost' || searchForm.subtype !== 'mrs')
|
if (!searchForm.release || !currentSpec) return false;
|
||||||
return null;
|
return (currentSpec.releases_requiring_auth ?? []).includes(
|
||||||
const rel = searchForm.release;
|
searchForm.release
|
||||||
if (rel === 'dr5' || rel === 'dr6') {
|
);
|
||||||
return 'LAMOST 中分辨率(MRS)从 DR7 起才支持,当前版本无 MRS 数据';
|
}, [searchForm.release, currentSpec]);
|
||||||
|
|
||||||
|
// 交叉约束:当前 release 是否支持当前 subtype(从 capabilities 派生,不硬编码)
|
||||||
|
const subtypeBlocked = useMemo(() => {
|
||||||
|
if (!currentSpec || !searchForm.release || !searchForm.subtype) return null;
|
||||||
|
const map = currentSpec.subtypes_per_release;
|
||||||
|
if (!map || Object.keys(map).length === 0) return null; // 无交叉约束
|
||||||
|
const valid = map[searchForm.release];
|
||||||
|
if (valid && !valid.includes(searchForm.subtype)) {
|
||||||
|
return `'${searchForm.release.toUpperCase()}' 不支持子类型 '${searchForm.subtype}'`;
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}, [searchForm.source, searchForm.subtype, searchForm.release]);
|
}, [currentSpec, searchForm.release, searchForm.subtype]);
|
||||||
|
|
||||||
|
// 当前 release 对应的子版本列表(仅 LAMOST 有内容)
|
||||||
|
const versionOptions = useMemo(() => {
|
||||||
|
if (!currentSpec || !searchForm.release) return [];
|
||||||
|
const versionsMap = currentSpec.release_versions ?? {};
|
||||||
|
const versions = versionsMap[searchForm.release] ?? [];
|
||||||
|
return versions.map((v) => ({ value: v, label: v.toUpperCase() }));
|
||||||
|
}, [currentSpec, searchForm.release]);
|
||||||
|
|
||||||
const handleSubmitIds = () => {
|
const handleSubmitIds = () => {
|
||||||
const ids = idsText
|
const ids = idsText
|
||||||
@ -275,7 +297,7 @@ function SearchView({
|
|||||||
.map((c) => c.product);
|
.map((c) => c.product);
|
||||||
const safeProduct = newProducts.includes(f.product)
|
const safeProduct = newProducts.includes(f.product)
|
||||||
? f.product
|
? f.product
|
||||||
: newProducts[0] ?? f.product;
|
: (newProducts[0] ?? f.product);
|
||||||
return { ...f, source: v, product: safeProduct, subtype: '' };
|
return { ...f, source: v, product: safeProduct, subtype: '' };
|
||||||
});
|
});
|
||||||
}}
|
}}
|
||||||
@ -331,27 +353,67 @@ function SearchView({
|
|||||||
}
|
}
|
||||||
className="w-full"
|
className="w-full"
|
||||||
options={[
|
options={[
|
||||||
{ value: '__none__', label: `默认${currentSpec?.default_release ? `(${currentSpec.default_release.toUpperCase()})` : ''}` },
|
{
|
||||||
|
value: '__none__',
|
||||||
|
label: `默认${currentSpec?.default_release ? `(${currentSpec.default_release.toUpperCase()})` : ''}`,
|
||||||
|
},
|
||||||
...releaseOptions,
|
...releaseOptions,
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
</Field>
|
</Field>
|
||||||
|
{/* 子版本(仅 LAMOST 有内容;其他源该 Field 不渲染) */}
|
||||||
|
{versionOptions.length > 0 && (
|
||||||
|
<Field label="子版本">
|
||||||
|
<CustomSelect
|
||||||
|
value={searchForm.version || '__none__'}
|
||||||
|
onChange={(v) =>
|
||||||
|
setSearchForm((f) => ({
|
||||||
|
...f,
|
||||||
|
version: v === '__none__' ? '' : v,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
className="w-full"
|
||||||
|
options={[
|
||||||
|
{ value: '__none__', label: '默认(最新公开)' },
|
||||||
|
...versionOptions,
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
</Field>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* LAMOST MRS 版本约束提示 */}
|
{/* Internal DR 需登录提示 */}
|
||||||
{lamostMrsBlocked && (
|
{currentReleaseNeedsAuth && (
|
||||||
|
<div className="flex items-center gap-2 text-xs text-rose-700 bg-rose-50 border border-rose-200 rounded-md p-2.5">
|
||||||
|
<Lock className="w-3.5 h-3.5 shrink-0" />
|
||||||
|
<span>
|
||||||
|
{searchForm.release.toUpperCase()} 当前为 Internal 数据发布,需经
|
||||||
|
oauth.china-vo.org 登录认证后才能访问,公开检索暂不可用
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* 交叉约束提示:当前 release 不支持该 subtype */}
|
||||||
|
{subtypeBlocked && (
|
||||||
<div className="flex items-center gap-2 text-xs text-amber-700 bg-amber-50 border border-amber-200 rounded-md p-2.5">
|
<div className="flex items-center gap-2 text-xs text-amber-700 bg-amber-50 border border-amber-200 rounded-md p-2.5">
|
||||||
<AlertTriangle className="w-3.5 h-3.5 shrink-0" />
|
<AlertTriangle className="w-3.5 h-3.5 shrink-0" />
|
||||||
<span>{lamostMrsBlocked}</span>
|
<span>{subtypeBlocked}</span>
|
||||||
|
{currentSpec?.subtypes && currentSpec.subtypes.length > 0 && (
|
||||||
<button
|
<button
|
||||||
onClick={() =>
|
onClick={() => {
|
||||||
setSearchForm((f) => ({ ...f, subtype: 'lrs' }))
|
// 切换到该 release 支持的首个 subtype
|
||||||
}
|
const map = currentSpec?.subtypes_per_release;
|
||||||
|
const valid =
|
||||||
|
(map && searchForm.release && map[searchForm.release]) ||
|
||||||
|
currentSpec?.subtypes;
|
||||||
|
setSearchForm((f) => ({ ...f, subtype: valid?.[0] ?? '' }));
|
||||||
|
}}
|
||||||
className="ml-auto px-2 py-0.5 rounded bg-amber-100 hover:bg-amber-200 text-amber-800 font-bold text-[10px]"
|
className="ml-auto px-2 py-0.5 rounded bg-amber-100 hover:bg-amber-200 text-amber-800 font-bold text-[10px]"
|
||||||
>
|
>
|
||||||
切换为 LRS
|
切换可用子类型
|
||||||
</button>
|
</button>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
@ -410,14 +472,15 @@ function SearchView({
|
|||||||
<AlertTriangle className="w-3.5 h-3.5 shrink-0" />
|
<AlertTriangle className="w-3.5 h-3.5 shrink-0" />
|
||||||
<span>
|
<span>
|
||||||
当前半径 {searchForm.radius}° 超出建议值(≤
|
当前半径 {searchForm.radius}° 超出建议值(≤
|
||||||
{currentSpec.suggested_max_radius_deg}°),大范围查询可能因主表过大被服务端超时拒绝
|
{currentSpec.suggested_max_radius_deg}
|
||||||
|
°),大范围查询可能因主表过大被服务端超时拒绝
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
<div className="flex items-center gap-2 pt-1">
|
<div className="flex items-center gap-2 pt-1">
|
||||||
<button
|
<button
|
||||||
onClick={runSearch}
|
onClick={runSearch}
|
||||||
disabled={searching}
|
disabled={searching || currentReleaseNeedsAuth}
|
||||||
className="btn-console btn-console-primary px-4 py-2 rounded-md text-xs font-bold flex items-center gap-2 disabled:opacity-50"
|
className="btn-console btn-console-primary px-4 py-2 rounded-md text-xs font-bold flex items-center gap-2 disabled:opacity-50"
|
||||||
>
|
>
|
||||||
{searching ? (
|
{searching ? (
|
||||||
@ -432,7 +495,7 @@ function SearchView({
|
|||||||
</span>
|
</span>
|
||||||
<button
|
<button
|
||||||
onClick={() => downloadByCoordinates('nearest')}
|
onClick={() => downloadByCoordinates('nearest')}
|
||||||
disabled={downloading}
|
disabled={downloading || currentReleaseNeedsAuth}
|
||||||
className="ml-auto px-3 py-2 rounded-md text-xs font-bold flex items-center gap-1.5 bg-emerald-50 text-emerald-700 border border-emerald-200 hover:bg-emerald-100 disabled:opacity-50"
|
className="ml-auto px-3 py-2 rounded-md text-xs font-bold flex items-center gap-1.5 bg-emerald-50 text-emerald-700 border border-emerald-200 hover:bg-emerald-100 disabled:opacity-50"
|
||||||
>
|
>
|
||||||
<Download className="w-3.5 h-3.5" />
|
<Download className="w-3.5 h-3.5" />
|
||||||
@ -786,7 +849,10 @@ function LibraryView({
|
|||||||
<>
|
<>
|
||||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-3">
|
<div className="grid grid-cols-1 md:grid-cols-2 gap-3">
|
||||||
{libraryItems.map((rec) => (
|
{libraryItems.map((rec) => (
|
||||||
<LibraryCard key={`${rec.source}|${rec.product}|${rec.source_id}`} rec={rec} />
|
<LibraryCard
|
||||||
|
key={`${rec.source}|${rec.product}|${rec.source_id}`}
|
||||||
|
rec={rec}
|
||||||
|
/>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@ -863,7 +929,11 @@ function Field({
|
|||||||
compact?: boolean;
|
compact?: boolean;
|
||||||
}) {
|
}) {
|
||||||
return (
|
return (
|
||||||
<div className={compact ? 'w-full sm:w-36 space-y-1.5 font-bold' : 'space-y-1.5'}>
|
<div
|
||||||
|
className={
|
||||||
|
compact ? 'w-full sm:w-36 space-y-1.5 font-bold' : 'space-y-1.5'
|
||||||
|
}
|
||||||
|
>
|
||||||
<label className="block text-slate-500 font-bold text-xs">{label}</label>
|
<label className="block text-slate-500 font-bold text-xs">{label}</label>
|
||||||
{children}
|
{children}
|
||||||
</div>
|
</div>
|
||||||
@ -894,7 +964,12 @@ function LibraryCard({ rec }: { rec: ObservationRecord }) {
|
|||||||
const theme = SOURCE_THEME[rec.source] ?? SOURCE_THEME.lamost;
|
const theme = SOURCE_THEME[rec.source] ?? SOURCE_THEME.lamost;
|
||||||
const productLabel = PRODUCT_LABEL[rec.product] ?? rec.product;
|
const productLabel = PRODUCT_LABEL[rec.product] ?? rec.product;
|
||||||
|
|
||||||
let artifacts: Array<{ path: string; format: string; size?: number; band?: string }> = [];
|
let artifacts: Array<{
|
||||||
|
path: string;
|
||||||
|
format: string;
|
||||||
|
size?: number;
|
||||||
|
band?: string;
|
||||||
|
}> = [];
|
||||||
try {
|
try {
|
||||||
const parsed = JSON.parse(rec.artifacts_json);
|
const parsed = JSON.parse(rec.artifacts_json);
|
||||||
if (Array.isArray(parsed)) artifacts = parsed;
|
if (Array.isArray(parsed)) artifacts = parsed;
|
||||||
@ -910,7 +985,9 @@ function LibraryCard({ rec }: { rec: ObservationRecord }) {
|
|||||||
<div className="bg-white border border-slate-200 rounded-lg p-3.5 shadow-xs hover:shadow-md hover:border-slate-300 transition-all">
|
<div className="bg-white border border-slate-200 rounded-lg p-3.5 shadow-xs hover:shadow-md hover:border-slate-300 transition-all">
|
||||||
<div className="flex items-center justify-between mb-2 pb-2 border-b border-slate-100">
|
<div className="flex items-center justify-between mb-2 pb-2 border-b border-slate-100">
|
||||||
<div className="flex items-center gap-1.5">
|
<div className="flex items-center gap-1.5">
|
||||||
<span className={`px-2 py-0.5 rounded text-[10px] font-bold ${theme.badge}`}>
|
<span
|
||||||
|
className={`px-2 py-0.5 rounded text-[10px] font-bold ${theme.badge}`}
|
||||||
|
>
|
||||||
{theme.label}
|
{theme.label}
|
||||||
</span>
|
</span>
|
||||||
<span className="px-2 py-0.5 rounded text-[10px] font-bold bg-slate-200 text-slate-700">
|
<span className="px-2 py-0.5 rounded text-[10px] font-bold bg-slate-200 text-slate-700">
|
||||||
@ -927,7 +1004,10 @@ function LibraryCard({ rec }: { rec: ObservationRecord }) {
|
|||||||
<div className="text-[9px] font-bold text-slate-400 tracking-widest uppercase">
|
<div className="text-[9px] font-bold text-slate-400 tracking-widest uppercase">
|
||||||
Source ID
|
Source ID
|
||||||
</div>
|
</div>
|
||||||
<div className="font-mono text-xs text-slate-800 font-semibold truncate" title={rec.source_id}>
|
<div
|
||||||
|
className="font-mono text-xs text-slate-800 font-semibold truncate"
|
||||||
|
title={rec.source_id}
|
||||||
|
>
|
||||||
{rec.source_id}
|
{rec.source_id}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@ -250,6 +250,12 @@ export interface CapabilitySpec {
|
|||||||
subtypes: string[];
|
subtypes: string[];
|
||||||
releases: string[];
|
releases: string[];
|
||||||
default_release?: string | null;
|
default_release?: string | null;
|
||||||
|
/// 每个 release 对应的子版本列表(仅 LAMOST 有内容,如 {dr11: ["v0","v1.0","v1.1","v2.0"]})
|
||||||
|
release_versions?: Record<string, string[]>;
|
||||||
|
/// 需要登录认证才能访问的 release 列表(如 LAMOST Internal DR12-14),空表示全部公开
|
||||||
|
releases_requiring_auth?: string[];
|
||||||
|
/// 每个 release 实际支持的 subtype(交叉约束,如 LAMOST DR5/6 无 MRS);空表示全部支持
|
||||||
|
subtypes_per_release?: Record<string, string[]>;
|
||||||
suggested_max_radius_deg: number;
|
suggested_max_radius_deg: number;
|
||||||
hard_max_radius_deg: number;
|
hard_max_radius_deg: number;
|
||||||
identifier_format?: string | null;
|
identifier_format?: string | null;
|
||||||
|
|||||||
@ -16,10 +16,7 @@ interface AxiosLikeError {
|
|||||||
///
|
///
|
||||||
/// 优先级:response.data 为字符串 → 对象里的 error/message/detail/description
|
/// 优先级:response.data 为字符串 → 对象里的 error/message/detail/description
|
||||||
/// → 整体 JSON 序列化兜底 → axios message → fallback
|
/// → 整体 JSON 序列化兜底 → axios message → fallback
|
||||||
export function extractErrorMessage(
|
export function extractErrorMessage(e: unknown, fallback: string): string {
|
||||||
e: unknown,
|
|
||||||
fallback: string
|
|
||||||
): string {
|
|
||||||
const axiosError = e as AxiosLikeError;
|
const axiosError = e as AxiosLikeError;
|
||||||
const data = axiosError?.response?.data;
|
const data = axiosError?.response?.data;
|
||||||
if (typeof data === 'string') return data;
|
if (typeof data === 'string') return data;
|
||||||
|
|||||||
86
deploy.sh
Executable file
86
deploy.sh
Executable file
@ -0,0 +1,86 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# deploy.sh — AstroResearch 一键本地打包并推送服务器脚本 (Compose 部署版)
|
||||||
|
|
||||||
|
# 退出脚本如果发生错误
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# 配置参数
|
||||||
|
# =============================================================================
|
||||||
|
REMOTE_USER="fmq"
|
||||||
|
REMOTE_IP="100.66.1.2"
|
||||||
|
REMOTE_PORT="22"
|
||||||
|
REMOTE_DIR="/vol1/1000/program/astroresearch"
|
||||||
|
IMAGE_NAME="astroresearch"
|
||||||
|
TAG="latest"
|
||||||
|
TAR_NAME="astroresearch.tar.gz"
|
||||||
|
|
||||||
|
echo "=== 1. 开始在本地构建 Docker 镜像 (基于 Dockerfile.modeB) ==="
|
||||||
|
docker build -f Dockerfile.modeB -t ${IMAGE_NAME}:${TAG} .
|
||||||
|
|
||||||
|
echo "=== 2. 将镜像导出并压缩为本地临时文件 ${TAR_NAME} ==="
|
||||||
|
docker save ${IMAGE_NAME}:${TAG} | gzip > ${TAR_NAME}
|
||||||
|
|
||||||
|
echo "=== 3. 上传镜像包到服务器目标目录 ==="
|
||||||
|
scp -P ${REMOTE_PORT} ${TAR_NAME} ${REMOTE_USER}@${REMOTE_IP}:${REMOTE_DIR}/
|
||||||
|
|
||||||
|
echo "=== 4. 连接服务器:载入镜像并使用 Docker Compose 启动 ==="
|
||||||
|
# 使用 'EOF' 锁死环境,防止本地变量混淆远程环境变量
|
||||||
|
ssh -p ${REMOTE_PORT} ${REMOTE_USER}@${REMOTE_IP} << 'EOF'
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# 远程运行参数
|
||||||
|
REMOTE_DIR="/vol1/1000/program/astroresearch"
|
||||||
|
IMAGE_NAME="astroresearch"
|
||||||
|
TAG="latest"
|
||||||
|
TAR_NAME="astroresearch.tar.gz"
|
||||||
|
|
||||||
|
echo " -> 正在导入 Docker 镜像..."
|
||||||
|
docker load < ${REMOTE_DIR}/${TAR_NAME}
|
||||||
|
|
||||||
|
echo " -> 清理服务器上的临时镜像压缩包..."
|
||||||
|
rm -f ${REMOTE_DIR}/${TAR_NAME}
|
||||||
|
|
||||||
|
echo " -> 在服务器上写入/更新生产环境的 docker-compose.yml..."
|
||||||
|
cat > ${REMOTE_DIR}/docker-compose.yml << 'INNER_EOF'
|
||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
astroresearch:
|
||||||
|
image: astroresearch:latest
|
||||||
|
container_name: astroresearch
|
||||||
|
restart: always
|
||||||
|
ports:
|
||||||
|
- "8001:8000"
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
environment:
|
||||||
|
- DATABASE_URL=sqlite:///app/library/astro_research.db
|
||||||
|
- LIBRARY_DIR=/app/library
|
||||||
|
- SKILLS_DIR=/app/skills
|
||||||
|
- LOG_DIR=/app/logs
|
||||||
|
- PORT=8000
|
||||||
|
volumes:
|
||||||
|
- ./library:/app/library
|
||||||
|
- ./logs:/app/logs
|
||||||
|
- ./skills:/app/skills:ro
|
||||||
|
INNER_EOF
|
||||||
|
|
||||||
|
# 进入远程部署目录
|
||||||
|
cd ${REMOTE_DIR}
|
||||||
|
|
||||||
|
echo " -> 停止并移除已有的旧服务容器 (docker compose down)..."
|
||||||
|
docker compose down 2>/dev/null || true
|
||||||
|
|
||||||
|
echo " -> 使用 Docker Compose 启动新服务 (docker compose up -d)..."
|
||||||
|
docker compose up -d
|
||||||
|
|
||||||
|
echo " -> 部署成功!正在等待 3 秒检查服务运行状态..."
|
||||||
|
sleep 3
|
||||||
|
docker compose ps
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo "=== 5. 清理本地临时压缩包 ==="
|
||||||
|
rm -f ${TAR_NAME}
|
||||||
|
|
||||||
|
echo "🎉 所有部署操作已圆满完成!服务已在服务器上通过 Docker Compose 后台运行。"
|
||||||
@ -25,7 +25,7 @@ open http://localhost:8000
|
|||||||
### 两种镜像模式对比
|
### 两种镜像模式对比
|
||||||
|
|
||||||
| | Mode A (`Dockerfile`) | Mode B (`Dockerfile.modeB`) |
|
| | Mode A (`Dockerfile`) | Mode B (`Dockerfile.modeB`) |
|
||||||
|---|---|---|
|
| --------------------- | ----------------------- | ------------------------------------------------ |
|
||||||
| **镜像大小** | ~23 MB | ~87 MB |
|
| **镜像大小** | ~23 MB | ~87 MB |
|
||||||
| **基础镜像** | `alpine:3.21` | `distroless/cc-debian12:nonroot` |
|
| **基础镜像** | `alpine:3.21` | `distroless/cc-debian12:nonroot` |
|
||||||
| **libc** | musl (静态链接) | glibc (动态链接) |
|
| **libc** | musl (静态链接) | glibc (动态链接) |
|
||||||
@ -44,11 +44,13 @@ open http://localhost:8000
|
|||||||
镜像最小(23 MB),Obscura 作为外部二进制通过 bind mount 注入,可独立更新。
|
镜像最小(23 MB),Obscura 作为外部二进制通过 bind mount 注入,可独立更新。
|
||||||
|
|
||||||
**构建:**
|
**构建:**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker build -t astroresearch:latest .
|
docker build -t astroresearch:latest .
|
||||||
```
|
```
|
||||||
|
|
||||||
**运行:**
|
**运行:**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker run -d --name astro -p 8000:8000 --env-file .env \
|
docker run -d --name astro -p 8000:8000 --env-file .env \
|
||||||
-v ./library:/app/library \
|
-v ./library:/app/library \
|
||||||
@ -61,6 +63,7 @@ docker run -d --name astro -p 8000:8000 --env-file .env \
|
|||||||
> `bin/` 目录需包含编译好的 Obscura 二进制文件(`obscura` 和 `obscura-worker`)。
|
> `bin/` 目录需包含编译好的 Obscura 二进制文件(`obscura` 和 `obscura-worker`)。
|
||||||
|
|
||||||
**docker-compose.yml(已内置):**
|
**docker-compose.yml(已内置):**
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
services:
|
services:
|
||||||
astroresearch:
|
astroresearch:
|
||||||
@ -90,17 +93,20 @@ services:
|
|||||||
Obscura (V8 + BoringSSL) 编译在二进制内,单容器零外部二进制依赖。适合对外交付的一键部署包。
|
Obscura (V8 + BoringSSL) 编译在二进制内,单容器零外部二进制依赖。适合对外交付的一键部署包。
|
||||||
|
|
||||||
**前置条件:**
|
**前置条件:**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
mkdir -p libs
|
mkdir -p libs
|
||||||
git clone https://github.com/h4ckf0r0day/obscura libs/obscura
|
git clone https://github.com/h4ckf0r0day/obscura libs/obscura
|
||||||
```
|
```
|
||||||
|
|
||||||
**构建:**
|
**构建:**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker build -f Dockerfile.modeB -t astroresearch-all:latest .
|
docker build -f Dockerfile.modeB -t astroresearch-all:latest .
|
||||||
```
|
```
|
||||||
|
|
||||||
**运行:**
|
**运行:**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker run -d --name astro -p 8000:8000 --env-file .env \
|
docker run -d --name astro -p 8000:8000 --env-file .env \
|
||||||
-v ./library:/app/library \
|
-v ./library:/app/library \
|
||||||
@ -124,7 +130,7 @@ docker build --build-arg USE_MIRRORS=0 -t astroresearch:latest .
|
|||||||
**加速源:**
|
**加速源:**
|
||||||
|
|
||||||
| 工具 | 镜像 |
|
| 工具 | 镜像 |
|
||||||
|------|------|
|
| ---------- | -------------------------- |
|
||||||
| npm | `registry.npmmirror.com` |
|
| npm | `registry.npmmirror.com` |
|
||||||
| Alpine apk | `mirrors.aliyun.com` |
|
| Alpine apk | `mirrors.aliyun.com` |
|
||||||
| Debian apt | `mirrors.ustc.edu.cn` |
|
| Debian apt | `mirrors.ustc.edu.cn` |
|
||||||
@ -134,7 +140,7 @@ docker build --build-arg USE_MIRRORS=0 -t astroresearch:latest .
|
|||||||
### 持久化数据卷
|
### 持久化数据卷
|
||||||
|
|
||||||
| 容器路径 | 说明 | 推荐权限 |
|
| 容器路径 | 说明 | 推荐权限 |
|
||||||
|----------|------|---------|
|
| ---------------- | ----------------------------------------- | -------------- |
|
||||||
| `/app/library` | SQLite 数据库 + PDF/HTML 全文 | 读写 |
|
| `/app/library` | SQLite 数据库 + PDF/HTML 全文 | 读写 |
|
||||||
| `/app/logs` | 应用日志(仅`LOG_OUTPUTS=file` 时写入) | 读写 |
|
| `/app/logs` | 应用日志(仅`LOG_OUTPUTS=file` 时写入) | 读写 |
|
||||||
| `/app/skills` | Agent 技能文件 (SKILL.md) | 只读 (`:ro`) |
|
| `/app/skills` | Agent 技能文件 (SKILL.md) | 只读 (`:ro`) |
|
||||||
@ -157,21 +163,26 @@ docker build --build-arg USE_MIRRORS=0 -t astroresearch:latest .
|
|||||||
### 构建步骤
|
### 构建步骤
|
||||||
|
|
||||||
**步骤 1:构建前端**
|
**步骤 1:构建前端**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd dashboard
|
cd dashboard
|
||||||
npm install
|
npm install
|
||||||
npm run build
|
npm run build
|
||||||
```
|
```
|
||||||
|
|
||||||
产物位于 `dashboard/dist/`。
|
产物位于 `dashboard/dist/`。
|
||||||
|
|
||||||
**步骤 2:编译后端**
|
**步骤 2:编译后端**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd ..
|
cd ..
|
||||||
cargo build --release
|
cargo build --release
|
||||||
```
|
```
|
||||||
|
|
||||||
产物位于 `target/release/astroresearch`。
|
产物位于 `target/release/astroresearch`。
|
||||||
|
|
||||||
**步骤 3(可选):健康检查工具**
|
**步骤 3(可选):健康检查工具**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cargo build --release --bin health_check
|
cargo build --release --bin health_check
|
||||||
```
|
```
|
||||||
@ -195,11 +206,12 @@ cp .env.example .env # 编辑填入 API Key
|
|||||||
Obscura 作为独立二进制运行,与主进程隔离。适合常规生产环境。
|
Obscura 作为独立二进制运行,与主进程隔离。适合常规生产环境。
|
||||||
|
|
||||||
| 部署方式 | 步骤 |
|
| 部署方式 | 步骤 |
|
||||||
|----------|------|
|
| ---------------- | ----------------------------------------------------- |
|
||||||
| **Docker** | `docker compose up -d`(自动 bind mount `./bin`) |
|
| **Docker** | `docker compose up -d`(自动 bind mount `./bin`) |
|
||||||
| **源码** | 下载 obscura 到`bin/`,赋予执行权限后启动 |
|
| **源码** | 下载 obscura 到`bin/`,赋予执行权限后启动 |
|
||||||
|
|
||||||
二进制安装(源码部署时):
|
二进制安装(源码部署时):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
mkdir -p bin/
|
mkdir -p bin/
|
||||||
# 下载 obscura 和 obscura-worker 到 bin/,赋予执行权限
|
# 下载 obscura 和 obscura-worker 到 bin/,赋予执行权限
|
||||||
@ -211,7 +223,7 @@ chmod +x bin/obscura bin/obscura-worker
|
|||||||
Obscura (V8 + BoringSSL) 编译进二进制,单文件零外部依赖。
|
Obscura (V8 + BoringSSL) 编译进二进制,单文件零外部依赖。
|
||||||
|
|
||||||
| 部署方式 | 构建命令 |
|
| 部署方式 | 构建命令 |
|
||||||
|----------|---------|
|
| ---------------- | ------------------------------------------------------------------ |
|
||||||
| **Docker** | `docker build -f Dockerfile.modeB -t astroresearch-all:latest .` |
|
| **Docker** | `docker build -f Dockerfile.modeB -t astroresearch-all:latest .` |
|
||||||
| **源码** | `cargo build --release --features obscura-inprocess` |
|
| **源码** | `cargo build --release --features obscura-inprocess` |
|
||||||
|
|
||||||
@ -228,7 +240,7 @@ Obscura (V8 + BoringSSL) 编译进二进制,单文件零外部依赖。
|
|||||||
**优化指标 (Mode A, release-min):**
|
**优化指标 (Mode A, release-min):**
|
||||||
|
|
||||||
| 指标 | release | release-min | 降幅 |
|
| 指标 | release | release-min | 降幅 |
|
||||||
|------|---------|-------------|------|
|
| --------------- | ------- | ----------- | ---- |
|
||||||
| 二进制大小 | 17.0 MB | 8.3 MB | 51% |
|
| 二进制大小 | 17.0 MB | 8.3 MB | 51% |
|
||||||
| 启动 RSS | 34.8 MB | 32.9 MB | 5% |
|
| 启动 RSS | 34.8 MB | 32.9 MB | 5% |
|
||||||
| 虚拟内存 (VSZ) | 1.27 GB | 302 MB | 76% |
|
| 虚拟内存 (VSZ) | 1.27 GB | 302 MB | 76% |
|
||||||
@ -243,6 +255,7 @@ docker build -t astroresearch:latest .
|
|||||||
```
|
```
|
||||||
|
|
||||||
**限制 Tokio 线程数:**
|
**限制 Tokio 线程数:**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
PORT=8000 TOKIO_WORKER_THREADS=1 ./astroresearch
|
PORT=8000 TOKIO_WORKER_THREADS=1 ./astroresearch
|
||||||
```
|
```
|
||||||
@ -252,7 +265,7 @@ PORT=8000 TOKIO_WORKER_THREADS=1 ./astroresearch
|
|||||||
## 5. 环境变量
|
## 5. 环境变量
|
||||||
|
|
||||||
| 变量名 | 必需 | 默认值 | 说明 |
|
| 变量名 | 必需 | 默认值 | 说明 |
|
||||||
| :--- | :--- | :--- | :--- |
|
| :-------------------------- | :--- | :------------------------------------------ | :------------------------------- |
|
||||||
| `DATABASE_URL` | 否 | `sqlite:///app/library/astro_research.db` | SQLite 连接 URL |
|
| `DATABASE_URL` | 否 | `sqlite:///app/library/astro_research.db` | SQLite 连接 URL |
|
||||||
| `ADS_API_KEY` | 是 | - | NASA ADS API Token |
|
| `ADS_API_KEY` | 是 | - | NASA ADS API Token |
|
||||||
| `LLM_API_KEY` | 是 | - | LLM API Key |
|
| `LLM_API_KEY` | 是 | - | LLM API Key |
|
||||||
@ -277,11 +290,13 @@ PORT=8000 TOKIO_WORKER_THREADS=1 ./astroresearch
|
|||||||
## 6. 健康检查与维护
|
## 6. 健康检查与维护
|
||||||
|
|
||||||
**Docker 健康检查(Mode A):**
|
**Docker 健康检查(Mode A):**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker ps --filter “health=healthy” --filter “name=astroresearch”
|
docker ps --filter “health=healthy” --filter “name=astroresearch”
|
||||||
```
|
```
|
||||||
|
|
||||||
Mode B (distroless) 无内置 healthcheck,可在编排层配置:
|
Mode B (distroless) 无内置 healthcheck,可在编排层配置:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
# docker-compose 或 k8s
|
# docker-compose 或 k8s
|
||||||
healthcheck:
|
healthcheck:
|
||||||
@ -289,6 +304,7 @@ healthcheck:
|
|||||||
```
|
```
|
||||||
|
|
||||||
**源码部署健康检查:**
|
**源码部署健康检查:**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 只读扫描
|
# 只读扫描
|
||||||
./health_check
|
./health_check
|
||||||
|
|||||||
@ -250,7 +250,7 @@ pub fn rough_estimate_tokens(messages: &[ChatMessage]) -> usize {
|
|||||||
messages
|
messages
|
||||||
.iter()
|
.iter()
|
||||||
.map(|m| {
|
.map(|m| {
|
||||||
let content_len = m.text().map_or(0, |c| estimate_tokens_from_text(c));
|
let content_len = m.text().map_or(0, estimate_tokens_from_text);
|
||||||
content_len + 4 // 消息结构 overhead 约 4 token
|
content_len + 4 // 消息结构 overhead 约 4 token
|
||||||
})
|
})
|
||||||
.sum()
|
.sum()
|
||||||
@ -697,7 +697,6 @@ fn spawn_memory_extraction(
|
|||||||
memory_manager: std::sync::Arc<tokio::sync::Mutex<crate::agent::memory::MemoryManager>>,
|
memory_manager: std::sync::Arc<tokio::sync::Mutex<crate::agent::memory::MemoryManager>>,
|
||||||
app_state: std::sync::Arc<crate::api::AppState>,
|
app_state: std::sync::Arc<crate::api::AppState>,
|
||||||
) {
|
) {
|
||||||
|
|
||||||
let sid = session_id.to_string();
|
let sid = session_id.to_string();
|
||||||
let mem_mgr = memory_manager.clone();
|
let mem_mgr = memory_manager.clone();
|
||||||
let app = app_state.clone();
|
let app = app_state.clone();
|
||||||
|
|||||||
@ -2,7 +2,6 @@
|
|||||||
//
|
//
|
||||||
// 内置 Hooks — CancellationHook、MetricsHook、AuditLogHook。
|
// 内置 Hooks — CancellationHook、MetricsHook、AuditLogHook。
|
||||||
|
|
||||||
use std::collections::HashSet;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use tokio::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
|
|
||||||
@ -48,11 +47,11 @@ impl ContextDeduplicator {
|
|||||||
|
|
||||||
/// 取消检查 Hook — 在每次工具执行前检查用户是否中止了会话。
|
/// 取消检查 Hook — 在每次工具执行前检查用户是否中止了会话。
|
||||||
pub struct CancellationHook {
|
pub struct CancellationHook {
|
||||||
cancelled_runs: Arc<Mutex<HashSet<String>>>,
|
cancelled_runs: Arc<dashmap::DashMap<String, ()>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CancellationHook {
|
impl CancellationHook {
|
||||||
pub fn new(cancelled_runs: Arc<Mutex<HashSet<String>>>) -> Self {
|
pub fn new(cancelled_runs: Arc<dashmap::DashMap<String, ()>>) -> Self {
|
||||||
CancellationHook { cancelled_runs }
|
CancellationHook { cancelled_runs }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -68,8 +67,7 @@ impl AgentHook for CancellationHook {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn pre_tool_use(&self, ctx: &PreToolUseContext) -> PreToolUseAction {
|
async fn pre_tool_use(&self, ctx: &PreToolUseContext) -> PreToolUseAction {
|
||||||
let runs = self.cancelled_runs.lock().await;
|
if self.cancelled_runs.contains_key(&ctx.session_id) {
|
||||||
if runs.contains(&ctx.session_id) {
|
|
||||||
warn!(
|
warn!(
|
||||||
"[CancellationHook] 会话 {} 已被用户取消,阻止工具 {} 执行",
|
"[CancellationHook] 会话 {} 已被用户取消,阻止工具 {} 执行",
|
||||||
ctx.session_id, ctx.tool_name
|
ctx.session_id, ctx.tool_name
|
||||||
@ -83,8 +81,7 @@ impl AgentHook for CancellationHook {
|
|||||||
|
|
||||||
async fn on_session_stop(&self, ctx: &SessionStopContext<'_>) {
|
async fn on_session_stop(&self, ctx: &SessionStopContext<'_>) {
|
||||||
// 清理取消状态
|
// 清理取消状态
|
||||||
let mut runs = self.cancelled_runs.lock().await;
|
self.cancelled_runs.remove(&ctx.session_id);
|
||||||
runs.remove(&ctx.session_id);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -201,9 +201,8 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_cancellation_hook_blocks_when_cancelled() {
|
async fn test_cancellation_hook_blocks_when_cancelled() {
|
||||||
let mut cancelled = HashSet::new();
|
let cancelled_runs = Arc::new(dashmap::DashMap::new());
|
||||||
cancelled.insert("test_session".to_string());
|
cancelled_runs.insert("test_session".to_string(), ());
|
||||||
let cancelled_runs = Arc::new(tokio::sync::Mutex::new(cancelled));
|
|
||||||
|
|
||||||
let hook = CancellationHook::new(cancelled_runs);
|
let hook = CancellationHook::new(cancelled_runs);
|
||||||
let ctx = PreToolUseContext {
|
let ctx = PreToolUseContext {
|
||||||
@ -219,7 +218,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_cancellation_hook_allows_when_not_cancelled() {
|
async fn test_cancellation_hook_allows_when_not_cancelled() {
|
||||||
let cancelled_runs = Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new()));
|
let cancelled_runs = Arc::new(dashmap::DashMap::new());
|
||||||
|
|
||||||
let hook = CancellationHook::new(cancelled_runs);
|
let hook = CancellationHook::new(cancelled_runs);
|
||||||
let ctx = PreToolUseContext {
|
let ctx = PreToolUseContext {
|
||||||
|
|||||||
@ -52,7 +52,7 @@ impl HookRegistry {
|
|||||||
/// 使得 AgentRuntime.get_metrics() 能查询到实际运行数据。
|
/// 使得 AgentRuntime.get_metrics() 能查询到实际运行数据。
|
||||||
pub fn with_builtins(
|
pub fn with_builtins(
|
||||||
db: SqlitePool,
|
db: SqlitePool,
|
||||||
cancelled_runs: Arc<tokio::sync::Mutex<std::collections::HashSet<String>>>,
|
cancelled_runs: Arc<dashmap::DashMap<String, ()>>,
|
||||||
metrics_data: Option<Arc<tokio::sync::Mutex<MetricsData>>>,
|
metrics_data: Option<Arc<tokio::sync::Mutex<MetricsData>>>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let mut registry = Self::new();
|
let mut registry = Self::new();
|
||||||
|
|||||||
@ -138,6 +138,7 @@ pub(super) async fn process_single_result(
|
|||||||
max_output_chars,
|
max_output_chars,
|
||||||
&tool_results_dir,
|
&tool_results_dir,
|
||||||
)
|
)
|
||||||
|
.await
|
||||||
};
|
};
|
||||||
|
|
||||||
// PostToolUse hook
|
// PostToolUse hook
|
||||||
|
|||||||
@ -538,8 +538,7 @@ pub async fn execute_parallel(
|
|||||||
let cancel_handle = tokio::spawn(async move {
|
let cancel_handle = tokio::spawn(async move {
|
||||||
loop {
|
loop {
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||||
let locked = app_state_ref.cancelled_runs.lock().await;
|
if app_state_ref.cancelled_runs.contains_key(&sid_ref) {
|
||||||
if locked.contains(&sid_ref) {
|
|
||||||
cancel_flag.store(true, Ordering::SeqCst);
|
cancel_flag.store(true, Ordering::SeqCst);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -96,10 +96,12 @@ pub struct AgentRuntime {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl AgentRuntime {
|
impl AgentRuntime {
|
||||||
/// 创建新的运行时实例
|
/// 共享初始化逻辑:根据 config + mode 构建完整运行时。
|
||||||
pub fn new(app_state: Arc<AppState>) -> Self {
|
fn init(
|
||||||
let mut config = AgentConfig::default();
|
app_state: Arc<AppState>,
|
||||||
let mode_registry = ModeRegistry::builtins();
|
mut config: AgentConfig,
|
||||||
|
mode_registry: ModeRegistry,
|
||||||
|
) -> Self {
|
||||||
let mode = mode_registry.get(&config.mode).copied().unwrap_or_else(|| {
|
let mode = mode_registry.get(&config.mode).copied().unwrap_or_else(|| {
|
||||||
tracing::warn!("[AgentRuntime] 未知模式 '{}',回退到默认模式", config.mode);
|
tracing::warn!("[AgentRuntime] 未知模式 '{}',回退到默认模式", config.mode);
|
||||||
mode_registry
|
mode_registry
|
||||||
@ -123,14 +125,13 @@ impl AgentRuntime {
|
|||||||
tool_registry.add_tool(Box::new(crate::agent::tools::memory::SaveMemoryTool::new(
|
tool_registry.add_tool(Box::new(crate::agent::tools::memory::SaveMemoryTool::new(
|
||||||
app_state.memory_manager.clone(),
|
app_state.memory_manager.clone(),
|
||||||
)));
|
)));
|
||||||
// 替换 DelegateResearchTool 为带有 permission_checker 的版本(SSE 通道通过 ToolContext 注入)
|
// 替换 DelegateResearchTool 为带有 permission_checker 的版本
|
||||||
tool_registry.replace_tool(Box::new(
|
tool_registry.replace_tool(Box::new(
|
||||||
crate::agent::tools::subagent::SubAgentTool::new_with_hooks(
|
crate::agent::tools::subagent::SubAgentTool::new_with_hooks(
|
||||||
None,
|
None,
|
||||||
permission_checker.clone(),
|
permission_checker.clone(),
|
||||||
),
|
),
|
||||||
));
|
));
|
||||||
// 会话级权限检查器将在 run_react_loop 中按 session_id 注册
|
|
||||||
|
|
||||||
// 视觉模型可用时注册 analyze_image 工具
|
// 视觉模型可用时注册 analyze_image 工具
|
||||||
if app_state.vision_llm.is_some() {
|
if app_state.vision_llm.is_some() {
|
||||||
@ -140,9 +141,9 @@ impl AgentRuntime {
|
|||||||
// ── 应用模式的工具集过滤 ──
|
// ── 应用模式的工具集过滤 ──
|
||||||
apply_mode_tool_filter(&mut tool_registry, mode);
|
apply_mode_tool_filter(&mut tool_registry, mode);
|
||||||
|
|
||||||
// 初始化 checkpoint 管理器
|
// 初始化 checkpoint 管理器(存储在 library_dir/.checkpoints 下)
|
||||||
let checkpoint_enabled = true;
|
let checkpoint_enabled = true;
|
||||||
let checkpoint_store = app_state.config.library_dir.join("..").join(".checkpoints");
|
let checkpoint_store = app_state.config.library_dir.join(".checkpoints");
|
||||||
let checkpoint_manager = Arc::new(checkpoint::CheckpointManager::new(
|
let checkpoint_manager = Arc::new(checkpoint::CheckpointManager::new(
|
||||||
std::fs::canonicalize(&checkpoint_store).unwrap_or(checkpoint_store),
|
std::fs::canonicalize(&checkpoint_store).unwrap_or(checkpoint_store),
|
||||||
checkpoint_enabled,
|
checkpoint_enabled,
|
||||||
@ -168,78 +169,25 @@ impl AgentRuntime {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 创建新的运行时实例
|
||||||
|
pub fn new(app_state: Arc<AppState>) -> Self {
|
||||||
|
let config = AgentConfig::default();
|
||||||
|
let mode_registry = ModeRegistry::builtins();
|
||||||
|
Self::init(app_state, config, mode_registry)
|
||||||
|
}
|
||||||
|
|
||||||
/// 创建带自定义配置的运行时实例
|
/// 创建带自定义配置的运行时实例
|
||||||
pub fn with_config(app_state: Arc<AppState>, mut config: AgentConfig) -> Self {
|
pub fn with_config(app_state: Arc<AppState>, config: AgentConfig) -> Self {
|
||||||
let mode_registry = ModeRegistry::builtins();
|
let mode_registry = ModeRegistry::builtins();
|
||||||
let mode = mode_registry.get(&config.mode).copied().unwrap_or_else(|| {
|
Self::init(app_state, config, mode_registry)
|
||||||
tracing::warn!("[AgentRuntime] 未知模式 '{}',回退到默认模式", config.mode);
|
|
||||||
mode_registry
|
|
||||||
.get(ModeRegistry::default_id())
|
|
||||||
.copied()
|
|
||||||
.unwrap_or(&modes::default::DEFAULT_MODE)
|
|
||||||
});
|
|
||||||
// 合并模式配置预设
|
|
||||||
apply_mode_config(&mut config, mode);
|
|
||||||
|
|
||||||
let queue = Arc::new(BgNotificationQueue::new());
|
|
||||||
let metrics_data = Arc::new(tokio::sync::Mutex::new(super::hooks::MetricsData::default()));
|
|
||||||
let permission_checker = Arc::new(permission::PermissionChecker::from_config(&config));
|
|
||||||
let denial_tracker = Arc::new(std::sync::Mutex::new(denial_tracker::DenialTracker::new(
|
|
||||||
config.denial_max_consecutive,
|
|
||||||
config.denial_max_total,
|
|
||||||
)));
|
|
||||||
let skill_registry = app_state.skill_registry.clone();
|
|
||||||
let mut tool_registry = ToolRegistry::new_with_queue(Some(queue.clone()), skill_registry);
|
|
||||||
tool_registry.add_tool(Box::new(crate::agent::tools::memory::SaveMemoryTool::new(
|
|
||||||
app_state.memory_manager.clone(),
|
|
||||||
)));
|
|
||||||
tool_registry.replace_tool(Box::new(
|
|
||||||
crate::agent::tools::subagent::SubAgentTool::new_with_hooks(
|
|
||||||
None,
|
|
||||||
permission_checker.clone(),
|
|
||||||
),
|
|
||||||
));
|
|
||||||
// 会话级权限检查器将在 run_react_loop 中按 session_id 注册
|
|
||||||
|
|
||||||
// 视觉模型可用时注册 analyze_image 工具
|
|
||||||
if app_state.vision_llm.is_some() {
|
|
||||||
tool_registry.add_tool(Box::new(crate::agent::tools::astro::AnalyzeImageTool));
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── 应用模式的工具集过滤 ──
|
|
||||||
apply_mode_tool_filter(&mut tool_registry, mode);
|
|
||||||
|
|
||||||
// 初始化 checkpoint 管理器
|
|
||||||
let checkpoint_enabled = true;
|
|
||||||
let checkpoint_store = app_state.config.library_dir.join("..").join(".checkpoints");
|
|
||||||
let checkpoint_manager = Arc::new(checkpoint::CheckpointManager::new(
|
|
||||||
std::fs::canonicalize(&checkpoint_store).unwrap_or(checkpoint_store),
|
|
||||||
checkpoint_enabled,
|
|
||||||
));
|
|
||||||
|
|
||||||
AgentRuntime {
|
|
||||||
app_state,
|
|
||||||
config,
|
|
||||||
tool_registry,
|
|
||||||
bg_notification_queue: queue,
|
|
||||||
metrics_data,
|
|
||||||
compaction_breaker: Arc::new(std::sync::Mutex::new(
|
|
||||||
circuit_breaker::CompactionCircuitBreaker::new(),
|
|
||||||
)),
|
|
||||||
permission_checker,
|
|
||||||
denial_tracker,
|
|
||||||
read_file_state: Arc::new(std::sync::Mutex::new(file_cache::FileStateCache::new())),
|
|
||||||
prompt_cache: std::sync::Mutex::new(SystemPromptCache::new()),
|
|
||||||
collapse_log: Arc::new(std::sync::Mutex::new(compact::collapse::CollapseLog::new())),
|
|
||||||
checkpoint_manager,
|
|
||||||
mode,
|
|
||||||
mode_registry,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 返回当前运行指标快照(锁异常时返回默认值并记录警告)
|
/// 返回当前运行指标快照(锁异常时返回默认值并记录警告)
|
||||||
pub fn get_metrics(&self) -> super::hooks::MetricsData {
|
pub fn get_metrics(&self) -> super::hooks::MetricsData {
|
||||||
self.metrics_data.try_lock().map(|m| m.clone()).unwrap_or_else(|_| {
|
self.metrics_data
|
||||||
|
.try_lock()
|
||||||
|
.map(|m| m.clone())
|
||||||
|
.unwrap_or_else(|_| {
|
||||||
warn!("[AgentRuntime] 获取指标锁失败,返回默认值");
|
warn!("[AgentRuntime] 获取指标锁失败,返回默认值");
|
||||||
super::hooks::MetricsData::default()
|
super::hooks::MetricsData::default()
|
||||||
})
|
})
|
||||||
@ -456,7 +404,7 @@ impl AgentRuntime {
|
|||||||
|
|
||||||
// Phase 4: 会话收尾(传入实际的终止原因 + trajectory 导出参数)
|
// Phase 4: 会话收尾(传入实际的终止原因 + trajectory 导出参数)
|
||||||
let system_prompt = self.system_prompt();
|
let system_prompt = self.system_prompt();
|
||||||
let model_name = self.app_state.llm.model();
|
let model_name = self.app_state.llm.model().await;
|
||||||
finalize::finalize_turn(
|
finalize::finalize_turn(
|
||||||
db,
|
db,
|
||||||
&session_info.session_id,
|
&session_info.session_id,
|
||||||
@ -491,12 +439,10 @@ impl AgentRuntime {
|
|||||||
let turn_index = session_info.turn_index;
|
let turn_index = session_info.turn_index;
|
||||||
|
|
||||||
// 注册当前会话的权限检查器(如不存在则从全局配置初始化)
|
// 注册当前会话的权限检查器(如不存在则从全局配置初始化)
|
||||||
{
|
self.app_state
|
||||||
let mut checkers = self.app_state.session_permission_checkers.write().await;
|
.session_permission_checkers
|
||||||
checkers
|
|
||||||
.entry(sid.clone())
|
.entry(sid.clone())
|
||||||
.or_insert_with(|| (*self.permission_checker).clone());
|
.or_insert_with(|| (*self.permission_checker).clone());
|
||||||
}
|
|
||||||
|
|
||||||
let tool_defs = self.tool_registry.definitions();
|
let tool_defs = self.tool_registry.definitions();
|
||||||
let mut duplicate_detector = DuplicateDetector::default();
|
let mut duplicate_detector = DuplicateDetector::default();
|
||||||
@ -522,10 +468,7 @@ impl AgentRuntime {
|
|||||||
self.checkpoint_manager.new_turn();
|
self.checkpoint_manager.new_turn();
|
||||||
|
|
||||||
// 检查用户取消
|
// 检查用户取消
|
||||||
let is_cancelled = {
|
let is_cancelled = self.app_state.cancelled_runs.remove(sid).is_some();
|
||||||
let mut cancelled = self.app_state.cancelled_runs.lock().await;
|
|
||||||
cancelled.remove(sid)
|
|
||||||
};
|
|
||||||
|
|
||||||
if is_cancelled {
|
if is_cancelled {
|
||||||
warn!("[AgentRuntime] 用户手动中止了会话 {} 的智能体执行", sid);
|
warn!("[AgentRuntime] 用户手动中止了会话 {} 的智能体执行", sid);
|
||||||
@ -868,14 +811,11 @@ impl AgentRuntime {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 并行执行工具(带权限检查、checkpoint 和分区器)
|
// 并行执行工具(带权限检查、checkpoint 和分区器)
|
||||||
let session_checker_snapshot = {
|
let session_checker_snapshot = self
|
||||||
self.app_state
|
.app_state
|
||||||
.session_permission_checkers
|
.session_permission_checkers
|
||||||
.read()
|
|
||||||
.await
|
|
||||||
.get(sid)
|
.get(sid)
|
||||||
.cloned()
|
.map(|r| r.value().clone());
|
||||||
};
|
|
||||||
let exec_result = executor::execute_parallel(
|
let exec_result = executor::execute_parallel(
|
||||||
&prepared_calls,
|
&prepared_calls,
|
||||||
&self.tool_registry,
|
&self.tool_registry,
|
||||||
@ -947,8 +887,7 @@ impl AgentRuntime {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if exec_result.was_cancelled {
|
if exec_result.was_cancelled {
|
||||||
let mut locked = self.app_state.cancelled_runs.lock().await;
|
self.app_state.cancelled_runs.remove(sid);
|
||||||
locked.remove(sid);
|
|
||||||
warn!(
|
warn!(
|
||||||
"[AgentRuntime] 工具执行期间被用户手动中止,会话 ID: {}",
|
"[AgentRuntime] 工具执行期间被用户手动中止,会话 ID: {}",
|
||||||
sid
|
sid
|
||||||
@ -1016,8 +955,7 @@ impl AgentRuntime {
|
|||||||
match output.status {
|
match output.status {
|
||||||
StreamStatus::Success => return Some(output),
|
StreamStatus::Success => return Some(output),
|
||||||
StreamStatus::Cancelled => {
|
StreamStatus::Cancelled => {
|
||||||
let mut cancelled = self.app_state.cancelled_runs.lock().await;
|
self.app_state.cancelled_runs.remove(session_id);
|
||||||
cancelled.remove(session_id);
|
|
||||||
warn!(
|
warn!(
|
||||||
"[AgentRuntime] 流式调用期间被用户手动中止,会话 ID: {}",
|
"[AgentRuntime] 流式调用期间被用户手动中止,会话 ID: {}",
|
||||||
session_id
|
session_id
|
||||||
@ -1077,7 +1015,7 @@ impl AgentRuntime {
|
|||||||
"[AgentRuntime] 连续 {} 次过载,切换到备用模型: {}",
|
"[AgentRuntime] 连续 {} 次过载,切换到备用模型: {}",
|
||||||
consecutive_overloads, fallback
|
consecutive_overloads, fallback
|
||||||
);
|
);
|
||||||
llm.set_model(fallback.clone());
|
llm.set_model(fallback.clone()).await;
|
||||||
consecutive_overloads = 0;
|
consecutive_overloads = 0;
|
||||||
} else if !self.app_state.config.llm_fallback_chain.is_empty() {
|
} else if !self.app_state.config.llm_fallback_chain.is_empty() {
|
||||||
let idx = ((consecutive_overloads as usize - 3)
|
let idx = ((consecutive_overloads as usize - 3)
|
||||||
@ -1088,15 +1026,14 @@ impl AgentRuntime {
|
|||||||
"[AgentRuntime] 连续 {} 次过载,从链中切换: {}",
|
"[AgentRuntime] 连续 {} 次过载,从链中切换: {}",
|
||||||
consecutive_overloads, alt
|
consecutive_overloads, alt
|
||||||
);
|
);
|
||||||
llm.set_model(alt.clone());
|
llm.set_model(alt.clone()).await;
|
||||||
consecutive_overloads = 0;
|
consecutive_overloads = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 检查用户取消
|
// 检查用户取消
|
||||||
let cancelled = self.app_state.cancelled_runs.lock().await;
|
if self.app_state.cancelled_runs.contains_key(session_id) {
|
||||||
if cancelled.contains(session_id) {
|
|
||||||
warn!("[AgentRuntime] 退避重试期间被用户取消");
|
warn!("[AgentRuntime] 退避重试期间被用户取消");
|
||||||
let _ = tx.send(AgentStreamEvent::Error {
|
let _ = tx.send(AgentStreamEvent::Error {
|
||||||
message: "用户已手动中止执行。".to_string(),
|
message: "用户已手动中止执行。".to_string(),
|
||||||
@ -1123,8 +1060,7 @@ impl AgentRuntime {
|
|||||||
return Some(retry_output);
|
return Some(retry_output);
|
||||||
}
|
}
|
||||||
StreamStatus::Cancelled => {
|
StreamStatus::Cancelled => {
|
||||||
let mut cancelled = self.app_state.cancelled_runs.lock().await;
|
self.app_state.cancelled_runs.remove(session_id);
|
||||||
cancelled.remove(session_id);
|
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
StreamStatus::Error(_) => {
|
StreamStatus::Error(_) => {
|
||||||
@ -1232,8 +1168,7 @@ impl AgentRuntime {
|
|||||||
return Some(retry_output);
|
return Some(retry_output);
|
||||||
}
|
}
|
||||||
StreamStatus::Cancelled => {
|
StreamStatus::Cancelled => {
|
||||||
let mut cancelled = self.app_state.cancelled_runs.lock().await;
|
self.app_state.cancelled_runs.remove(session_id);
|
||||||
cancelled.remove(session_id);
|
|
||||||
warn!("[AgentRuntime] 恢复期间被用户中止");
|
warn!("[AgentRuntime] 恢复期间被用户中止");
|
||||||
let _ = tx.send(AgentStreamEvent::Error {
|
let _ = tx.send(AgentStreamEvent::Error {
|
||||||
message: "用户已手动中止执行。".to_string(),
|
message: "用户已手动中止执行。".to_string(),
|
||||||
@ -1400,7 +1335,7 @@ impl AgentRuntime {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let today = chrono::Utc::now().format("%Y-%m-%d").to_string();
|
let today = chrono::Utc::now().format("%Y-%m-%d").to_string();
|
||||||
let model_name = self.app_state.llm.model().to_string();
|
let model_name = self.app_state.llm.model_name_snapshot();
|
||||||
|
|
||||||
let mut lines = vec![
|
let mut lines = vec![
|
||||||
"# 环境信息".to_string(),
|
"# 环境信息".to_string(),
|
||||||
|
|||||||
@ -83,7 +83,7 @@ pub async fn create_or_resume_session(
|
|||||||
)
|
)
|
||||||
.bind(&new_id)
|
.bind(&new_id)
|
||||||
.bind("")
|
.bind("")
|
||||||
.bind(llm.model())
|
.bind(llm.model().await)
|
||||||
.bind(mode)
|
.bind(mode)
|
||||||
.execute(db)
|
.execute(db)
|
||||||
.await?;
|
.await?;
|
||||||
@ -544,7 +544,9 @@ pub async fn branch_session(db: &SqlitePool, session_id: &str) -> anyhow::Result
|
|||||||
|
|
||||||
let forked_at = last_active_id.unwrap_or(0);
|
let forked_at = last_active_id.unwrap_or(0);
|
||||||
|
|
||||||
// 4. 创建新会话
|
// 4. 创建新会话与复制消息(同一个事务内)
|
||||||
|
let mut tx = db.begin().await?;
|
||||||
|
|
||||||
let branch_id = uuid::Uuid::new_v4().to_string();
|
let branch_id = uuid::Uuid::new_v4().to_string();
|
||||||
let branch_title = if title.is_empty() {
|
let branch_title = if title.is_empty() {
|
||||||
format!("分支 (来自 {})", &session_id[..8.min(session_id.len())])
|
format!("分支 (来自 {})", &session_id[..8.min(session_id.len())])
|
||||||
@ -565,7 +567,7 @@ pub async fn branch_session(db: &SqlitePool, session_id: &str) -> anyhow::Result
|
|||||||
.bind(&branch_title)
|
.bind(&branch_title)
|
||||||
.bind(session_id)
|
.bind(session_id)
|
||||||
.bind(serde_json::to_string(&branch_meta).unwrap_or_default())
|
.bind(serde_json::to_string(&branch_meta).unwrap_or_default())
|
||||||
.execute(db)
|
.execute(&mut *tx)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// 5. 复制所有 active=1 的消息到新会话
|
// 5. 复制所有 active=1 的消息到新会话
|
||||||
@ -573,7 +575,7 @@ pub async fn branch_session(db: &SqlitePool, session_id: &str) -> anyhow::Result
|
|||||||
"SELECT COUNT(*) FROM agent_messages WHERE session_id = ? AND active = 1",
|
"SELECT COUNT(*) FROM agent_messages WHERE session_id = ? AND active = 1",
|
||||||
)
|
)
|
||||||
.bind(session_id)
|
.bind(session_id)
|
||||||
.fetch_one(db)
|
.fetch_one(&mut *tx)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
@ -588,9 +590,11 @@ pub async fn branch_session(db: &SqlitePool, session_id: &str) -> anyhow::Result
|
|||||||
)
|
)
|
||||||
.bind(&branch_id)
|
.bind(&branch_id)
|
||||||
.bind(session_id)
|
.bind(session_id)
|
||||||
.execute(db)
|
.execute(&mut *tx)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
tx.commit().await?;
|
||||||
|
|
||||||
info!(
|
info!(
|
||||||
"[Session] 分叉完成: parent={}, branch={}, copied={} messages, forked_at={}",
|
"[Session] 分叉完成: parent={}, branch={}, copied={} messages, forked_at={}",
|
||||||
session_id, branch_id, copied, forked_at
|
session_id, branch_id, copied, forked_at
|
||||||
|
|||||||
@ -43,7 +43,7 @@ pub async fn process_llm_stream(
|
|||||||
tx: &mpsc::UnboundedSender<AgentStreamEvent>,
|
tx: &mpsc::UnboundedSender<AgentStreamEvent>,
|
||||||
step: usize,
|
step: usize,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
cancelled_runs: Arc<tokio::sync::Mutex<std::collections::HashSet<String>>>,
|
cancelled_runs: Arc<dashmap::DashMap<String, ()>>,
|
||||||
enable_thinking: bool,
|
enable_thinking: bool,
|
||||||
) -> StreamOutput {
|
) -> StreamOutput {
|
||||||
// 1. 发起 LLM 流式调用
|
// 1. 发起 LLM 流式调用
|
||||||
@ -76,8 +76,7 @@ pub async fn process_llm_stream(
|
|||||||
let cancel_fut = async {
|
let cancel_fut = async {
|
||||||
loop {
|
loop {
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||||
let cancelled = cancelled_runs.lock().await;
|
if cancelled_runs.contains_key(&sid) {
|
||||||
if cancelled.contains(&sid) {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -590,10 +590,7 @@ mod tests {
|
|||||||
ads,
|
ads,
|
||||||
arxiv,
|
arxiv,
|
||||||
vizier,
|
vizier,
|
||||||
lamost: crate::clients::lamost::LamostClient::new(
|
lamost: crate::clients::lamost::LamostClient::new("https://www.lamost.org", 60)
|
||||||
"https://www.lamost.org",
|
|
||||||
60,
|
|
||||||
)
|
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
gaia: crate::clients::gaia::GaiaClient::new(
|
gaia: crate::clients::gaia::GaiaClient::new(
|
||||||
"https://gea.esac.esa.int/tap-server/tap",
|
"https://gea.esac.esa.int/tap-server/tap",
|
||||||
@ -621,7 +618,7 @@ mod tests {
|
|||||||
harvest_status: Arc::new(tokio::sync::Mutex::new(MetaSyncStatus::default())),
|
harvest_status: Arc::new(tokio::sync::Mutex::new(MetaSyncStatus::default())),
|
||||||
batch_status: Arc::new(tokio::sync::Mutex::new(AssetBatchStatus::default())),
|
batch_status: Arc::new(tokio::sync::Mutex::new(AssetBatchStatus::default())),
|
||||||
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
|
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
|
||||||
cancelled_runs: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())),
|
cancelled_runs: Arc::new(dashmap::DashMap::new()),
|
||||||
skill_registry: Arc::new(tokio::sync::RwLock::new(SkillRegistry::new(PathBuf::from(
|
skill_registry: Arc::new(tokio::sync::RwLock::new(SkillRegistry::new(PathBuf::from(
|
||||||
"/tmp/sk",
|
"/tmp/sk",
|
||||||
)))),
|
)))),
|
||||||
@ -629,16 +626,16 @@ mod tests {
|
|||||||
pending_permissions: Arc::new(
|
pending_permissions: Arc::new(
|
||||||
tokio::sync::Mutex::new(std::collections::HashMap::new()),
|
tokio::sync::Mutex::new(std::collections::HashMap::new()),
|
||||||
),
|
),
|
||||||
session_permission_checkers: Arc::new(tokio::sync::RwLock::new(
|
session_permission_checkers: Arc::new(dashmap::DashMap::new()),
|
||||||
std::collections::HashMap::new(),
|
|
||||||
)),
|
|
||||||
sse_broadcast: None,
|
sse_broadcast: None,
|
||||||
memory_manager: Arc::new(tokio::sync::Mutex::new(MemoryManager::new(PathBuf::from(
|
memory_manager: Arc::new(tokio::sync::Mutex::new(MemoryManager::new(PathBuf::from(
|
||||||
"/tmp/test_mem",
|
"/tmp/test_mem",
|
||||||
)))),
|
)))),
|
||||||
sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
|
sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
|
||||||
login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
||||||
upload_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
upload_rate_limiter: Arc::new(
|
||||||
|
tokio::sync::Mutex::new(std::collections::HashMap::new()),
|
||||||
|
),
|
||||||
});
|
});
|
||||||
|
|
||||||
ToolContext::new(app_state)
|
ToolContext::new(app_state)
|
||||||
|
|||||||
@ -26,7 +26,7 @@ impl SkillCreator {
|
|||||||
///
|
///
|
||||||
/// 返回创建的 skill 名称列表。
|
/// 返回创建的 skill 名称列表。
|
||||||
/// 如果目标 skill 目录已存在则跳过(不覆盖已有 skill)。
|
/// 如果目标 skill 目录已存在则跳过(不覆盖已有 skill)。
|
||||||
pub fn create_from_patterns(
|
pub async fn create_from_patterns(
|
||||||
&self,
|
&self,
|
||||||
patterns: &[pattern_detector::DetectedPattern],
|
patterns: &[pattern_detector::DetectedPattern],
|
||||||
) -> std::io::Result<Vec<String>> {
|
) -> std::io::Result<Vec<String>> {
|
||||||
@ -46,13 +46,13 @@ impl SkillCreator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 创建 skill 目录
|
// 创建 skill 目录
|
||||||
std::fs::create_dir_all(&skill_dir)?;
|
tokio::fs::create_dir_all(&skill_dir).await?;
|
||||||
|
|
||||||
// 生成 SKILL.md 内容
|
// 生成 SKILL.md 内容
|
||||||
let content = self.generate_skill_md(pattern, &skill_name);
|
let content = self.generate_skill_md(pattern, &skill_name);
|
||||||
|
|
||||||
let md_path = skill_dir.join("SKILL.md");
|
let md_path = skill_dir.join("SKILL.md");
|
||||||
std::fs::write(&md_path, &content)?;
|
tokio::fs::write(&md_path, &content).await?;
|
||||||
|
|
||||||
info!(
|
info!(
|
||||||
"[SkillCreator] 已从模式创建 Skill '{}': {}",
|
"[SkillCreator] 已从模式创建 Skill '{}': {}",
|
||||||
@ -217,7 +217,7 @@ impl SelfImprovePipeline {
|
|||||||
info!("[SelfImprove] 检测到 {} 个候选模式", patterns.len());
|
info!("[SelfImprove] 检测到 {} 个候选模式", patterns.len());
|
||||||
|
|
||||||
// Step 2: 创建 Skills
|
// Step 2: 创建 Skills
|
||||||
let created = self.creator.create_from_patterns(&patterns)?;
|
let created = self.creator.create_from_patterns(&patterns).await?;
|
||||||
|
|
||||||
// Step 3: Curator 分析
|
// Step 3: Curator 分析
|
||||||
let curator_report = self.curator.analyze(usage_stats, skill_metas);
|
let curator_report = self.curator.analyze(usage_stats, skill_metas);
|
||||||
|
|||||||
@ -340,9 +340,12 @@ impl Curator {
|
|||||||
/// 归档指定的 stale skills(移动到 archive 目录而非删除)。
|
/// 归档指定的 stale skills(移动到 archive 目录而非删除)。
|
||||||
///
|
///
|
||||||
/// 返回成功归档的 skill 名称列表。
|
/// 返回成功归档的 skill 名称列表。
|
||||||
pub fn archive_stale_skills(&self, skill_names: &[String]) -> std::io::Result<Vec<String>> {
|
pub async fn archive_stale_skills(
|
||||||
|
&self,
|
||||||
|
skill_names: &[String],
|
||||||
|
) -> std::io::Result<Vec<String>> {
|
||||||
// 确保归档目录存在
|
// 确保归档目录存在
|
||||||
std::fs::create_dir_all(&self.archive_dir)?;
|
tokio::fs::create_dir_all(&self.archive_dir).await?;
|
||||||
|
|
||||||
let mut archived = Vec::new();
|
let mut archived = Vec::new();
|
||||||
|
|
||||||
@ -355,17 +358,16 @@ impl Curator {
|
|||||||
|
|
||||||
let archive_path = self.archive_dir.join(name);
|
let archive_path = self.archive_dir.join(name);
|
||||||
if archive_path.exists() {
|
if archive_path.exists() {
|
||||||
// 避免覆盖已有归档:添加时间戳后缀
|
|
||||||
let ts = Utc::now().format("%Y%m%d%H%M%S");
|
let ts = Utc::now().format("%Y%m%d%H%M%S");
|
||||||
let renamed = self.archive_dir.join(format!("{}-{}", name, ts));
|
let renamed = self.archive_dir.join(format!("{}-{}", name, ts));
|
||||||
info!(
|
info!(
|
||||||
"[Curator] 归档目录已存在,使用新名称: {}",
|
"[Curator] 归档目录已存在,使用新名称: {}",
|
||||||
renamed.display()
|
renamed.display()
|
||||||
);
|
);
|
||||||
std::fs::rename(&skill_dir, &renamed)?;
|
tokio::fs::rename(&skill_dir, &renamed).await?;
|
||||||
archived.push(format!("{}-{}", name, ts));
|
archived.push(format!("{}-{}", name, ts));
|
||||||
} else {
|
} else {
|
||||||
std::fs::rename(&skill_dir, &archive_path)?;
|
tokio::fs::rename(&skill_dir, &archive_path).await?;
|
||||||
archived.push(name.clone());
|
archived.push(name.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -64,29 +64,39 @@ pub fn inbox_path(team_dir: &Path, agent_name: &str) -> PathBuf {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// 向收件箱追加一条消息
|
/// 向收件箱追加一条消息
|
||||||
pub fn append_message(team_dir: &Path, agent_name: &str, msg: &TeamMessage) -> std::io::Result<()> {
|
pub async fn append_message(
|
||||||
|
team_dir: &Path,
|
||||||
|
agent_name: &str,
|
||||||
|
msg: &TeamMessage,
|
||||||
|
) -> std::io::Result<()> {
|
||||||
let inbox = inbox_path(team_dir, agent_name);
|
let inbox = inbox_path(team_dir, agent_name);
|
||||||
if let Some(parent) = inbox.parent() {
|
if let Some(parent) = inbox.parent() {
|
||||||
std::fs::create_dir_all(parent)?;
|
tokio::fs::create_dir_all(parent).await?;
|
||||||
}
|
}
|
||||||
let line = serde_json::to_string(msg).unwrap_or_default();
|
let line = serde_json::to_string(msg).unwrap_or_default();
|
||||||
|
let inbox_clone = inbox.clone();
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
let mut file = std::fs::OpenOptions::new()
|
let mut file = std::fs::OpenOptions::new()
|
||||||
.create(true)
|
.create(true)
|
||||||
.append(true)
|
.append(true)
|
||||||
.open(&inbox)?;
|
.open(&inbox_clone)?;
|
||||||
file.write_all(line.as_bytes())?;
|
file.write_all(line.as_bytes())?;
|
||||||
file.write_all(b"\n")?;
|
file.write_all(b"\n")?;
|
||||||
Ok(())
|
Ok(())
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 读取并清空收件箱
|
/// 读取并清空收件箱
|
||||||
pub fn drain_inbox(team_dir: &Path, agent_name: &str) -> Vec<TeamMessage> {
|
pub async fn drain_inbox(team_dir: &Path, agent_name: &str) -> Vec<TeamMessage> {
|
||||||
let inbox = inbox_path(team_dir, agent_name);
|
let inbox = inbox_path(team_dir, agent_name);
|
||||||
if !inbox.exists() {
|
if !inbox.exists() {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
}
|
}
|
||||||
|
|
||||||
let content = match std::fs::read_to_string(&inbox) {
|
let inbox_clone = inbox.clone();
|
||||||
|
let content = match tokio::fs::read_to_string(&inbox_clone).await {
|
||||||
Ok(c) => c,
|
Ok(c) => c,
|
||||||
Err(_) => return Vec::new(),
|
Err(_) => return Vec::new(),
|
||||||
};
|
};
|
||||||
@ -98,7 +108,7 @@ pub fn drain_inbox(team_dir: &Path, agent_name: &str) -> Vec<TeamMessage> {
|
|||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
// 清空文件
|
// 清空文件
|
||||||
let _ = std::fs::write(&inbox, "");
|
let _ = tokio::fs::write(&inbox, "").await;
|
||||||
|
|
||||||
messages
|
messages
|
||||||
}
|
}
|
||||||
|
|||||||
@ -157,9 +157,15 @@ impl TeamManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// 发送消息给指定队友
|
/// 发送消息给指定队友
|
||||||
pub fn send_message(&self, from: &str, to: &str, content: &str, msg_type: TeamMessageType) {
|
pub async fn send_message(
|
||||||
|
&self,
|
||||||
|
from: &str,
|
||||||
|
to: &str,
|
||||||
|
content: &str,
|
||||||
|
msg_type: TeamMessageType,
|
||||||
|
) {
|
||||||
let msg = TeamMessage::new(from, to, content, msg_type);
|
let msg = TeamMessage::new(from, to, content, msg_type);
|
||||||
if let Err(e) = inbox::append_message(&self.team_dir, to, &msg) {
|
if let Err(e) = inbox::append_message(&self.team_dir, to, &msg).await {
|
||||||
warn!("[TeamManager] 发送消息失败: {}", e);
|
warn!("[TeamManager] 发送消息失败: {}", e);
|
||||||
} else {
|
} else {
|
||||||
info!(
|
info!(
|
||||||
@ -172,7 +178,7 @@ impl TeamManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// 广播消息给所有队友
|
/// 广播消息给所有队友
|
||||||
pub fn broadcast(&self, from: &str, content: &str) {
|
pub async fn broadcast(&self, from: &str, content: &str) {
|
||||||
info!(
|
info!(
|
||||||
"[TeamManager] 广播消息: {}",
|
"[TeamManager] 广播消息: {}",
|
||||||
content.chars().take(80).collect::<String>()
|
content.chars().take(80).collect::<String>()
|
||||||
@ -180,14 +186,14 @@ impl TeamManager {
|
|||||||
let msg = TeamMessage::new(from, "all", content, TeamMessageType::Status);
|
let msg = TeamMessage::new(from, "all", content, TeamMessageType::Status);
|
||||||
for member in &self.config.members {
|
for member in &self.config.members {
|
||||||
if member.name != from {
|
if member.name != from {
|
||||||
let _ = inbox::append_message(&self.team_dir, &member.name, &msg);
|
let _ = inbox::append_message(&self.team_dir, &member.name, &msg).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 检查并清空指定 agent 的收件箱
|
/// 检查并清空指定 agent 的收件箱
|
||||||
pub fn check_inbox(&self, agent_name: &str) -> Vec<TeamMessage> {
|
pub async fn check_inbox(&self, agent_name: &str) -> Vec<TeamMessage> {
|
||||||
inbox::drain_inbox(&self.team_dir, agent_name)
|
inbox::drain_inbox(&self.team_dir, agent_name).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 列出所有队友状态
|
/// 列出所有队友状态
|
||||||
|
|||||||
@ -61,14 +61,14 @@ pub async fn run_teammate_loop(
|
|||||||
&format!("队友 {} ({}) 已退出。", name, role),
|
&format!("队友 {} ({}) 已退出。", name, role),
|
||||||
TeamMessageType::Status,
|
TeamMessageType::Status,
|
||||||
);
|
);
|
||||||
let _ = inbox::append_message(&team_dir, "lead", &goodbye);
|
let _ = inbox::append_message(&team_dir, "lead", &goodbye).await;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── IDLE 阶段:检查收件箱 ──
|
// ── IDLE 阶段:检查收件箱 ──
|
||||||
*status.lock().await = MemberStatus::Idle;
|
*status.lock().await = MemberStatus::Idle;
|
||||||
|
|
||||||
let inbox_msgs = inbox::drain_inbox(&team_dir, &name);
|
let inbox_msgs = inbox::drain_inbox(&team_dir, &name).await;
|
||||||
|
|
||||||
if inbox_msgs.is_empty() {
|
if inbox_msgs.is_empty() {
|
||||||
// 等待新消息或取消,poll 间隔 5 秒,最长 60 秒
|
// 等待新消息或取消,poll 间隔 5 秒,最长 60 秒
|
||||||
@ -84,7 +84,7 @@ pub async fn run_teammate_loop(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 再次 drain(可能因为 pending 标志被唤醒)
|
// 再次 drain(可能因为 pending 标志被唤醒)
|
||||||
let inbox_msgs = inbox::drain_inbox(&team_dir, &name);
|
let inbox_msgs = inbox::drain_inbox(&team_dir, &name).await;
|
||||||
if inbox_msgs.is_empty() {
|
if inbox_msgs.is_empty() {
|
||||||
continue; // 超时,没有新消息,继续 idle
|
continue; // 超时,没有新消息,继续 idle
|
||||||
}
|
}
|
||||||
@ -115,7 +115,7 @@ pub async fn run_teammate_loop(
|
|||||||
if let Some(ref summary) = result {
|
if let Some(ref summary) = result {
|
||||||
let reply =
|
let reply =
|
||||||
super::inbox::TeamMessage::new(&name, "lead", summary, TeamMessageType::Result);
|
super::inbox::TeamMessage::new(&name, "lead", summary, TeamMessageType::Result);
|
||||||
let _ = inbox::append_message(&team_dir, "lead", &reply);
|
let _ = inbox::append_message(&team_dir, "lead", &reply).await;
|
||||||
info!(
|
info!(
|
||||||
"[Teammate:{}] 任务完成,已发送结果 ({}字符)",
|
"[Teammate:{}] 任务完成,已发送结果 ({}字符)",
|
||||||
name,
|
name,
|
||||||
@ -147,7 +147,7 @@ pub async fn run_teammate_loop(
|
|||||||
if let Some(ref summary) = result {
|
if let Some(ref summary) = result {
|
||||||
let reply =
|
let reply =
|
||||||
super::inbox::TeamMessage::new(&name, "lead", summary, TeamMessageType::Result);
|
super::inbox::TeamMessage::new(&name, "lead", summary, TeamMessageType::Result);
|
||||||
let _ = inbox::append_message(&team_dir, "lead", &reply);
|
let _ = inbox::append_message(&team_dir, "lead", &reply).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -44,7 +44,7 @@ impl AnalyzeImageTool {
|
|||||||
json!({
|
json!({
|
||||||
"image_path": image_path,
|
"image_path": image_path,
|
||||||
"question": question,
|
"question": question,
|
||||||
"model": vision_llm.model(),
|
"model": vision_llm.model().await,
|
||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
@ -237,7 +237,7 @@ mod tests {
|
|||||||
.filter(|e| matches!(e, AgentStreamEvent::TextDelta { .. }))
|
.filter(|e| matches!(e, AgentStreamEvent::TextDelta { .. }))
|
||||||
.count();
|
.count();
|
||||||
|
|
||||||
println!("视觉模型: {}", vision_llm.model());
|
println!("视觉模型: {}", vision_llm.model().await);
|
||||||
println!("SSE 事件总数: {}, TextDelta: {}", events.len(), delta_count);
|
println!("SSE 事件总数: {}, TextDelta: {}", events.len(), delta_count);
|
||||||
println!("分析结果:\n{}", output.content);
|
println!("分析结果:\n{}", output.content);
|
||||||
|
|
||||||
|
|||||||
@ -253,7 +253,14 @@ impl AgentTool for GetCitationNetworkTool {
|
|||||||
|
|
||||||
// 有 query → 引用查找模式:在关联文献中按作者+年份搜索
|
// 有 query → 引用查找模式:在关联文献中按作者+年份搜索
|
||||||
if let Some(ref q) = query {
|
if let Some(ref q) = query {
|
||||||
match crate::services::citation::search_citations(&state.db, &paper.bibcode, direction, q).await {
|
match crate::services::citation::search_citations(
|
||||||
|
&state.db,
|
||||||
|
&paper.bibcode,
|
||||||
|
direction,
|
||||||
|
q,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(results) => {
|
Ok(results) => {
|
||||||
if results.is_empty() {
|
if results.is_empty() {
|
||||||
let dir_label = if direction == "citations" {
|
let dir_label = if direction == "citations" {
|
||||||
@ -315,7 +322,14 @@ impl AgentTool for GetCitationNetworkTool {
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// 无 query → 分页浏览模式
|
// 无 query → 分页浏览模式
|
||||||
match crate::services::citation::get_citations_paginated(&state.db, &paper.bibcode, direction, sort, offset, limit)
|
match crate::services::citation::get_citations_paginated(
|
||||||
|
&state.db,
|
||||||
|
&paper.bibcode,
|
||||||
|
direction,
|
||||||
|
sort,
|
||||||
|
offset,
|
||||||
|
limit,
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok((rows, total)) => {
|
Ok((rows, total)) => {
|
||||||
|
|||||||
@ -11,9 +11,9 @@ pub mod target;
|
|||||||
pub mod vizier;
|
pub mod vizier;
|
||||||
|
|
||||||
pub use library::{GetCitationNetworkTool, SearchLocalLibraryTool};
|
pub use library::{GetCitationNetworkTool, SearchLocalLibraryTool};
|
||||||
pub use observation::FindObservationTool;
|
|
||||||
pub use metadata::GetPaperMetadataTool;
|
pub use metadata::GetPaperMetadataTool;
|
||||||
pub use note::SaveNoteTool;
|
pub use note::SaveNoteTool;
|
||||||
|
pub use observation::FindObservationTool;
|
||||||
pub use paper::{GetPaperContentTool, GetPaperOutlineTool};
|
pub use paper::{GetPaperContentTool, GetPaperOutlineTool};
|
||||||
pub use rag::RagSearchTool;
|
pub use rag::RagSearchTool;
|
||||||
pub use target::QueryTargetTool;
|
pub use target::QueryTargetTool;
|
||||||
|
|||||||
@ -58,6 +58,7 @@ impl AgentTool for SaveNoteTool {
|
|||||||
let state = &ctx.app_state;
|
let state = &ctx.app_state;
|
||||||
|
|
||||||
match crate::services::note::save_note_service(&state.config.library_dir, &title, &content)
|
match crate::services::note::save_note_service(&state.config.library_dir, &title, &content)
|
||||||
|
.await
|
||||||
{
|
{
|
||||||
Ok((filename, filepath, size)) => ToolOutput::success(
|
Ok((filename, filepath, size)) => ToolOutput::success(
|
||||||
format!("笔记已保存: {}", filename),
|
format!("笔记已保存: {}", filename),
|
||||||
|
|||||||
@ -52,23 +52,26 @@ impl AgentTool for FindObservationTool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn parameters(&self) -> serde_json::Value {
|
fn parameters(&self) -> serde_json::Value {
|
||||||
|
// enum 列表从枚举的 valid_values() 动态生成,新增源/产品时自动同步
|
||||||
|
let source_enum: Vec<&str> = Source::valid_values().to_vec();
|
||||||
|
let product_enum: Vec<&str> = ProductType::valid_values().to_vec();
|
||||||
json!({
|
json!({
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
"source": {
|
"source": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"enum": ["lamost", "gaia", "sdss", "desi"],
|
"enum": source_enum,
|
||||||
"description": "数据源"
|
"description": "数据源"
|
||||||
},
|
},
|
||||||
"product": {
|
"product": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"enum": ["spectrum", "lightcurve", "photometry", "image"],
|
"enum": product_enum,
|
||||||
"description": "产品类型,默认 spectrum",
|
"description": "产品类型,默认 spectrum",
|
||||||
"default": "spectrum"
|
"default": "spectrum"
|
||||||
},
|
},
|
||||||
"subtype": {
|
"subtype": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "产品子类型(可选)。LAMOST spectrum: lrs/mrs;Gaia spectrum: xp_continuous/xp_sampled/rvs;Gaia lightcurve: epoch_photometry;SDSS spectrum: spec/apstar/aspcap;DESI spectrum: coadd"
|
"description": "产品子类型(可选)。各源可选值通过 GET /api/observation/capabilities 查询;LAMOST spectrum: lrs/mrs;Gaia spectrum: xp_continuous/xp_sampled/rvs;Gaia lightcurve: epoch_photometry;SDSS spectrum: spec/apstar/aspcap;DESI spectrum: coadd"
|
||||||
},
|
},
|
||||||
"ra": {
|
"ra": {
|
||||||
"type": "number",
|
"type": "number",
|
||||||
@ -80,7 +83,7 @@ impl AgentTool for FindObservationTool {
|
|||||||
},
|
},
|
||||||
"radius_deg": {
|
"radius_deg": {
|
||||||
"type": "number",
|
"type": "number",
|
||||||
"description": "检索半径(度),坐标模式用,默认 0.1。各源无硬性 API 上限,但建议值:LAMOST≤5°,Gaia/SDSS/DESI≤1°(主表很大,超出易超时)。硬性上限 30°,超出报错。可通过 GET /api/observation/capabilities 查询各源建议值",
|
"description": "检索半径(度),坐标模式用,默认 0.1。建议值与硬上限可通过 GET /api/observation/capabilities 查询(LAMOST 建议≤5°,其他≤1°;硬上限 30°)",
|
||||||
"default": 0.1
|
"default": 0.1
|
||||||
},
|
},
|
||||||
"strategy": {
|
"strategy": {
|
||||||
@ -96,7 +99,11 @@ impl AgentTool for FindObservationTool {
|
|||||||
},
|
},
|
||||||
"release": {
|
"release": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "数据发布版本(可选,留空用各源默认)。可选值与默认值可通过 GET /api/observation/capabilities 查询;LAMOST: dr5..dr11(默认dr10,MRS 需 dr7+);Gaia: dr3;SDSS spec: dr16/dr17(默认dr17);SDSS apstar/aspcap: dr17;DESI: edr/dr1(默认dr1)"
|
"description": "数据发布版本(可选,留空用各源默认)。可选值与默认值通过 GET /api/observation/capabilities 查询"
|
||||||
|
},
|
||||||
|
"version": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "数据发布的子版本(可选,仅 LAMOST 有意义)。留空用该 DR 的默认(最新公开)子版本。可选值通过 GET /api/observation/capabilities 的 release_versions 字段查询"
|
||||||
},
|
},
|
||||||
"force": {
|
"force": {
|
||||||
"type": "boolean",
|
"type": "boolean",
|
||||||
@ -124,36 +131,41 @@ impl AgentTool for FindObservationTool {
|
|||||||
let state = &ctx.app_state;
|
let state = &ctx.app_state;
|
||||||
|
|
||||||
let source = match args.get("source").and_then(|v| v.as_str()) {
|
let source = match args.get("source").and_then(|v| v.as_str()) {
|
||||||
Some(s) => match s.to_lowercase().as_str() {
|
Some(s) => match Source::from_str(s) {
|
||||||
"lamost" => Source::Lamost,
|
Ok(src) => src,
|
||||||
"gaia" => Source::Gaia,
|
Err(e) => return ToolOutput::error(e),
|
||||||
"sdss" => Source::Sdss,
|
|
||||||
"desi" => Source::Desi,
|
|
||||||
other => return ToolOutput::error(format!(
|
|
||||||
"不支持的 source '{}',可选: {:?}", other, Source::valid_values()
|
|
||||||
)),
|
|
||||||
},
|
},
|
||||||
None => return ToolOutput::error("缺少必需参数 'source'(lamost/gaia/sdss/desi)"),
|
None => return ToolOutput::error("缺少必需参数 'source'(lamost/gaia/sdss/desi)"),
|
||||||
};
|
};
|
||||||
let product_type = match args.get("product").and_then(|v| v.as_str()) {
|
let product_type = match args.get("product").and_then(|v| v.as_str()) {
|
||||||
None | Some("spectrum") => ProductType::Spectrum,
|
None => ProductType::Spectrum,
|
||||||
Some("lightcurve") | Some("light_curve") | Some("lc") => ProductType::LightCurve,
|
Some(s) => match ProductType::from_str(s) {
|
||||||
Some("photometry") => ProductType::Photometry,
|
Ok(p) => p,
|
||||||
Some("image") => ProductType::Image,
|
Err(e) => return ToolOutput::error(e),
|
||||||
Some(other) => return ToolOutput::error(format!(
|
},
|
||||||
"不支持的 product '{}',可选: {:?}", other, ProductType::valid_values()
|
|
||||||
)),
|
|
||||||
};
|
};
|
||||||
let subtype = args.get("subtype")
|
let subtype = args
|
||||||
|
.get("subtype")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.map(|s| s.to_string());
|
.map(|s| s.to_string());
|
||||||
let product = ProductSpec { product: product_type, subtype };
|
let product = ProductSpec {
|
||||||
|
product: product_type,
|
||||||
|
subtype,
|
||||||
|
};
|
||||||
let force = args.get("force").and_then(|v| v.as_bool()).unwrap_or(false);
|
let force = args.get("force").and_then(|v| v.as_bool()).unwrap_or(false);
|
||||||
let release = args.get("release").and_then(|v| v.as_str()).map(|s| s.to_string());
|
let release = args
|
||||||
|
.get("release")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
let version = args
|
||||||
|
.get("version")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
|
||||||
// 标识符模式 vs 坐标模式
|
// 标识符模式 vs 坐标模式
|
||||||
let request = if let Some(ids) = args.get("source_ids").and_then(|v| v.as_array()) {
|
let request = if let Some(ids) = args.get("source_ids").and_then(|v| v.as_array()) {
|
||||||
let identifiers: Vec<String> = ids.iter()
|
let identifiers: Vec<String> = ids
|
||||||
|
.iter()
|
||||||
.filter_map(|v| v.as_str().map(|s| s.to_string()))
|
.filter_map(|v| v.as_str().map(|s| s.to_string()))
|
||||||
.collect();
|
.collect();
|
||||||
if identifiers.is_empty() {
|
if identifiers.is_empty() {
|
||||||
@ -161,23 +173,38 @@ impl AgentTool for FindObservationTool {
|
|||||||
}
|
}
|
||||||
info!(
|
info!(
|
||||||
"[FindObservation] by_id source={:?} product={:?} count={}",
|
"[FindObservation] by_id source={:?} product={:?} count={}",
|
||||||
source, product.product, identifiers.len()
|
source,
|
||||||
|
product.product,
|
||||||
|
identifiers.len()
|
||||||
);
|
);
|
||||||
ObservationRequest::ByIdentifier { source, product, identifiers, release }
|
ObservationRequest::ByIdentifier {
|
||||||
|
source,
|
||||||
|
product,
|
||||||
|
identifiers,
|
||||||
|
release,
|
||||||
|
version,
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
let ra = match args.get("ra").and_then(|v| v.as_f64()) {
|
let ra = match args.get("ra").and_then(|v| v.as_f64()) {
|
||||||
Some(v) => v,
|
Some(v) => v,
|
||||||
None => return ToolOutput::error(
|
None => {
|
||||||
|
return ToolOutput::error(
|
||||||
"坐标模式缺少必需参数 'ra'(赤经,度),或改用 source_ids 标识符模式",
|
"坐标模式缺少必需参数 'ra'(赤经,度),或改用 source_ids 标识符模式",
|
||||||
),
|
)
|
||||||
|
}
|
||||||
};
|
};
|
||||||
let dec = match args.get("dec").and_then(|v| v.as_f64()) {
|
let dec = match args.get("dec").and_then(|v| v.as_f64()) {
|
||||||
Some(v) => v,
|
Some(v) => v,
|
||||||
None => return ToolOutput::error(
|
None => {
|
||||||
|
return ToolOutput::error(
|
||||||
"坐标模式缺少必需参数 'dec'(赤纬,度),或改用 source_ids 标识符模式",
|
"坐标模式缺少必需参数 'dec'(赤纬,度),或改用 source_ids 标识符模式",
|
||||||
),
|
)
|
||||||
|
}
|
||||||
};
|
};
|
||||||
let radius = args.get("radius_deg").and_then(|v| v.as_f64()).unwrap_or(0.1);
|
let radius = args
|
||||||
|
.get("radius_deg")
|
||||||
|
.and_then(|v| v.as_f64())
|
||||||
|
.unwrap_or(0.1);
|
||||||
let strategy = match args.get("strategy").and_then(|v| v.as_str()) {
|
let strategy = match args.get("strategy").and_then(|v| v.as_str()) {
|
||||||
Some("all") => FindStrategy::All,
|
Some("all") => FindStrategy::All,
|
||||||
_ => FindStrategy::Nearest,
|
_ => FindStrategy::Nearest,
|
||||||
@ -187,7 +214,14 @@ impl AgentTool for FindObservationTool {
|
|||||||
source, product.product, ra, dec, radius, strategy
|
source, product.product, ra, dec, radius, strategy
|
||||||
);
|
);
|
||||||
ObservationRequest::ByCoordinates {
|
ObservationRequest::ByCoordinates {
|
||||||
source, product, ra, dec, radius_deg: radius, strategy, release,
|
source,
|
||||||
|
product,
|
||||||
|
ra,
|
||||||
|
dec,
|
||||||
|
radius_deg: radius,
|
||||||
|
strategy,
|
||||||
|
release,
|
||||||
|
version,
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -203,7 +237,11 @@ impl AgentTool for FindObservationTool {
|
|||||||
|
|
||||||
/// 渲染 ObservationBatch 为可读文本(支持多 artifact 展示)
|
/// 渲染 ObservationBatch 为可读文本(支持多 artifact 展示)
|
||||||
fn render_batch(b: &crate::services::observation::ObservationBatch) -> String {
|
fn render_batch(b: &crate::services::observation::ObservationBatch) -> String {
|
||||||
let mut content = format!("{} {} 下载", b.source.display(), b.product.product.display());
|
let mut content = format!(
|
||||||
|
"{} {} 下载",
|
||||||
|
b.source.display(),
|
||||||
|
b.product.product.display()
|
||||||
|
);
|
||||||
if let (Some(ra), Some(dec), Some(r)) = (b.ra, b.dec, b.radius_deg) {
|
if let (Some(ra), Some(dec), Some(r)) = (b.ra, b.dec, b.radius_deg) {
|
||||||
content.push_str(&format!("(ra={}, dec={}, radius={}°)", ra, dec, r));
|
content.push_str(&format!("(ra={}, dec={}, radius={}°)", ra, dec, r));
|
||||||
}
|
}
|
||||||
@ -219,18 +257,30 @@ fn render_batch(b: &crate::services::observation::ObservationBatch) -> String {
|
|||||||
|
|
||||||
for p in &b.products {
|
for p in &b.products {
|
||||||
// 多 artifact 展示(如 Gaia 光变 G/BP/RP 三波段)
|
// 多 artifact 展示(如 Gaia 光变 G/BP/RP 三波段)
|
||||||
let artifact_summary: Vec<String> = p.artifacts.iter().map(|a| {
|
let artifact_summary: Vec<String> = p
|
||||||
match &a.band {
|
.artifacts
|
||||||
Some(band) => format!("{}波段 {} ({})",
|
.iter()
|
||||||
band, a.file_format.to_uppercase(), format_size(a.size_bytes)),
|
.map(|a| match &a.band {
|
||||||
None => format!("{} ({})",
|
Some(band) => format!(
|
||||||
a.file_format.to_uppercase(), format_size(a.size_bytes)),
|
"{}波段 {} ({})",
|
||||||
}
|
band,
|
||||||
}).collect();
|
a.file_format.to_uppercase(),
|
||||||
|
format_size(a.size_bytes)
|
||||||
|
),
|
||||||
|
None => format!(
|
||||||
|
"{} ({})",
|
||||||
|
a.file_format.to_uppercase(),
|
||||||
|
format_size(a.size_bytes)
|
||||||
|
),
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
content.push_str(&format!(
|
content.push_str(&format!(
|
||||||
"\n✓ 已下载({}): {} —— {}\n",
|
"\n✓ 已下载({}): {} —— {}\n",
|
||||||
if p.artifacts.iter().all(|a| a.cached) { "缓存" } else { "新下载" }
|
if p.artifacts.iter().all(|a| a.cached) {
|
||||||
.to_string(),
|
"缓存"
|
||||||
|
} else {
|
||||||
|
"新下载"
|
||||||
|
},
|
||||||
p.source_label,
|
p.source_label,
|
||||||
artifact_summary.join(", ")
|
artifact_summary.join(", ")
|
||||||
));
|
));
|
||||||
@ -238,7 +288,10 @@ fn render_batch(b: &crate::services::observation::ObservationBatch) -> String {
|
|||||||
content.push_str(&format!(
|
content.push_str(&format!(
|
||||||
" {}{}\n",
|
" {}{}\n",
|
||||||
a.file_path,
|
a.file_path,
|
||||||
a.band.as_ref().map(|b| format!(" [{}]", b)).unwrap_or_default()
|
a.band
|
||||||
|
.as_ref()
|
||||||
|
.map(|b| format!(" [{}]", b))
|
||||||
|
.unwrap_or_default()
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -185,10 +185,7 @@ impl AgentTool for CatalogOperationTool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn do_catalog_search(
|
async fn do_catalog_search(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
|
||||||
state: &crate::api::AppState,
|
|
||||||
args: &serde_json::Value,
|
|
||||||
) -> ToolOutput {
|
|
||||||
let keyword = match args.get("keyword").and_then(|v| v.as_str()) {
|
let keyword = match args.get("keyword").and_then(|v| v.as_str()) {
|
||||||
Some(k) => k,
|
Some(k) => k,
|
||||||
None => return ToolOutput::error("search 需要 'keyword' 参数"),
|
None => return ToolOutput::error("search 需要 'keyword' 参数"),
|
||||||
@ -226,10 +223,7 @@ async fn do_catalog_search(
|
|||||||
ToolOutput::success(content, json!(items))
|
ToolOutput::success(content, json!(items))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn do_catalog_describe(
|
async fn do_catalog_describe(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
|
||||||
state: &crate::api::AppState,
|
|
||||||
args: &serde_json::Value,
|
|
||||||
) -> ToolOutput {
|
|
||||||
let table = match args.get("table").and_then(|v| v.as_str()) {
|
let table = match args.get("table").and_then(|v| v.as_str()) {
|
||||||
Some(t) => t,
|
Some(t) => t,
|
||||||
None => return ToolOutput::error("describe 需要 'table' 参数"),
|
None => return ToolOutput::error("describe 需要 'table' 参数"),
|
||||||
@ -259,10 +253,7 @@ async fn do_catalog_describe(
|
|||||||
ToolOutput::success(content, json!(items))
|
ToolOutput::success(content, json!(items))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn do_catalog_query(
|
async fn do_catalog_query(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
|
||||||
state: &crate::api::AppState,
|
|
||||||
args: &serde_json::Value,
|
|
||||||
) -> ToolOutput {
|
|
||||||
let limit = args
|
let limit = args
|
||||||
.get("limit")
|
.get("limit")
|
||||||
.and_then(|v| v.as_i64())
|
.and_then(|v| v.as_i64())
|
||||||
@ -275,8 +266,7 @@ async fn do_catalog_query(
|
|||||||
limit,
|
limit,
|
||||||
adql.chars().take(150).collect::<String>()
|
adql.chars().take(150).collect::<String>()
|
||||||
);
|
);
|
||||||
crate::services::cds::vizier::query_adql_cached(&state.db, &state.vizier, adql, limit)
|
crate::services::cds::vizier::query_adql_cached(&state.db, &state.vizier, adql, limit).await
|
||||||
.await
|
|
||||||
} else if let Some(table) = args.get("table").and_then(|v| v.as_str()) {
|
} else if let Some(table) = args.get("table").and_then(|v| v.as_str()) {
|
||||||
let columns: Vec<String> = args
|
let columns: Vec<String> = args
|
||||||
.get("columns")
|
.get("columns")
|
||||||
@ -284,13 +274,7 @@ async fn do_catalog_query(
|
|||||||
.map(|c| c.split(',').map(|s| s.trim().to_string()).collect())
|
.map(|c| c.split(',').map(|s| s.trim().to_string()).collect())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
info!("[CatalogOp:query] table={} limit={}", table, limit);
|
info!("[CatalogOp:query] table={} limit={}", table, limit);
|
||||||
crate::services::cds::vizier::query_table(
|
crate::services::cds::vizier::query_table(&state.db, &state.vizier, table, &columns, limit)
|
||||||
&state.db,
|
|
||||||
&state.vizier,
|
|
||||||
table,
|
|
||||||
&columns,
|
|
||||||
limit,
|
|
||||||
)
|
|
||||||
.await
|
.await
|
||||||
} else {
|
} else {
|
||||||
return ToolOutput::error("query 需要 'adql' 或 'table' 参数之一");
|
return ToolOutput::error("query 需要 'adql' 或 'table' 参数之一");
|
||||||
@ -305,10 +289,7 @@ async fn do_catalog_query(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn do_catalog_cone(
|
async fn do_catalog_cone(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
|
||||||
state: &crate::api::AppState,
|
|
||||||
args: &serde_json::Value,
|
|
||||||
) -> ToolOutput {
|
|
||||||
let coords = match args.get("coords").and_then(|v| v.as_object()) {
|
let coords = match args.get("coords").and_then(|v| v.as_object()) {
|
||||||
Some(c) => c,
|
Some(c) => c,
|
||||||
None => return ToolOutput::error("cone 需要 'coords' 参数(含 ra/dec)"),
|
None => return ToolOutput::error("cone 需要 'coords' 参数(含 ra/dec)"),
|
||||||
@ -338,7 +319,11 @@ async fn do_catalog_cone(
|
|||||||
|
|
||||||
info!(
|
info!(
|
||||||
"[CatalogOp:cone] ra={} dec={} radius={}° table={} strategy={}",
|
"[CatalogOp:cone] ra={} dec={} radius={}° table={} strategy={}",
|
||||||
ra, dec, radius, table, if nearest { "nearest" } else { "all" }
|
ra,
|
||||||
|
dec,
|
||||||
|
radius,
|
||||||
|
table,
|
||||||
|
if nearest { "nearest" } else { "all" }
|
||||||
);
|
);
|
||||||
|
|
||||||
match crate::services::cds::vizier::cone_search(
|
match crate::services::cds::vizier::cone_search(
|
||||||
@ -365,10 +350,7 @@ async fn do_catalog_cone(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn do_catalog_export(
|
async fn do_catalog_export(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
|
||||||
state: &crate::api::AppState,
|
|
||||||
args: &serde_json::Value,
|
|
||||||
) -> ToolOutput {
|
|
||||||
let limit = args
|
let limit = args
|
||||||
.get("limit")
|
.get("limit")
|
||||||
.and_then(|v| v.as_i64())
|
.and_then(|v| v.as_i64())
|
||||||
@ -384,13 +366,16 @@ async fn do_catalog_export(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let catalog = crate::services::cds::vizier::VizierCatalog::new(&state.db, &state.vizier);
|
let catalog = crate::services::cds::vizier::VizierCatalog::new(&state.db, &state.vizier);
|
||||||
let result = match catalog.export_to_file(
|
let result = match catalog
|
||||||
|
.export_to_file(
|
||||||
state.config.library_dir.to_str().unwrap_or("."),
|
state.config.library_dir.to_str().unwrap_or("."),
|
||||||
adql,
|
adql,
|
||||||
table,
|
table,
|
||||||
columns,
|
columns,
|
||||||
limit,
|
limit,
|
||||||
).await {
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(r) => r,
|
Ok(r) => r,
|
||||||
Err(e) => return ToolOutput::error(format!("导出失败: {}", e)),
|
Err(e) => return ToolOutput::error(format!("导出失败: {}", e)),
|
||||||
};
|
};
|
||||||
@ -410,10 +395,7 @@ async fn do_catalog_export(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn do_catalog_lookup(
|
async fn do_catalog_lookup(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
|
||||||
state: &crate::api::AppState,
|
|
||||||
args: &serde_json::Value,
|
|
||||||
) -> ToolOutput {
|
|
||||||
let bibcode = match args.get("bibcode").and_then(|v| v.as_str()) {
|
let bibcode = match args.get("bibcode").and_then(|v| v.as_str()) {
|
||||||
Some(b) => b,
|
Some(b) => b,
|
||||||
None => return ToolOutput::error("lookup 需要 'bibcode' 参数"),
|
None => return ToolOutput::error("lookup 需要 'bibcode' 参数"),
|
||||||
@ -424,11 +406,8 @@ async fn do_catalog_lookup(
|
|||||||
.unwrap_or(10)
|
.unwrap_or(10)
|
||||||
.clamp(1, 30) as usize;
|
.clamp(1, 30) as usize;
|
||||||
|
|
||||||
let catalog = crate::services::cds::vizier::VizierCatalog::with_ads(
|
let catalog =
|
||||||
&state.db,
|
crate::services::cds::vizier::VizierCatalog::with_ads(&state.db, &state.vizier, &state.ads);
|
||||||
&state.vizier,
|
|
||||||
&state.ads,
|
|
||||||
);
|
|
||||||
let results = match catalog.lookup(bibcode, limit).await {
|
let results = match catalog.lookup(bibcode, limit).await {
|
||||||
Ok(r) => r,
|
Ok(r) => r,
|
||||||
Err(e) => return ToolOutput::error(format!("查找失败: {}", e)),
|
Err(e) => return ToolOutput::error(format!("查找失败: {}", e)),
|
||||||
|
|||||||
@ -245,7 +245,12 @@ impl AgentTool for FileEditTool {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let original = match std::fs::read_to_string(&file_path) {
|
let read_path = file_path.clone();
|
||||||
|
let original =
|
||||||
|
match tokio::task::spawn_blocking(move || std::fs::read_to_string(&read_path))
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
|
||||||
|
{
|
||||||
Ok(c) => c,
|
Ok(c) => c,
|
||||||
Err(e) => return ToolOutput::error(format!("读取文件失败: {}", e)),
|
Err(e) => return ToolOutput::error(format!("读取文件失败: {}", e)),
|
||||||
};
|
};
|
||||||
@ -317,7 +322,12 @@ impl AgentTool for FileEditTool {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
match std::fs::write(&file_path, &modified) {
|
let write_path = file_path.clone();
|
||||||
|
let write_content = modified.clone();
|
||||||
|
match tokio::task::spawn_blocking(move || std::fs::write(&write_path, &write_content))
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
|
||||||
|
{
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
let count = if replace_all { match_count } else { 1 };
|
let count = if replace_all { match_count } else { 1 };
|
||||||
info!("[FileEdit] {} 处替换: {}", count, file_path.display());
|
info!("[FileEdit] {} 处替换: {}", count, file_path.display());
|
||||||
|
|||||||
@ -89,20 +89,27 @@ impl AgentTool for GrepFilesTool {
|
|||||||
Err(e) => return ToolOutput::error(format!("正则表达式无效: {}", e)),
|
Err(e) => return ToolOutput::error(format!("正则表达式无效: {}", e)),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let search_re = re.clone();
|
||||||
|
let search_inc = include_pattern.map(|s| s.to_string());
|
||||||
|
let search_p = search_path.clone();
|
||||||
|
let (results, count) = tokio::task::spawn_blocking(move || {
|
||||||
let mut results = Vec::new();
|
let mut results = Vec::new();
|
||||||
let mut count = 0;
|
let mut count = 0;
|
||||||
|
if search_p.is_file() {
|
||||||
if search_path.is_file() {
|
count += Self::search_file(&search_p, &search_re, &mut results);
|
||||||
count += Self::search_file(&search_path, &re, &mut results);
|
} else if search_p.is_dir() {
|
||||||
} else if search_path.is_dir() {
|
|
||||||
count += Self::search_dir(
|
count += Self::search_dir(
|
||||||
&search_path,
|
&search_p,
|
||||||
&re,
|
&search_re,
|
||||||
include_pattern,
|
search_inc.as_deref(),
|
||||||
max_results,
|
max_results,
|
||||||
&mut results,
|
&mut results,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
(results, count)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|_| (Vec::new(), 0));
|
||||||
|
|
||||||
if results.is_empty() {
|
if results.is_empty() {
|
||||||
ToolOutput::success(
|
ToolOutput::success(
|
||||||
|
|||||||
@ -113,18 +113,25 @@ impl AgentTool for ReadFileTool {
|
|||||||
}
|
}
|
||||||
// ── 去重检查结束 ──
|
// ── 去重检查结束 ──
|
||||||
|
|
||||||
// 预检文件大小,避免 OOM
|
// 预检文件大小 + 读取文件内容(spawn_blocking 避免阻塞 tokio)
|
||||||
const MAX_READ_BYTES: u64 = 10 * 1024 * 1024; // 10MB
|
const MAX_READ_BYTES: u64 = 10 * 1024 * 1024; // 10MB
|
||||||
if let Ok(meta) = std::fs::metadata(&path) {
|
let read_path = path.clone();
|
||||||
if meta.len() > MAX_READ_BYTES {
|
let read_result = tokio::task::spawn_blocking(move || {
|
||||||
return ToolOutput::error(format!(
|
let meta = std::fs::metadata(&read_path).ok();
|
||||||
|
if let Some(ref m) = meta {
|
||||||
|
if m.len() > MAX_READ_BYTES {
|
||||||
|
return Err(format!(
|
||||||
"文件过大 ({} MB),超过 10MB 限制。请使用 read_file 的 offset/limit 参数分段读取。",
|
"文件过大 ({} MB),超过 10MB 限制。请使用 read_file 的 offset/limit 参数分段读取。",
|
||||||
meta.len() / (1024 * 1024)
|
m.len() / (1024 * 1024)
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
std::fs::read_to_string(&read_path).map_err(|e| format!("读取文件失败: {}", e))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|e| Err(format!("任务执行失败: {}", e)));
|
||||||
|
|
||||||
match std::fs::read_to_string(&path) {
|
match read_result {
|
||||||
Ok(content) => {
|
Ok(content) => {
|
||||||
let total_lines = content.lines().count();
|
let total_lines = content.lines().count();
|
||||||
let truncated = if max_lines < total_lines {
|
let truncated = if max_lines < total_lines {
|
||||||
@ -141,8 +148,6 @@ impl AgentTool for ReadFileTool {
|
|||||||
content.clone()
|
content.clone()
|
||||||
};
|
};
|
||||||
|
|
||||||
// read_file 不截断输出——LLM 明确要求读取完整内容,
|
|
||||||
// 且 skip_persist 确保不会被二次持久化。
|
|
||||||
let content_len = truncated.len();
|
let content_len = truncated.len();
|
||||||
info!(
|
info!(
|
||||||
"[ReadFile] 读取 {}: {} 字符 ({} 行)",
|
"[ReadFile] 读取 {}: {} 字符 ({} 行)",
|
||||||
@ -163,14 +168,13 @@ impl AgentTool for ReadFileTool {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
// ── 缓存更新结束 ──
|
|
||||||
|
|
||||||
ToolOutput::success_skip_persist(
|
ToolOutput::success_skip_persist(
|
||||||
truncated,
|
truncated,
|
||||||
json!({"file_path": file_path_str, "total_lines": total_lines}),
|
json!({"file_path": file_path_str, "total_lines": total_lines}),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
Err(e) => ToolOutput::error(format!("读取文件失败: {}", e)),
|
Err(e) => ToolOutput::error(e),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -69,12 +69,21 @@ impl AgentTool for FileWriteTool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some(parent) = path.parent() {
|
if let Some(parent) = path.parent() {
|
||||||
if let Err(e) = std::fs::create_dir_all(parent) {
|
let parent = parent.to_path_buf();
|
||||||
|
if let Err(e) = tokio::task::spawn_blocking(move || std::fs::create_dir_all(&parent))
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
|
||||||
|
{
|
||||||
return ToolOutput::error(format!("创建父目录失败: {}", e));
|
return ToolOutput::error(format!("创建父目录失败: {}", e));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
match std::fs::write(&path, &content) {
|
let write_path = path.clone();
|
||||||
|
let write_content = content.clone();
|
||||||
|
match tokio::task::spawn_blocking(move || std::fs::write(&write_path, &write_content))
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
|
||||||
|
{
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
let line_count = content.lines().count();
|
let line_count = content.lines().count();
|
||||||
info!("[FileWrite] 写入 {}: {} 行", file_path_str, line_count);
|
info!("[FileWrite] 写入 {}: {} 行", file_path_str, line_count);
|
||||||
|
|||||||
@ -17,7 +17,7 @@ use tracing::info;
|
|||||||
///
|
///
|
||||||
/// 返回 (最终内容, 持久化文件路径)。
|
/// 返回 (最终内容, 持久化文件路径)。
|
||||||
/// 如果内容未超过限制,直接返回原内容且持久化路径为 None。
|
/// 如果内容未超过限制,直接返回原内容且持久化路径为 None。
|
||||||
pub fn maybe_persist_tool_result(
|
pub async fn maybe_persist_tool_result(
|
||||||
content: &str,
|
content: &str,
|
||||||
tool_call_id: &str,
|
tool_call_id: &str,
|
||||||
max_chars: usize,
|
max_chars: usize,
|
||||||
@ -28,8 +28,12 @@ pub fn maybe_persist_tool_result(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 创建目录(幂等)
|
// 创建目录(幂等)
|
||||||
if let Err(_e) = std::fs::create_dir_all(tool_results_dir) {
|
let dir = tool_results_dir.to_path_buf();
|
||||||
// 无法创建目录则直接截断(不持久化)
|
if tokio::task::spawn_blocking(move || std::fs::create_dir_all(&dir))
|
||||||
|
.await
|
||||||
|
.unwrap_or(Ok(()))
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
let truncated: String = content.chars().take(max_chars).collect();
|
let truncated: String = content.chars().take(max_chars).collect();
|
||||||
return (
|
return (
|
||||||
format!(
|
format!(
|
||||||
@ -60,19 +64,20 @@ pub fn maybe_persist_tool_result(
|
|||||||
let file_path = tool_results_dir.join(format!("{}.txt", safe_id));
|
let file_path = tool_results_dir.join(format!("{}.txt", safe_id));
|
||||||
|
|
||||||
// 独占创建——只在文件不存在时写入,保证幂等性
|
// 独占创建——只在文件不存在时写入,保证幂等性
|
||||||
let written = match std::fs::OpenOptions::new()
|
let fp = file_path.clone();
|
||||||
|
let content_clone = content.to_string();
|
||||||
|
let written = tokio::task::spawn_blocking(move || {
|
||||||
|
match std::fs::OpenOptions::new()
|
||||||
.write(true)
|
.write(true)
|
||||||
.create_new(true)
|
.create_new(true)
|
||||||
.open(&file_path)
|
.open(&fp)
|
||||||
{
|
{
|
||||||
Ok(_) => {
|
Ok(_) => match std::fs::write(&fp, &content_clone) {
|
||||||
// 文件创建成功,写入内容
|
|
||||||
match std::fs::write(&file_path, content) {
|
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
info!(
|
info!(
|
||||||
"[Persist] 工具结果已持久化: {} ({} 字符)",
|
"[Persist] 工具结果已持久化: {} ({} 字符)",
|
||||||
file_path.display(),
|
fp.display(),
|
||||||
content.len()
|
content_clone.len()
|
||||||
);
|
);
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
@ -80,18 +85,19 @@ pub fn maybe_persist_tool_result(
|
|||||||
tracing::warn!("[Persist] 写入失败: {}", e);
|
tracing::warn!("[Persist] 写入失败: {}", e);
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
}
|
|
||||||
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
|
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
|
||||||
// 文件已存在,无需重复写入
|
info!("[Persist] 工具结果已存在,跳过: {}", fp.display());
|
||||||
info!("[Persist] 工具结果已存在,跳过: {}", file_path.display());
|
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!("[Persist] 创建文件失败: {}", e);
|
tracing::warn!("[Persist] 创建文件失败: {}", e);
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
if !written {
|
if !written {
|
||||||
// 持久化失败,回退到截断
|
// 持久化失败,回退到截断
|
||||||
@ -148,19 +154,19 @@ mod tests {
|
|||||||
(dir, Cleanup(dir_clone))
|
(dir, Cleanup(dir_clone))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn test_small_content_not_persisted() {
|
async fn test_small_content_not_persisted() {
|
||||||
let (dir, _cleanup) = temp_dir();
|
let (dir, _cleanup) = temp_dir();
|
||||||
let (content, path) = maybe_persist_tool_result("small result", "call_1", 4000, &dir);
|
let (content, path) = maybe_persist_tool_result("small result", "call_1", 4000, &dir).await;
|
||||||
assert_eq!(content, "small result");
|
assert_eq!(content, "small result");
|
||||||
assert!(path.is_none());
|
assert!(path.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn test_large_content_persisted() {
|
async fn test_large_content_persisted() {
|
||||||
let (dir, _cleanup) = temp_dir();
|
let (dir, _cleanup) = temp_dir();
|
||||||
let large = "x".repeat(5000);
|
let large = "x".repeat(5000);
|
||||||
let (content, path) = maybe_persist_tool_result(&large, "call_2", 100, &dir);
|
let (content, path) = maybe_persist_tool_result(&large, "call_2", 100, &dir).await;
|
||||||
assert!(content.contains("<persisted-output>"));
|
assert!(content.contains("<persisted-output>"));
|
||||||
assert!(content.contains("call_2.txt"));
|
assert!(content.contains("call_2.txt"));
|
||||||
assert!(path.is_some());
|
assert!(path.is_some());
|
||||||
@ -170,14 +176,14 @@ mod tests {
|
|||||||
assert_eq!(written, large);
|
assert_eq!(written, large);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn test_idempotent_write() {
|
async fn test_idempotent_write() {
|
||||||
let (dir, _cleanup) = temp_dir();
|
let (dir, _cleanup) = temp_dir();
|
||||||
let large1 = "a".repeat(5000);
|
let large1 = "a".repeat(5000);
|
||||||
let large2 = "b".repeat(5000);
|
let large2 = "b".repeat(5000);
|
||||||
|
|
||||||
let (_, path1) = maybe_persist_tool_result(&large1, "call_3", 100, &dir);
|
let (_, path1) = maybe_persist_tool_result(&large1, "call_3", 100, &dir).await;
|
||||||
let (_content2, path2) = maybe_persist_tool_result(&large2, "call_3", 100, &dir);
|
let (_content2, path2) = maybe_persist_tool_result(&large2, "call_3", 100, &dir).await;
|
||||||
|
|
||||||
assert!(path1.is_some());
|
assert!(path1.is_some());
|
||||||
assert!(path2.is_some());
|
assert!(path2.is_some());
|
||||||
@ -185,11 +191,11 @@ mod tests {
|
|||||||
assert_eq!(written, large1);
|
assert_eq!(written, large1);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn test_preview_at_newline_boundary() {
|
async fn test_preview_at_newline_boundary() {
|
||||||
let (dir, _cleanup) = temp_dir();
|
let (dir, _cleanup) = temp_dir();
|
||||||
let large = format!("Short line\n{}", "x".repeat(5000));
|
let large = format!("Short line\n{}", "x".repeat(5000));
|
||||||
let (content, _) = maybe_persist_tool_result(&large, "call_4", 100, &dir);
|
let (content, _) = maybe_persist_tool_result(&large, "call_4", 100, &dir).await;
|
||||||
let preview_start = content.find("preview:").unwrap();
|
let preview_start = content.find("preview:").unwrap();
|
||||||
let preview_section = &content[preview_start..];
|
let preview_section = &content[preview_start..];
|
||||||
assert!(preview_section.contains("Short line"));
|
assert!(preview_section.contains("Short line"));
|
||||||
|
|||||||
@ -151,7 +151,8 @@ impl AgentTool for SendTeammateMessageTool {
|
|||||||
let tm_lock = self.team_manager.lock().await;
|
let tm_lock = self.team_manager.lock().await;
|
||||||
match tm_lock.as_ref() {
|
match tm_lock.as_ref() {
|
||||||
Some(tm) => {
|
Some(tm) => {
|
||||||
tm.send_message("lead", &to, &content, TeamMessageType::Task);
|
tm.send_message("lead", &to, &content, TeamMessageType::Task)
|
||||||
|
.await;
|
||||||
ToolOutput::success(
|
ToolOutput::success(
|
||||||
format!(
|
format!(
|
||||||
"📤 消息已发送给 {}: {}",
|
"📤 消息已发送给 {}: {}",
|
||||||
@ -214,7 +215,7 @@ impl AgentTool for TeamBroadcastTool {
|
|||||||
let tm_lock = self.team_manager.lock().await;
|
let tm_lock = self.team_manager.lock().await;
|
||||||
match tm_lock.as_ref() {
|
match tm_lock.as_ref() {
|
||||||
Some(tm) => {
|
Some(tm) => {
|
||||||
tm.broadcast("lead", &content);
|
tm.broadcast("lead", &content).await;
|
||||||
ToolOutput::success("📢 已广播消息给所有队友。", json!({}))
|
ToolOutput::success("📢 已广播消息给所有队友。", json!({}))
|
||||||
}
|
}
|
||||||
None => ToolOutput::error("团队管理器未初始化"),
|
None => ToolOutput::error("团队管理器未初始化"),
|
||||||
@ -278,7 +279,7 @@ impl AgentTool for CheckTeamInboxTool {
|
|||||||
let tm_lock = self.team_manager.lock().await;
|
let tm_lock = self.team_manager.lock().await;
|
||||||
match tm_lock.as_ref() {
|
match tm_lock.as_ref() {
|
||||||
Some(tm) => {
|
Some(tm) => {
|
||||||
let msgs = tm.check_inbox(agent);
|
let msgs = tm.check_inbox(agent).await;
|
||||||
if msgs.is_empty() {
|
if msgs.is_empty() {
|
||||||
ToolOutput::success(
|
ToolOutput::success(
|
||||||
format!("📭 {} 的收件箱为空。", agent),
|
format!("📭 {} 的收件箱为空。", agent),
|
||||||
|
|||||||
@ -259,10 +259,7 @@ mod tests {
|
|||||||
ads,
|
ads,
|
||||||
arxiv,
|
arxiv,
|
||||||
vizier,
|
vizier,
|
||||||
lamost: crate::clients::lamost::LamostClient::new(
|
lamost: crate::clients::lamost::LamostClient::new("https://www.lamost.org", 60)
|
||||||
"https://www.lamost.org",
|
|
||||||
60,
|
|
||||||
)
|
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
gaia: crate::clients::gaia::GaiaClient::new(
|
gaia: crate::clients::gaia::GaiaClient::new(
|
||||||
"https://gea.esac.esa.int/tap-server/tap",
|
"https://gea.esac.esa.int/tap-server/tap",
|
||||||
@ -290,7 +287,7 @@ mod tests {
|
|||||||
harvest_status: Arc::new(tokio::sync::Mutex::new(MetaSyncStatus::default())),
|
harvest_status: Arc::new(tokio::sync::Mutex::new(MetaSyncStatus::default())),
|
||||||
batch_status: Arc::new(tokio::sync::Mutex::new(AssetBatchStatus::default())),
|
batch_status: Arc::new(tokio::sync::Mutex::new(AssetBatchStatus::default())),
|
||||||
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
|
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
|
||||||
cancelled_runs: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())),
|
cancelled_runs: Arc::new(dashmap::DashMap::new()),
|
||||||
skill_registry: Arc::new(tokio::sync::RwLock::new(
|
skill_registry: Arc::new(tokio::sync::RwLock::new(
|
||||||
crate::agent::skills::SkillRegistry::new(PathBuf::from("/tmp/test_skills")),
|
crate::agent::skills::SkillRegistry::new(PathBuf::from("/tmp/test_skills")),
|
||||||
)),
|
)),
|
||||||
@ -298,16 +295,16 @@ mod tests {
|
|||||||
pending_permissions: Arc::new(
|
pending_permissions: Arc::new(
|
||||||
tokio::sync::Mutex::new(std::collections::HashMap::new()),
|
tokio::sync::Mutex::new(std::collections::HashMap::new()),
|
||||||
),
|
),
|
||||||
session_permission_checkers: Arc::new(tokio::sync::RwLock::new(
|
session_permission_checkers: Arc::new(dashmap::DashMap::new()),
|
||||||
std::collections::HashMap::new(),
|
|
||||||
)),
|
|
||||||
sse_broadcast: None,
|
sse_broadcast: None,
|
||||||
memory_manager: Arc::new(tokio::sync::Mutex::new(
|
memory_manager: Arc::new(tokio::sync::Mutex::new(
|
||||||
crate::agent::memory::MemoryManager::new(PathBuf::from("/tmp/test_memory")),
|
crate::agent::memory::MemoryManager::new(PathBuf::from("/tmp/test_memory")),
|
||||||
)),
|
)),
|
||||||
sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
|
sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
|
||||||
login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
||||||
upload_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
upload_rate_limiter: Arc::new(
|
||||||
|
tokio::sync::Mutex::new(std::collections::HashMap::new()),
|
||||||
|
),
|
||||||
});
|
});
|
||||||
|
|
||||||
ToolContext {
|
ToolContext {
|
||||||
|
|||||||
@ -299,8 +299,7 @@ pub async fn stop_agent(
|
|||||||
State(state): State<Arc<AppState>>,
|
State(state): State<Arc<AppState>>,
|
||||||
Path(session_id): Path<String>,
|
Path(session_id): Path<String>,
|
||||||
) -> ApiResult<Json<serde_json::Value>> {
|
) -> ApiResult<Json<serde_json::Value>> {
|
||||||
let mut cancelled = state.cancelled_runs.lock().await;
|
state.cancelled_runs.insert(session_id.clone(), ());
|
||||||
cancelled.insert(session_id.clone());
|
|
||||||
info!("已接收并记录手动中止请求,会话 ID: {}", session_id);
|
info!("已接收并记录手动中止请求,会话 ID: {}", session_id);
|
||||||
Ok(Json(
|
Ok(Json(
|
||||||
serde_json::json!({ "status": "stopping", "session_id": session_id }),
|
serde_json::json!({ "status": "stopping", "session_id": session_id }),
|
||||||
@ -412,7 +411,12 @@ pub async fn respond_permission(
|
|||||||
|
|
||||||
match perm_id {
|
match perm_id {
|
||||||
Some(id) => {
|
Some(id) => {
|
||||||
let perm = perms.remove(&id).unwrap();
|
let perm = match perms.remove(&id) {
|
||||||
|
Some(p) => p,
|
||||||
|
None => {
|
||||||
|
return Err(AppError::internal("权限请求在查找后消失"));
|
||||||
|
}
|
||||||
|
};
|
||||||
match perm.response_tx.send(req) {
|
match perm.response_tx.send(req) {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
info!(
|
info!(
|
||||||
|
|||||||
@ -124,14 +124,9 @@ pub async fn login(
|
|||||||
let mut limiter = state.login_rate_limiter.lock().await;
|
let mut limiter = state.login_rate_limiter.lock().await;
|
||||||
// 批量清理过期条目,防止内存无限增长
|
// 批量清理过期条目,防止内存无限增长
|
||||||
limiter.retain(|_, (_, first_fail)| first_fail.elapsed().as_secs() < WINDOW_SECS);
|
limiter.retain(|_, (_, first_fail)| first_fail.elapsed().as_secs() < WINDOW_SECS);
|
||||||
// 容量保护:超过上限时强制清理最旧的一半条目
|
// 容量保护:超过上限时直接清空(比 O(n log n) 排序更高效,且不影响安全性)
|
||||||
if limiter.len() > MAX_RATE_LIMITER_ENTRIES {
|
if limiter.len() > MAX_RATE_LIMITER_ENTRIES {
|
||||||
let mut entries: Vec<_> = limiter.iter().map(|(k, v)| (k.clone(), v.1)).collect();
|
limiter.clear();
|
||||||
entries.sort_by_key(|(_, t)| *t);
|
|
||||||
let cutoff = entries.len() / 2;
|
|
||||||
for (key, _) in &entries[..cutoff] {
|
|
||||||
limiter.remove(key);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some((count, first_fail)) = limiter.get(&client_ip) {
|
if let Some((count, first_fail)) = limiter.get(&client_ip) {
|
||||||
@ -242,16 +237,42 @@ pub async fn auth_middleware(
|
|||||||
let token = extract_auth_token_from_headers(req.headers());
|
let token = extract_auth_token_from_headers(req.headers());
|
||||||
|
|
||||||
let mut is_valid = false;
|
let mut is_valid = false;
|
||||||
if let Some(tok) = token {
|
if let Some(ref tok) = token {
|
||||||
// 1. 优先校验内存中的动态 Session
|
// 1. 优先校验内存中的动态 Session
|
||||||
|
// 先用读锁快速检查会话是否存在,并获取其最后活跃时间
|
||||||
|
let mut last_active_opt = None;
|
||||||
|
{
|
||||||
|
let sessions = state.sessions.read().await;
|
||||||
|
if let Some(last_active) = sessions.get(tok) {
|
||||||
|
is_valid = true;
|
||||||
|
last_active_opt = Some(*last_active);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 会话存在时,如果最后活跃时间超过 60 秒,则获取写锁更新最后活跃时间 + 过期清理
|
||||||
|
if is_valid {
|
||||||
|
let needs_update = match last_active_opt {
|
||||||
|
Some(last_active) => {
|
||||||
|
if let Ok(dur) = chrono::Utc::now()
|
||||||
|
.signed_duration_since(last_active)
|
||||||
|
.to_std()
|
||||||
|
{
|
||||||
|
dur.as_secs() > 60
|
||||||
|
} else {
|
||||||
|
true // 时钟回拨等异常情况,强制更新
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => true,
|
||||||
|
};
|
||||||
|
|
||||||
|
if needs_update {
|
||||||
let mut sessions = state.sessions.write().await;
|
let mut sessions = state.sessions.write().await;
|
||||||
// 顺便执行过期会话自动清理垃圾收集
|
// 顺便执行过期会话自动清理垃圾收集
|
||||||
prune_expired_sessions(&mut sessions);
|
prune_expired_sessions(&mut sessions);
|
||||||
|
if let Some(last_active) = sessions.get_mut(tok) {
|
||||||
if let Some(last_active) = sessions.get_mut(&tok) {
|
|
||||||
// 会话存在且未过期,更新最后活跃时间
|
|
||||||
*last_active = chrono::Utc::now();
|
*last_active = chrono::Utc::now();
|
||||||
is_valid = true;
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. 校验书签专属永久 API 密钥(严格隔离限制:仅允许用于特定的书签导入和同步路径)
|
// 2. 校验书签专属永久 API 密钥(严格隔离限制:仅允许用于特定的书签导入和同步路径)
|
||||||
@ -259,7 +280,7 @@ pub async fn auth_middleware(
|
|||||||
let path = req.uri().path();
|
let path = req.uri().path();
|
||||||
if path == "/api/upload" || path == "/api/active_bibcode" {
|
if path == "/api/upload" || path == "/api/active_bibcode" {
|
||||||
let expected_key = get_bookmarklet_api_key(&state.config.admin_password);
|
let expected_key = get_bookmarklet_api_key(&state.config.admin_password);
|
||||||
if tok == expected_key {
|
if *tok == expected_key {
|
||||||
is_valid = true;
|
is_valid = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -85,18 +85,15 @@ pub struct AppState {
|
|||||||
pub harvest_status: Arc<tokio::sync::Mutex<crate::services::batch::MetaSyncStatus>>,
|
pub harvest_status: Arc<tokio::sync::Mutex<crate::services::batch::MetaSyncStatus>>,
|
||||||
pub batch_status: Arc<tokio::sync::Mutex<crate::services::batch::AssetBatchStatus>>,
|
pub batch_status: Arc<tokio::sync::Mutex<crate::services::batch::AssetBatchStatus>>,
|
||||||
pub active_bibcode: Arc<tokio::sync::Mutex<Option<String>>>,
|
pub active_bibcode: Arc<tokio::sync::Mutex<Option<String>>>,
|
||||||
pub cancelled_runs: Arc<Mutex<std::collections::HashSet<String>>>,
|
pub cancelled_runs: Arc<dashmap::DashMap<String, ()>>,
|
||||||
pub skill_registry: Arc<RwLock<SkillRegistry>>,
|
pub skill_registry: Arc<RwLock<SkillRegistry>>,
|
||||||
/// ask_user 工具 — 待回答的问题
|
/// ask_user 工具 — 待回答的问题
|
||||||
pub pending_questions: Arc<Mutex<HashMap<String, PendingQuestion>>>,
|
pub pending_questions: Arc<Mutex<HashMap<String, PendingQuestion>>>,
|
||||||
/// 权限检查 — 待处理的权限确认请求
|
/// 权限检查 — 待处理的权限确认请求
|
||||||
pub pending_permissions: Arc<Mutex<HashMap<String, PendingPermission>>>,
|
pub pending_permissions: Arc<Mutex<HashMap<String, PendingPermission>>>,
|
||||||
/// 会话级权限检查器注册表(按 session_id 隔离,支持 API 动态添加/移除规则)
|
/// 会话级权限检查器注册表(按 session_id 隔离,支持 API 动态添加/移除规则)
|
||||||
pub session_permission_checkers: Arc<
|
pub session_permission_checkers:
|
||||||
RwLock<
|
Arc<dashmap::DashMap<String, crate::agent::runtime::permission::PermissionChecker>>,
|
||||||
std::collections::HashMap<String, crate::agent::runtime::permission::PermissionChecker>,
|
|
||||||
>,
|
|
||||||
>,
|
|
||||||
/// SSE 广播通道(agent 运行时向所有连接的客户端推送事件)
|
/// SSE 广播通道(agent 运行时向所有连接的客户端推送事件)
|
||||||
pub sse_broadcast: Option<broadcast::Sender<AppEvent>>,
|
pub sse_broadcast: Option<broadcast::Sender<AppEvent>>,
|
||||||
/// 项目记忆管理器(跨会话持久化)
|
/// 项目记忆管理器(跨会话持久化)
|
||||||
|
|||||||
@ -31,47 +31,15 @@ fn default_radius() -> f64 {
|
|||||||
0.1
|
0.1
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_source(s: &str) -> Result<crate::services::observation::Source, AppError> {
|
/// 把 (source, product, subtype) 字符串参数聚合为 ProductSpec。
|
||||||
use crate::services::observation::Source;
|
/// source/product 的解析统一委托给枚举的 from_str() 方法,不再重复 match。
|
||||||
Ok(match s.to_lowercase().as_str() {
|
|
||||||
"lamost" => Source::Lamost,
|
|
||||||
"gaia" => Source::Gaia,
|
|
||||||
"sdss" => Source::Sdss,
|
|
||||||
"desi" => Source::Desi,
|
|
||||||
other => {
|
|
||||||
return Err(AppError::bad_request(format!(
|
|
||||||
"不支持的 source '{}',可选: {:?}",
|
|
||||||
other,
|
|
||||||
Source::valid_values()
|
|
||||||
)))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_product(s: &str) -> Result<crate::services::observation::ProductType, AppError> {
|
|
||||||
use crate::services::observation::ProductType;
|
|
||||||
Ok(match s.to_lowercase().as_str() {
|
|
||||||
"spectrum" => ProductType::Spectrum,
|
|
||||||
"lightcurve" | "light_curve" | "lc" => ProductType::LightCurve,
|
|
||||||
"photometry" => ProductType::Photometry,
|
|
||||||
"image" => ProductType::Image,
|
|
||||||
other => {
|
|
||||||
return Err(AppError::bad_request(format!(
|
|
||||||
"不支持的 product '{}',可选: {:?}",
|
|
||||||
other,
|
|
||||||
ProductType::valid_values()
|
|
||||||
)))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 把 (source, product, subtype) 字符串参数聚合为 ProductSpec
|
|
||||||
fn build_product_spec(
|
fn build_product_spec(
|
||||||
product: &str,
|
product: &str,
|
||||||
subtype: &Option<String>,
|
subtype: &Option<String>,
|
||||||
) -> Result<crate::services::observation::ProductSpec, AppError> {
|
) -> Result<crate::services::observation::ProductSpec, AppError> {
|
||||||
Ok(crate::services::observation::ProductSpec {
|
Ok(crate::services::observation::ProductSpec {
|
||||||
product: parse_product(product)?,
|
product: crate::services::observation::ProductType::from_str(product)
|
||||||
|
.map_err(AppError::bad_request)?,
|
||||||
subtype: subtype.clone(),
|
subtype: subtype.clone(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -91,13 +59,16 @@ pub struct ObservationSearchParams {
|
|||||||
#[serde(default = "default_radius")]
|
#[serde(default = "default_radius")]
|
||||||
pub radius: f64,
|
pub radius: f64,
|
||||||
pub release: Option<String>,
|
pub release: Option<String>,
|
||||||
|
/// 数据发布的子版本(仅 LAMOST 有意义,如 "v2.0");None 时用该 DR 的默认版本
|
||||||
|
pub version: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn observation_search(
|
pub async fn observation_search(
|
||||||
State(state): State<std::sync::Arc<AppState>>,
|
State(state): State<std::sync::Arc<AppState>>,
|
||||||
Query(params): Query<ObservationSearchParams>,
|
Query(params): Query<ObservationSearchParams>,
|
||||||
) -> ApiResult<Json<Vec<crate::services::observation::Candidate>>> {
|
) -> ApiResult<Json<Vec<crate::services::observation::Candidate>>> {
|
||||||
let source = parse_source(¶ms.source)?;
|
let source = crate::services::observation::Source::from_str(¶ms.source)
|
||||||
|
.map_err(AppError::bad_request)?;
|
||||||
let product = build_product_spec(¶ms.product, ¶ms.subtype)?;
|
let product = build_product_spec(¶ms.product, ¶ms.subtype)?;
|
||||||
|
|
||||||
let candidates = crate::services::observation::search_observation(
|
let candidates = crate::services::observation::search_observation(
|
||||||
@ -109,6 +80,7 @@ pub async fn observation_search(
|
|||||||
params.dec,
|
params.dec,
|
||||||
params.radius,
|
params.radius,
|
||||||
params.release.as_deref(),
|
params.release.as_deref(),
|
||||||
|
params.version.as_deref(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
@ -140,6 +112,8 @@ pub struct ObservationDownloadRequest {
|
|||||||
pub product: String,
|
pub product: String,
|
||||||
pub subtype: Option<String>,
|
pub subtype: Option<String>,
|
||||||
pub release: Option<String>,
|
pub release: Option<String>,
|
||||||
|
/// 数据发布的子版本(仅 LAMOST 有意义,如 "v2.0");None 时用该 DR 的默认版本
|
||||||
|
pub version: Option<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub force: bool,
|
pub force: bool,
|
||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
@ -168,7 +142,8 @@ pub async fn observation_download(
|
|||||||
) -> ApiResult<Json<crate::services::observation::ObservationBatch>> {
|
) -> ApiResult<Json<crate::services::observation::ObservationBatch>> {
|
||||||
use crate::services::observation::{download_observation, ObservationRequest};
|
use crate::services::observation::{download_observation, ObservationRequest};
|
||||||
|
|
||||||
let source = parse_source(&req.source)?;
|
let source = crate::services::observation::Source::from_str(&req.source)
|
||||||
|
.map_err(AppError::bad_request)?;
|
||||||
let product = build_product_spec(&req.product, &req.subtype)?;
|
let product = build_product_spec(&req.product, &req.subtype)?;
|
||||||
|
|
||||||
let request = match req.mode {
|
let request = match req.mode {
|
||||||
@ -185,6 +160,7 @@ pub async fn observation_download(
|
|||||||
radius_deg,
|
radius_deg,
|
||||||
strategy,
|
strategy,
|
||||||
release: req.release.clone(),
|
release: req.release.clone(),
|
||||||
|
version: req.version.clone(),
|
||||||
},
|
},
|
||||||
DownloadMode::Identifiers { source_ids } => {
|
DownloadMode::Identifiers { source_ids } => {
|
||||||
let identifiers: Vec<String> = source_ids
|
let identifiers: Vec<String> = source_ids
|
||||||
@ -200,6 +176,7 @@ pub async fn observation_download(
|
|||||||
product,
|
product,
|
||||||
identifiers,
|
identifiers,
|
||||||
release: req.release.clone(),
|
release: req.release.clone(),
|
||||||
|
version: req.version.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@ -350,6 +350,11 @@ pub async fn upload_paper_file(
|
|||||||
return Err(AppError::bad_request("bibcode 包含非法字符"));
|
return Err(AppError::bad_request("bibcode 包含非法字符"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 防御性安全边界:虽然合法 bibcode 允许双点(..),旧 arXiv 允许斜杠(/),但二者不能共存。
|
||||||
|
if bibcode.contains("..") && (bibcode.contains('/') || bibcode.contains('\\')) {
|
||||||
|
return Err(AppError::bad_request("bibcode 包含非法的路径遍历字符"));
|
||||||
|
}
|
||||||
|
|
||||||
let updated_paper = crate::services::paper::upload_paper_file_service(
|
let updated_paper = crate::services::paper::upload_paper_file_service(
|
||||||
&state.db,
|
&state.db,
|
||||||
&state.config.library_dir,
|
&state.config.library_dir,
|
||||||
|
|||||||
@ -130,19 +130,19 @@ pub async fn update_permission_rules(
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut checkers = state.session_permission_checkers.write().await;
|
let mut entry = state
|
||||||
let checker = checkers
|
.session_permission_checkers
|
||||||
.entry(session_id.clone())
|
.entry(session_id.clone())
|
||||||
.or_insert_with(PermissionChecker::new);
|
.or_insert_with(PermissionChecker::new);
|
||||||
checker.add_rule_dynamic(session_rule);
|
entry.add_rule_dynamic(session_rule);
|
||||||
Json(PermissionActionResponse {
|
Json(PermissionActionResponse {
|
||||||
success: true,
|
success: true,
|
||||||
message: format!("已添加 {}:{} 规则", req.kind, req.rule),
|
message: format!("已添加 {}:{} 规则", req.kind, req.rule),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
"remove" => {
|
"remove" => {
|
||||||
let mut checkers = state.session_permission_checkers.write().await;
|
let removed =
|
||||||
let removed = if let Some(checker) = checkers.get_mut(&session_id) {
|
if let Some(mut checker) = state.session_permission_checkers.get_mut(&session_id) {
|
||||||
checker.remove_rule_dynamic(&req.rule, &req.kind)
|
checker.remove_rule_dynamic(&req.rule, &req.kind)
|
||||||
} else {
|
} else {
|
||||||
0
|
0
|
||||||
@ -182,19 +182,15 @@ pub async fn update_permission_mode(
|
|||||||
session_id, mode_str
|
session_id, mode_str
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut checkers = state.session_permission_checkers.write().await;
|
let mut checker = state
|
||||||
if let Some(checker) = checkers.get_mut(&session_id) {
|
.session_permission_checkers
|
||||||
|
.entry(session_id.clone())
|
||||||
|
.or_insert_with(PermissionChecker::new);
|
||||||
checker.set_mode(mode);
|
checker.set_mode(mode);
|
||||||
Json(PermissionActionResponse {
|
Json(PermissionActionResponse {
|
||||||
success: true,
|
success: true,
|
||||||
message: format!("权限模式已切换为: {}", mode_str),
|
message: format!("权限模式已切换为: {}", mode_str),
|
||||||
})
|
})
|
||||||
} else {
|
|
||||||
Json(PermissionActionResponse {
|
|
||||||
success: false,
|
|
||||||
message: format!("会话 {} 不存在或无活跃 Agent", session_id),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 列出指定会话的所有权限规则。
|
/// 列出指定会话的所有权限规则。
|
||||||
@ -202,9 +198,7 @@ pub async fn list_permission_rules(
|
|||||||
Path(session_id): Path<String>,
|
Path(session_id): Path<String>,
|
||||||
State(state): State<Arc<AppState>>,
|
State(state): State<Arc<AppState>>,
|
||||||
) -> Json<PermissionRulesResponse> {
|
) -> Json<PermissionRulesResponse> {
|
||||||
let checkers = state.session_permission_checkers.read().await;
|
if let Some(checker) = state.session_permission_checkers.get(&session_id) {
|
||||||
|
|
||||||
if let Some(checker) = checkers.get(&session_id) {
|
|
||||||
let mut deny_rules = Vec::new();
|
let mut deny_rules = Vec::new();
|
||||||
let mut allow_rules = Vec::new();
|
let mut allow_rules = Vec::new();
|
||||||
let mut ask_rules = Vec::new();
|
let mut ask_rules = Vec::new();
|
||||||
|
|||||||
@ -259,13 +259,13 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
astroresearch::services::batch::AssetBatchStatus::new(),
|
astroresearch::services::batch::AssetBatchStatus::new(),
|
||||||
)),
|
)),
|
||||||
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
|
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
|
||||||
cancelled_runs: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())),
|
cancelled_runs: Arc::new(dashmap::DashMap::new()),
|
||||||
skill_registry: Arc::new(tokio::sync::RwLock::new(
|
skill_registry: Arc::new(tokio::sync::RwLock::new(
|
||||||
astroresearch::agent::skills::SkillRegistry::new(config.skills_dir.clone()),
|
astroresearch::agent::skills::SkillRegistry::new(config.skills_dir.clone()),
|
||||||
)),
|
)),
|
||||||
pending_questions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
pending_questions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
||||||
pending_permissions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
pending_permissions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
||||||
session_permission_checkers: Arc::new(tokio::sync::RwLock::new(HashMap::new())),
|
session_permission_checkers: Arc::new(dashmap::DashMap::new()),
|
||||||
sse_broadcast: {
|
sse_broadcast: {
|
||||||
let (tx, _) = tokio::sync::broadcast::channel(256);
|
let (tx, _) = tokio::sync::broadcast::channel(256);
|
||||||
Some(tx)
|
Some(tx)
|
||||||
@ -631,7 +631,8 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
&config.library_dir,
|
&config.library_dir,
|
||||||
&title,
|
&title,
|
||||||
&content,
|
&content,
|
||||||
)?;
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
println!(
|
println!(
|
||||||
"✅ 笔记已成功保存!\n 文件名: {}\n 物理路径: {}\n 总字符数: {}",
|
"✅ 笔记已成功保存!\n 文件名: {}\n 物理路径: {}\n 总字符数: {}",
|
||||||
|
|||||||
@ -112,12 +112,11 @@ impl GaiaClient {
|
|||||||
Ok(GaiaClient {
|
Ok(GaiaClient {
|
||||||
client: reqwest::Client::builder()
|
client: reqwest::Client::builder()
|
||||||
// Gaia DataLink 端点(data-server/data)在传输完 ZIP 后直接断 TCP 而不发送
|
// Gaia DataLink 端点(data-server/data)在传输完 ZIP 后直接断 TCP 而不发送
|
||||||
// TLS close_notify 警报。rustls 严格遵守 RFC 5246,会把这种"无 close_notify 的
|
// TLS close_notify 警报。rustls 严格遵守 RFC 5246 会判 UnexpectedEof 致命错误。
|
||||||
// 关闭"判为 UnexpectedEof 致命错误(哪怕响应体已完整到达)。ESA 官方 Python
|
// 多数 UnexpectedEof 只在复用 idle 连接时触发,禁用连接池可避免:
|
||||||
// 客户端 astroquery 走系统 OpenSSL(native-tls),后者容忍此行为,故不受影响。
|
// 每次请求新建连接,响应读完即关闭,不会复用"已被对端不洁关闭"的连接。
|
||||||
// 这里对 Gaia 客户端单独启用 native-tls,使其与 astroquery 走相同 TLS 栈,
|
.pool_max_idle_per_host(0)
|
||||||
// 从而正确处理 DataLink 的连接关闭。项目其余 client 仍用默认 rustls。
|
.pool_idle_timeout(Some(Duration::from_secs(0)))
|
||||||
.use_native_tls()
|
|
||||||
.redirect(crate::utils::ssrf::safe_redirect_policy())
|
.redirect(crate::utils::ssrf::safe_redirect_policy())
|
||||||
.timeout(Duration::from_secs(timeout_secs))
|
.timeout(Duration::from_secs(timeout_secs))
|
||||||
.connect_timeout(Duration::from_secs(15))
|
.connect_timeout(Duration::from_secs(15))
|
||||||
|
|||||||
@ -99,13 +99,17 @@ impl LamostClient {
|
|||||||
/// GET {base}/{dr}/{ver}/{voservice|medvoservice}/conesearch?RA={ra}&DEC={dec}&SR={radius_deg}
|
/// GET {base}/{dr}/{ver}/{voservice|medvoservice}/conesearch?RA={ra}&DEC={dec}&SR={radius_deg}
|
||||||
/// - LRS(低分辨率)→ voservice 子路径
|
/// - LRS(低分辨率)→ voservice 子路径
|
||||||
/// - MRS(中分辨率)→ medvoservice 子路径
|
/// - MRS(中分辨率)→ medvoservice 子路径
|
||||||
|
///
|
||||||
/// 服务端返回 VOTable,用共享解析器解析后映射为 LamostSpectrumRow。
|
/// 服务端返回 VOTable,用共享解析器解析后映射为 LamostSpectrumRow。
|
||||||
|
///
|
||||||
|
/// `version` 为该 DR 的子版本路径段(如 "v2.0"/"v1.0"),由上层 parse_lamost_version 解析后传入。
|
||||||
pub async fn cone_search(
|
pub async fn cone_search(
|
||||||
&self,
|
&self,
|
||||||
ra: f64,
|
ra: f64,
|
||||||
dec: f64,
|
dec: f64,
|
||||||
radius_deg: f64,
|
radius_deg: f64,
|
||||||
release: crate::services::observation::lamost::LamostRelease,
|
release: crate::services::observation::lamost::LamostRelease,
|
||||||
|
version: &str,
|
||||||
resolution: crate::services::observation::lamost::LamostResolution,
|
resolution: crate::services::observation::lamost::LamostResolution,
|
||||||
) -> anyhow::Result<LamostConeResult> {
|
) -> anyhow::Result<LamostConeResult> {
|
||||||
use crate::services::observation::lamost::LamostResolution;
|
use crate::services::observation::lamost::LamostResolution;
|
||||||
@ -125,12 +129,13 @@ impl LamostClient {
|
|||||||
"{}/{}/{}/{}/conesearch",
|
"{}/{}/{}/{}/conesearch",
|
||||||
self.base_url,
|
self.base_url,
|
||||||
release.path_segment(),
|
release.path_segment(),
|
||||||
release.version_segment(),
|
version,
|
||||||
service_segment,
|
service_segment,
|
||||||
);
|
);
|
||||||
info!(
|
info!(
|
||||||
"[LAMOST] ConeSearch {} {} ra={:.4} dec={:.4} radius={:.4}°",
|
"[LAMOST] ConeSearch {}/{} {} ra={:.4} dec={:.4} radius={:.4}°",
|
||||||
release.path_segment(),
|
release.path_segment(),
|
||||||
|
version,
|
||||||
resolution.display(),
|
resolution.display(),
|
||||||
ra,
|
ra,
|
||||||
dec,
|
dec,
|
||||||
@ -172,21 +177,25 @@ impl LamostClient {
|
|||||||
///
|
///
|
||||||
/// GET {base}/{dr}/{ver}/spectrum/fits/{obsid} → application/gzip
|
/// GET {base}/{dr}/{ver}/spectrum/fits/{obsid} → application/gzip
|
||||||
/// 返回的 bytes 可能是 gzip 压缩(magic bytes 0x1f 0x8b),由业务层解压。
|
/// 返回的 bytes 可能是 gzip 压缩(magic bytes 0x1f 0x8b),由业务层解压。
|
||||||
|
///
|
||||||
|
/// `version` 为该 DR 的子版本路径段(如 "v2.0"),由上层 parse_lamost_version 解析后传入。
|
||||||
pub async fn download_fits(
|
pub async fn download_fits(
|
||||||
&self,
|
&self,
|
||||||
obsid: i64,
|
obsid: i64,
|
||||||
release: crate::services::observation::lamost::LamostRelease,
|
release: crate::services::observation::lamost::LamostRelease,
|
||||||
|
version: &str,
|
||||||
) -> anyhow::Result<Vec<u8>> {
|
) -> anyhow::Result<Vec<u8>> {
|
||||||
let url = format!(
|
let url = format!(
|
||||||
"{}/{}/{}/spectrum/fits/{}",
|
"{}/{}/{}/spectrum/fits/{}",
|
||||||
self.base_url,
|
self.base_url,
|
||||||
release.path_segment(),
|
release.path_segment(),
|
||||||
release.version_segment(),
|
version,
|
||||||
obsid
|
obsid
|
||||||
);
|
);
|
||||||
info!(
|
info!(
|
||||||
"[LAMOST] 下载 FITS {} obsid={}",
|
"[LAMOST] 下载 FITS {}/{} obsid={}",
|
||||||
release.path_segment(),
|
release.path_segment(),
|
||||||
|
version,
|
||||||
obsid
|
obsid
|
||||||
);
|
);
|
||||||
|
|
||||||
@ -366,13 +375,14 @@ mod tests {
|
|||||||
async fn test_live_cone_search() {
|
async fn test_live_cone_search() {
|
||||||
use crate::services::observation::lamost::{LamostRelease, LamostResolution};
|
use crate::services::observation::lamost::{LamostRelease, LamostResolution};
|
||||||
let client = LamostClient::new("https://www.lamost.org", 60).unwrap();
|
let client = LamostClient::new("https://www.lamost.org", 60).unwrap();
|
||||||
// M31 附近,DR10 LRS
|
// M31 附近,DR10 v2.0 LRS
|
||||||
let result = client
|
let result = client
|
||||||
.cone_search(
|
.cone_search(
|
||||||
10.6847,
|
10.6847,
|
||||||
41.2687,
|
41.2687,
|
||||||
0.1,
|
0.1,
|
||||||
LamostRelease::Dr10,
|
LamostRelease::Dr10,
|
||||||
|
"v2.0",
|
||||||
LamostResolution::Lrs,
|
LamostResolution::Lrs,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@ -403,6 +413,7 @@ mod tests {
|
|||||||
41.2687,
|
41.2687,
|
||||||
0.1,
|
0.1,
|
||||||
LamostRelease::Dr10,
|
LamostRelease::Dr10,
|
||||||
|
"v2.0",
|
||||||
LamostResolution::Lrs,
|
LamostResolution::Lrs,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@ -410,7 +421,7 @@ mod tests {
|
|||||||
let obsid = cone.rows.first().expect("应有结果").obsid;
|
let obsid = cone.rows.first().expect("应有结果").obsid;
|
||||||
|
|
||||||
let bytes = client
|
let bytes = client
|
||||||
.download_fits(obsid, LamostRelease::Dr10)
|
.download_fits(obsid, LamostRelease::Dr10, "v2.0")
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
println!("===== LAMOST FITS Live (obsid={}) =====", obsid);
|
println!("===== LAMOST FITS Live (obsid={}) =====", obsid);
|
||||||
|
|||||||
@ -24,7 +24,7 @@ pub trait ChatCompleter: Send + Sync {
|
|||||||
pub struct LlmClient {
|
pub struct LlmClient {
|
||||||
api_key: String,
|
api_key: String,
|
||||||
api_base: String,
|
api_base: String,
|
||||||
model: Arc<std::sync::RwLock<String>>,
|
model: Arc<tokio::sync::RwLock<String>>,
|
||||||
client: Client,
|
client: Client,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -46,21 +46,25 @@ impl LlmClient {
|
|||||||
Ok(LlmClient {
|
Ok(LlmClient {
|
||||||
api_key,
|
api_key,
|
||||||
api_base,
|
api_base,
|
||||||
model: Arc::new(std::sync::RwLock::new(model)),
|
model: Arc::new(tokio::sync::RwLock::new(model)),
|
||||||
client,
|
client,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn model(&self) -> String {
|
pub async fn model(&self) -> String {
|
||||||
self.model.read().unwrap_or_else(|e| e.into_inner()).clone()
|
self.model.read().await.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 同步获取模型名称快照(用于非 async 上下文,如环境信息展示)。
|
||||||
|
/// 如果锁被持有则返回空字符串。
|
||||||
|
pub fn model_name_snapshot(&self) -> String {
|
||||||
|
self.model.try_read().map(|m| m.clone()).unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 运行时切换模型(用于故障转移)。
|
/// 运行时切换模型(用于故障转移)。
|
||||||
pub fn set_model(&self, model: String) {
|
pub async fn set_model(&self, model: String) {
|
||||||
if let Ok(mut m) = self.model.write() {
|
tracing::info!("[LlmClient] 模型切换为: {}", model);
|
||||||
*m = model;
|
*self.model.write().await = model;
|
||||||
tracing::info!("[LlmClient] 模型切换为: {}", m);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn api_base(&self) -> &str {
|
pub fn api_base(&self) -> &str {
|
||||||
@ -79,7 +83,7 @@ impl LlmClient {
|
|||||||
let url = format!("{}/chat/completions", self.api_base);
|
let url = format!("{}/chat/completions", self.api_base);
|
||||||
|
|
||||||
let payload = serde_json::json!({
|
let payload = serde_json::json!({
|
||||||
"model": self.model(),
|
"model": self.model().await,
|
||||||
"messages": [
|
"messages": [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@ -143,7 +147,7 @@ impl LlmClient {
|
|||||||
let url = format!("{}/chat/completions", self.api_base);
|
let url = format!("{}/chat/completions", self.api_base);
|
||||||
|
|
||||||
let mut payload = serde_json::json!({
|
let mut payload = serde_json::json!({
|
||||||
"model": self.model(),
|
"model": self.model().await,
|
||||||
"messages": messages,
|
"messages": messages,
|
||||||
"temperature": 0.3
|
"temperature": 0.3
|
||||||
});
|
});
|
||||||
@ -151,7 +155,7 @@ impl LlmClient {
|
|||||||
// 前端可控的思考模式开关(仅对千问/DashScope 启用 enable_thinking 参数)(仅对千问/DashScope 启用 enable_thinking 参数)
|
// 前端可控的思考模式开关(仅对千问/DashScope 启用 enable_thinking 参数)(仅对千问/DashScope 启用 enable_thinking 参数)
|
||||||
if enable_thinking
|
if enable_thinking
|
||||||
&& (self.api_base.contains("dashscope.aliyuncs.com")
|
&& (self.api_base.contains("dashscope.aliyuncs.com")
|
||||||
|| self.model().to_lowercase().contains("qwen"))
|
|| self.model().await.to_lowercase().contains("qwen"))
|
||||||
{
|
{
|
||||||
if let Some(obj) = payload.as_object_mut() {
|
if let Some(obj) = payload.as_object_mut() {
|
||||||
obj.insert("enable_thinking".to_string(), serde_json::json!(true));
|
obj.insert("enable_thinking".to_string(), serde_json::json!(true));
|
||||||
@ -224,7 +228,7 @@ impl LlmClient {
|
|||||||
]);
|
]);
|
||||||
|
|
||||||
let payload = serde_json::json!({
|
let payload = serde_json::json!({
|
||||||
"model": self.model(),
|
"model": self.model().await,
|
||||||
"messages": [
|
"messages": [
|
||||||
{ "role": "system", "content": system_prompt },
|
{ "role": "system", "content": system_prompt },
|
||||||
{ "role": "user", "content": user_content }
|
{ "role": "user", "content": user_content }
|
||||||
@ -297,7 +301,7 @@ impl LlmClient {
|
|||||||
let url = format!("{}/chat/completions", self.api_base);
|
let url = format!("{}/chat/completions", self.api_base);
|
||||||
|
|
||||||
let mut payload = serde_json::json!({
|
let mut payload = serde_json::json!({
|
||||||
"model": self.model(),
|
"model": self.model().await,
|
||||||
"messages": messages,
|
"messages": messages,
|
||||||
"temperature": 0.3,
|
"temperature": 0.3,
|
||||||
"stream": true,
|
"stream": true,
|
||||||
@ -307,7 +311,7 @@ impl LlmClient {
|
|||||||
// 前端可控的思考模式开关(仅对千问/DashScope 启用 enable_thinking 参数)
|
// 前端可控的思考模式开关(仅对千问/DashScope 启用 enable_thinking 参数)
|
||||||
if enable_thinking
|
if enable_thinking
|
||||||
&& (self.api_base.contains("dashscope.aliyuncs.com")
|
&& (self.api_base.contains("dashscope.aliyuncs.com")
|
||||||
|| self.model().to_lowercase().contains("qwen"))
|
|| self.model().await.to_lowercase().contains("qwen"))
|
||||||
{
|
{
|
||||||
if let Some(obj) = payload.as_object_mut() {
|
if let Some(obj) = payload.as_object_mut() {
|
||||||
obj.insert("enable_thinking".to_string(), serde_json::json!(true));
|
obj.insert("enable_thinking".to_string(), serde_json::json!(true));
|
||||||
@ -564,13 +568,13 @@ mod tests {
|
|||||||
ChatMessage, FunctionCall, MessageRole, ToolCall, ToolDefinition,
|
ChatMessage, FunctionCall, MessageRole, ToolCall, ToolDefinition,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn test_llm_client_initialization() {
|
async fn test_llm_client_basics() {
|
||||||
let client =
|
let client =
|
||||||
LlmClient::new("key".to_string(), "base".to_string(), "model".to_string()).unwrap();
|
LlmClient::new("key".to_string(), "base".to_string(), "model".to_string()).unwrap();
|
||||||
assert_eq!(client.api_key(), "key");
|
assert_eq!(client.api_key(), "key");
|
||||||
assert_eq!(client.api_base(), "base");
|
assert_eq!(client.api_base(), "base");
|
||||||
assert_eq!(client.model(), "model");
|
assert_eq!(client.model().await, "model");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
63
src/lib.rs
63
src/lib.rs
@ -2,6 +2,19 @@
|
|||||||
use std::env;
|
use std::env;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
/// 从环境变量解析 u64 值,解析失败时打印警告并返回默认值。
|
||||||
|
fn env_u64(key: &str, default: u64) -> u64 {
|
||||||
|
env::var(key)
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| {
|
||||||
|
v.parse::<u64>().ok().or_else(|| {
|
||||||
|
tracing::warn!("{} '{}' 不是有效数字,使用默认值 {}", key, v, default);
|
||||||
|
None
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.unwrap_or(default)
|
||||||
|
}
|
||||||
|
|
||||||
// 系统配置结构体,加载并管理从环境变量或 .env 文件读取的参数
|
// 系统配置结构体,加载并管理从环境变量或 .env 文件读取的参数
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct Config {
|
pub struct Config {
|
||||||
@ -133,65 +146,25 @@ impl Config {
|
|||||||
|
|
||||||
let vizier_tap_url = env::var("VIZIER_TAP_URL")
|
let vizier_tap_url = env::var("VIZIER_TAP_URL")
|
||||||
.unwrap_or_else(|_| "https://tapvizier.cds.unistra.fr/TAPVizieR/tap".to_string());
|
.unwrap_or_else(|_| "https://tapvizier.cds.unistra.fr/TAPVizieR/tap".to_string());
|
||||||
let vizier_timeout_secs = env::var("VIZIER_TIMEOUT_SECS")
|
let vizier_timeout_secs = env_u64("VIZIER_TIMEOUT_SECS", 60);
|
||||||
.ok()
|
|
||||||
.and_then(|v| {
|
|
||||||
v.parse::<u64>().ok().or_else(|| {
|
|
||||||
tracing::warn!("VIZIER_TIMEOUT_SECS '{}' 不是有效数字,使用默认值 60", v);
|
|
||||||
None
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.unwrap_or(60);
|
|
||||||
|
|
||||||
let lamost_base_url =
|
let lamost_base_url =
|
||||||
env::var("LAMOST_BASE_URL").unwrap_or_else(|_| "https://www.lamost.org".to_string());
|
env::var("LAMOST_BASE_URL").unwrap_or_else(|_| "https://www.lamost.org".to_string());
|
||||||
let lamost_timeout_secs = env::var("LAMOST_TIMEOUT_SECS")
|
let lamost_timeout_secs = env_u64("LAMOST_TIMEOUT_SECS", 60);
|
||||||
.ok()
|
|
||||||
.and_then(|v| {
|
|
||||||
v.parse::<u64>().ok().or_else(|| {
|
|
||||||
tracing::warn!("LAMOST_TIMEOUT_SECS '{}' 不是有效数字,使用默认值 60", v);
|
|
||||||
None
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.unwrap_or(60);
|
|
||||||
|
|
||||||
let gaia_tap_url = env::var("GAIA_TAP_URL")
|
let gaia_tap_url = env::var("GAIA_TAP_URL")
|
||||||
.unwrap_or_else(|_| "https://gea.esac.esa.int/tap-server/tap".to_string());
|
.unwrap_or_else(|_| "https://gea.esac.esa.int/tap-server/tap".to_string());
|
||||||
let gaia_datalink_url = env::var("GAIA_DATALINK_URL")
|
let gaia_datalink_url = env::var("GAIA_DATALINK_URL")
|
||||||
.unwrap_or_else(|_| "https://gea.esac.esa.int/data-server".to_string());
|
.unwrap_or_else(|_| "https://gea.esac.esa.int/data-server".to_string());
|
||||||
let gaia_timeout_secs = env::var("GAIA_TIMEOUT_SECS")
|
let gaia_timeout_secs = env_u64("GAIA_TIMEOUT_SECS", 90);
|
||||||
.ok()
|
|
||||||
.and_then(|v| {
|
|
||||||
v.parse::<u64>().ok().or_else(|| {
|
|
||||||
tracing::warn!("GAIA_TIMEOUT_SECS '{}' 不是有效数字,使用默认值 90", v);
|
|
||||||
None
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.unwrap_or(90);
|
|
||||||
|
|
||||||
let sdss_tap_url = env::var("SDSS_TAP_URL")
|
let sdss_tap_url = env::var("SDSS_TAP_URL")
|
||||||
.unwrap_or_else(|_| "https://datalab.noirlab.edu/tap/sync".to_string());
|
.unwrap_or_else(|_| "https://datalab.noirlab.edu/tap/sync".to_string());
|
||||||
let sdss_timeout_secs = env::var("SDSS_TIMEOUT_SECS")
|
let sdss_timeout_secs = env_u64("SDSS_TIMEOUT_SECS", 90);
|
||||||
.ok()
|
|
||||||
.and_then(|v| {
|
|
||||||
v.parse::<u64>().ok().or_else(|| {
|
|
||||||
tracing::warn!("SDSS_TIMEOUT_SECS '{}' 不是有效数字,使用默认值 90", v);
|
|
||||||
None
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.unwrap_or(90);
|
|
||||||
|
|
||||||
let desi_tap_url = env::var("DESI_TAP_URL")
|
let desi_tap_url = env::var("DESI_TAP_URL")
|
||||||
.unwrap_or_else(|_| "https://datalab.noirlab.edu/tap/sync".to_string());
|
.unwrap_or_else(|_| "https://datalab.noirlab.edu/tap/sync".to_string());
|
||||||
let desi_timeout_secs = env::var("DESI_TIMEOUT_SECS")
|
let desi_timeout_secs = env_u64("DESI_TIMEOUT_SECS", 120);
|
||||||
.ok()
|
|
||||||
.and_then(|v| {
|
|
||||||
v.parse::<u64>().ok().or_else(|| {
|
|
||||||
tracing::warn!("DESI_TIMEOUT_SECS '{}' 不是有效数字,使用默认值 120", v);
|
|
||||||
None
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.unwrap_or(120);
|
|
||||||
|
|
||||||
Config {
|
Config {
|
||||||
database_url,
|
database_url,
|
||||||
|
|||||||
22
src/main.rs
22
src/main.rs
@ -290,7 +290,9 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
gaia,
|
gaia,
|
||||||
sdss,
|
sdss,
|
||||||
desi,
|
desi,
|
||||||
observation_registry: Arc::new(astroresearch::services::observation::ObservationRegistry::default()),
|
observation_registry: Arc::new(
|
||||||
|
astroresearch::services::observation::ObservationRegistry::default(),
|
||||||
|
),
|
||||||
llm,
|
llm,
|
||||||
medium_llm,
|
medium_llm,
|
||||||
fast_llm,
|
fast_llm,
|
||||||
@ -309,7 +311,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
astroresearch::services::batch::AssetBatchStatus::new(),
|
astroresearch::services::batch::AssetBatchStatus::new(),
|
||||||
)),
|
)),
|
||||||
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
|
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
|
||||||
cancelled_runs: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())),
|
cancelled_runs: Arc::new(dashmap::DashMap::new()),
|
||||||
skill_registry,
|
skill_registry,
|
||||||
pending_questions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
pending_questions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
||||||
pending_permissions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
pending_permissions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
||||||
@ -320,9 +322,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
memory_manager: Arc::new(tokio::sync::Mutex::new(
|
memory_manager: Arc::new(tokio::sync::Mutex::new(
|
||||||
astroresearch::agent::memory::MemoryManager::new(config.library_dir.clone()),
|
astroresearch::agent::memory::MemoryManager::new(config.library_dir.clone()),
|
||||||
)),
|
)),
|
||||||
session_permission_checkers: Arc::new(tokio::sync::RwLock::new(
|
session_permission_checkers: Arc::new(dashmap::DashMap::new()),
|
||||||
std::collections::HashMap::new(),
|
|
||||||
)),
|
|
||||||
sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
|
sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
|
||||||
login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
||||||
upload_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
upload_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
||||||
@ -530,6 +530,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
.nest_service("/api/files", protected_files)
|
.nest_service("/api/files", protected_files)
|
||||||
.fallback_service(serve_dir)
|
.fallback_service(serve_dir)
|
||||||
.layer(tower_http::trace::TraceLayer::new_for_http())
|
.layer(tower_http::trace::TraceLayer::new_for_http())
|
||||||
|
.layer(tower_http::catch_panic::CatchPanicLayer::new())
|
||||||
.layer(SetResponseHeaderLayer::overriding(
|
.layer(SetResponseHeaderLayer::overriding(
|
||||||
axum::http::header::X_CONTENT_TYPE_OPTIONS,
|
axum::http::header::X_CONTENT_TYPE_OPTIONS,
|
||||||
HeaderValue::from_static("nosniff"),
|
HeaderValue::from_static("nosniff"),
|
||||||
@ -575,17 +576,18 @@ async fn pdf_inline_middleware(
|
|||||||
req: axum::http::Request<axum::body::Body>,
|
req: axum::http::Request<axum::body::Body>,
|
||||||
next: axum::middleware::Next,
|
next: axum::middleware::Next,
|
||||||
) -> Result<axum::response::Response<axum::body::Body>, axum::http::StatusCode> {
|
) -> Result<axum::response::Response<axum::body::Body>, axum::http::StatusCode> {
|
||||||
let path = req.uri().path().to_lowercase();
|
let original_path = req.uri().path().to_string();
|
||||||
let is_pdf = path.ends_with(".pdf");
|
let is_pdf = original_path.to_lowercase().ends_with(".pdf");
|
||||||
|
|
||||||
let mut response = next.run(req).await;
|
let mut response = next.run(req).await;
|
||||||
|
|
||||||
if is_pdf && response.status().is_success() {
|
if is_pdf && response.status().is_success() {
|
||||||
// 获取文件名
|
// 使用原始路径提取文件名(保留大小写)
|
||||||
let filename = std::path::Path::new(&path)
|
let filename = std::path::Path::new(&original_path)
|
||||||
.file_name()
|
.file_name()
|
||||||
.and_then(|n| n.to_str())
|
.and_then(|n| n.to_str())
|
||||||
.unwrap_or("document.pdf");
|
.unwrap_or("document.pdf")
|
||||||
|
.replace('"', "_");
|
||||||
|
|
||||||
// 强行设置 Content-Type 为 application/pdf
|
// 强行设置 Content-Type 为 application/pdf
|
||||||
response.headers_mut().insert(
|
response.headers_mut().insert(
|
||||||
|
|||||||
@ -220,7 +220,7 @@ pub(super) async fn build_markdown_with_frontmatter(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// 将 Markdown 内容写入 library_dir 并更新数据库 markdown_path。
|
/// 将 Markdown 内容写入 library_dir 并更新数据库 markdown_path。
|
||||||
pub(super) fn write_markdown_and_update_db(
|
pub(super) async fn write_markdown_and_update_db(
|
||||||
config: &Config,
|
config: &Config,
|
||||||
bibcode: &str,
|
bibcode: &str,
|
||||||
content: &str,
|
content: &str,
|
||||||
@ -228,9 +228,9 @@ pub(super) fn write_markdown_and_update_db(
|
|||||||
let md_filename = format!("{}.md", bibcode);
|
let md_filename = format!("{}.md", bibcode);
|
||||||
let md_dest = config.library_dir.join("Markdown").join(&md_filename);
|
let md_dest = config.library_dir.join("Markdown").join(&md_filename);
|
||||||
if let Some(parent) = md_dest.parent() {
|
if let Some(parent) = md_dest.parent() {
|
||||||
let _ = std::fs::create_dir_all(parent);
|
let _ = tokio::fs::create_dir_all(parent).await;
|
||||||
}
|
}
|
||||||
if std::fs::write(&md_dest, content).is_ok() {
|
if tokio::fs::write(&md_dest, content).await.is_ok() {
|
||||||
Some(format!("Markdown/{}", md_filename))
|
Some(format!("Markdown/{}", md_filename))
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
|
|||||||
@ -170,7 +170,7 @@ pub(super) async fn process_single_bibcode(
|
|||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
if let Some(rel_path) =
|
if let Some(rel_path) =
|
||||||
write_markdown_and_update_db(config, &bibcode, &parsed_md)
|
write_markdown_and_update_db(config, &bibcode, &parsed_md).await
|
||||||
{
|
{
|
||||||
relative_md_path = rel_path;
|
relative_md_path = rel_path;
|
||||||
}
|
}
|
||||||
@ -288,7 +288,8 @@ pub(super) async fn process_single_bibcode(
|
|||||||
&config_clone,
|
&config_clone,
|
||||||
&bibcode_clone,
|
&bibcode_clone,
|
||||||
&parsed_md,
|
&parsed_md,
|
||||||
);
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
if let Some(rel_path) = rel_md {
|
if let Some(rel_path) = rel_md {
|
||||||
let _ = sqlx::query("UPDATE papers SET markdown_path = ? WHERE bibcode = ?")
|
let _ = sqlx::query("UPDATE papers SET markdown_path = ? WHERE bibcode = ?")
|
||||||
|
|||||||
@ -213,20 +213,36 @@ pub async fn extract_and_cache_targets(
|
|||||||
let mut results = Vec::new();
|
let mut results = Vec::new();
|
||||||
let mut new_targets = Vec::new();
|
let mut new_targets = Vec::new();
|
||||||
|
|
||||||
for name in names {
|
if names.is_empty() {
|
||||||
// 优先在 paper_targets 表中查找本地缓存
|
return results;
|
||||||
let row = sqlx::query_as::<_, TargetDbRow>(
|
}
|
||||||
|
|
||||||
|
// 1. 批量查询已存在的本地缓存记录,减少 N+1 数据库往返
|
||||||
|
let mut cached_map = std::collections::HashMap::new();
|
||||||
|
for chunk in names.chunks(crate::services::paper::SQLITE_PARAM_LIMIT) {
|
||||||
|
let placeholders = vec!["?"; chunk.len()].join(", ");
|
||||||
|
let sql = format!(
|
||||||
"SELECT target_name, ra, dec, parallax, parallax_err, spectral_type, v_magnitude, \
|
"SELECT target_name, ra, dec, parallax, parallax_err, spectral_type, v_magnitude, \
|
||||||
otype, oname, pm_ra, pm_de, radial_velocity, photometry, aliases \
|
otype, oname, pm_ra, pm_de, radial_velocity, photometry, aliases \
|
||||||
FROM paper_targets WHERE target_name = ? LIMIT 1",
|
FROM paper_targets WHERE target_name IN ({})",
|
||||||
)
|
placeholders
|
||||||
.bind(&name)
|
);
|
||||||
.fetch_optional(pool)
|
let mut query = sqlx::query_as::<_, TargetDbRow>(&sql);
|
||||||
.await;
|
for name in chunk {
|
||||||
|
query = query.bind(name);
|
||||||
|
}
|
||||||
|
if let Ok(rows) = query.fetch_all(pool).await {
|
||||||
|
for row in rows {
|
||||||
|
cached_map.insert(row.target_name.clone(), TargetInfo::from(row));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if let Ok(Some(db_row)) = row {
|
// 2. 遍历检查哪些命中缓存,哪些需要调用 Sesame
|
||||||
|
for name in names {
|
||||||
|
if let Some(info) = cached_map.remove(&name) {
|
||||||
info!("天体 {} 命中本地缓存", name);
|
info!("天体 {} 命中本地缓存", name);
|
||||||
results.push(TargetInfo::from(db_row));
|
results.push(info);
|
||||||
} else {
|
} else {
|
||||||
// 本地缓存未命中,释放连接并外部查询 Sesame
|
// 本地缓存未命中,释放连接并外部查询 Sesame
|
||||||
info!("天体 {} 本地缓存未命中,查询 CDS Sesame...", name);
|
info!("天体 {} 本地缓存未命中,查询 CDS Sesame...", name);
|
||||||
|
|||||||
@ -323,7 +323,10 @@ impl<'a> VizierCatalog<'a> {
|
|||||||
} else {
|
} else {
|
||||||
sanitize_identifier(table)?
|
sanitize_identifier(table)?
|
||||||
};
|
};
|
||||||
Ok(format!("SELECT TOP {} {} FROM {}", limit, columns, table_ref))
|
Ok(format!(
|
||||||
|
"SELECT TOP {} {} FROM {}",
|
||||||
|
limit, columns, table_ref
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 导出星表数据到 CSV 文件
|
/// 导出星表数据到 CSV 文件
|
||||||
@ -357,7 +360,13 @@ impl<'a> VizierCatalog<'a> {
|
|||||||
let catalog_name = table.unwrap_or("adql_query");
|
let catalog_name = table.unwrap_or("adql_query");
|
||||||
let safe_name: String = catalog_name
|
let safe_name: String = catalog_name
|
||||||
.chars()
|
.chars()
|
||||||
.map(|c| if c.is_alphanumeric() || c == '-' || c == '_' { c } else { '_' })
|
.map(|c| {
|
||||||
|
if c.is_alphanumeric() || c == '-' || c == '_' {
|
||||||
|
c
|
||||||
|
} else {
|
||||||
|
'_'
|
||||||
|
}
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let ts = chrono::Utc::now().format("%Y%m%d_%H%M%S");
|
let ts = chrono::Utc::now().format("%Y%m%d_%H%M%S");
|
||||||
@ -367,10 +376,12 @@ impl<'a> VizierCatalog<'a> {
|
|||||||
.join(format!("{}.csv", ts));
|
.join(format!("{}.csv", ts));
|
||||||
|
|
||||||
if let Some(parent) = output_path.parent() {
|
if let Some(parent) = output_path.parent() {
|
||||||
tokio::fs::create_dir_all(parent).await
|
tokio::fs::create_dir_all(parent)
|
||||||
|
.await
|
||||||
.map_err(|e| anyhow!("创建目录失败: {}", e))?;
|
.map_err(|e| anyhow!("创建目录失败: {}", e))?;
|
||||||
}
|
}
|
||||||
tokio::fs::write(&output_path, &export.csv).await
|
tokio::fs::write(&output_path, &export.csv)
|
||||||
|
.await
|
||||||
.map_err(|e| anyhow!("写入文件失败: {}", e))?;
|
.map_err(|e| anyhow!("写入文件失败: {}", e))?;
|
||||||
|
|
||||||
Ok(ExportToFileResult {
|
Ok(ExportToFileResult {
|
||||||
|
|||||||
@ -152,83 +152,74 @@ impl Downloader {
|
|||||||
doi: Option<&str>,
|
doi: Option<&str>,
|
||||||
library_dir: &Path,
|
library_dir: &Path,
|
||||||
) -> (Result<PathBuf, String>, Result<PathBuf, String>) {
|
) -> (Result<PathBuf, String>, Result<PathBuf, String>) {
|
||||||
let base = "https://ui.adsabs.harvard.edu/link_gateway";
|
|
||||||
let pdf_dest = library_dir.join("PDF").join(format!("{}.pdf", bibcode));
|
let pdf_dest = library_dir.join("PDF").join(format!("{}.pdf", bibcode));
|
||||||
let html_dest = library_dir.join("HTML").join(format!("{}.html", bibcode));
|
let html_dest = library_dir.join("HTML").join(format!("{}.html", bibcode));
|
||||||
|
|
||||||
let mut pdf_res = Err("未尝试任何下载通道".to_string());
|
let pdf_res = self.try_download_pdf(bibcode, doi, &pdf_dest).await;
|
||||||
let mut html_res = Err("未尝试任何下载通道".to_string());
|
let html_res = self.try_download_html(bibcode, &html_dest).await;
|
||||||
|
|
||||||
let mut pdf_errors = Vec::new();
|
(pdf_res, html_res)
|
||||||
let mut html_errors = Vec::new();
|
}
|
||||||
|
|
||||||
|
/// PDF 多级回退下载:PUB_PDF → ADS_PDF → EPRINT_PDF → CrossRef → SCAN
|
||||||
|
async fn try_download_pdf(
|
||||||
|
&self,
|
||||||
|
bibcode: &str,
|
||||||
|
doi: Option<&str>,
|
||||||
|
dest: &Path,
|
||||||
|
) -> Result<PathBuf, String> {
|
||||||
|
let base = "https://ui.adsabs.harvard.edu/link_gateway";
|
||||||
|
let mut errors = Vec::new();
|
||||||
|
|
||||||
// ── PDF 下载 ───────────────────────────────────────────
|
|
||||||
info!("[下载] 开始 PDF 下载: {}", bibcode);
|
info!("[下载] 开始 PDF 下载: {}", bibcode);
|
||||||
|
|
||||||
'pdf: {
|
|
||||||
// 1a. ADS PUB_PDF 网关
|
// 1a. ADS PUB_PDF 网关
|
||||||
let gw = format!("{}/{}/PUB_PDF", base, bibcode);
|
let gw = format!("{}/{}/PUB_PDF", base, bibcode);
|
||||||
match self.resolve_ads_gateway(&gw).await {
|
match self.resolve_ads_gateway(&gw).await {
|
||||||
Ok(resolved) => {
|
Ok(resolved) => {
|
||||||
let result = if resolved.contains("iopscience.iop.org") {
|
let result = if resolved.contains("iopscience.iop.org") {
|
||||||
// 提取 DOI 路径部分,走 IOP 专属策略
|
|
||||||
let doi = resolved
|
let doi = resolved
|
||||||
.trim_start_matches("https://iopscience.iop.org/article/")
|
.trim_start_matches("https://iopscience.iop.org/article/")
|
||||||
.trim_end_matches("/pdf")
|
.trim_end_matches("/pdf")
|
||||||
.trim_end_matches('/');
|
.trim_end_matches('/');
|
||||||
self.download_iop_pdf(doi, &pdf_dest).await
|
self.download_iop_pdf(doi, dest).await
|
||||||
} else if resolved.contains("link.springer.com")
|
} else if resolved.contains("link.springer.com") || resolved.contains("nature.com")
|
||||||
|| resolved.contains("nature.com")
|
|
||||||
{
|
{
|
||||||
// Springer/Nature:HTML 更可靠,PDF 用通用策略
|
self.download_pdf_direct(&resolved, dest, "Springer").await
|
||||||
self.download_pdf_direct(&resolved, &pdf_dest, "Springer")
|
|
||||||
.await
|
|
||||||
} else {
|
} else {
|
||||||
self.download_pdf_direct(&resolved, &pdf_dest, "PUB_PDF")
|
self.download_pdf_direct(&resolved, dest, "PUB_PDF").await
|
||||||
.await
|
|
||||||
};
|
};
|
||||||
match result {
|
match result {
|
||||||
Ok(_) => {
|
Ok(_) => return Ok(dest.to_path_buf()),
|
||||||
pdf_res = Ok(pdf_dest.clone());
|
|
||||||
break 'pdf;
|
|
||||||
}
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let msg = format!("PUB_PDF下载失败: {}", e);
|
let msg = format!("PUB_PDF下载失败: {}", e);
|
||||||
warn!("{}", msg);
|
warn!("{}", msg);
|
||||||
pdf_errors.push(msg);
|
errors.push(msg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let msg = format!("PUB_PDF网关解析失败: {}", e);
|
let msg = format!("PUB_PDF网关解析失败: {}", e);
|
||||||
warn!("{}", msg);
|
warn!("{}", msg);
|
||||||
pdf_errors.push(msg);
|
errors.push(msg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1b. ADS_PDF 网关 (经典 ADS 整合 PDF 直接通道)
|
// 1b. ADS_PDF 网关
|
||||||
let gw = format!("{}/{}/ADS_PDF", base, bibcode);
|
let gw = format!("{}/{}/ADS_PDF", base, bibcode);
|
||||||
match self.resolve_ads_gateway(&gw).await {
|
match self.resolve_ads_gateway(&gw).await {
|
||||||
Ok(resolved) => {
|
Ok(resolved) => match self.download_pdf_direct(&resolved, dest, "ADS_PDF").await {
|
||||||
match self
|
Ok(_) => return Ok(dest.to_path_buf()),
|
||||||
.download_pdf_direct(&resolved, &pdf_dest, "ADS_PDF")
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(_) => {
|
|
||||||
pdf_res = Ok(pdf_dest.clone());
|
|
||||||
break 'pdf;
|
|
||||||
}
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let msg = format!("ADS_PDF下载失败: {}", e);
|
let msg = format!("ADS_PDF下载失败: {}", e);
|
||||||
warn!("{}", msg);
|
warn!("{}", msg);
|
||||||
pdf_errors.push(msg);
|
errors.push(msg);
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
},
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let msg = format!("ADS_PDF网关解析失败: {}", e);
|
let msg = format!("ADS_PDF网关解析失败: {}", e);
|
||||||
warn!("{}", msg);
|
warn!("{}", msg);
|
||||||
pdf_errors.push(msg);
|
errors.push(msg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -237,142 +228,129 @@ impl Downloader {
|
|||||||
match self.resolve_ads_gateway(&gw).await {
|
match self.resolve_ads_gateway(&gw).await {
|
||||||
Ok(resolved) => {
|
Ok(resolved) => {
|
||||||
match self
|
match self
|
||||||
.download_pdf_direct(&resolved, &pdf_dest, "EPRINT_PDF")
|
.download_pdf_direct(&resolved, dest, "EPRINT_PDF")
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(_) => {
|
Ok(_) => return Ok(dest.to_path_buf()),
|
||||||
pdf_res = Ok(pdf_dest.clone());
|
|
||||||
break 'pdf;
|
|
||||||
}
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let msg = format!("EPRINT_PDF下载失败: {}", e);
|
let msg = format!("EPRINT_PDF下载失败: {}", e);
|
||||||
warn!("{}", msg);
|
warn!("{}", msg);
|
||||||
pdf_errors.push(msg);
|
errors.push(msg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let msg = format!("EPRINT_PDF网关解析失败: {}", e);
|
let msg = format!("EPRINT_PDF网关解析失败: {}", e);
|
||||||
warn!("{}", msg);
|
warn!("{}", msg);
|
||||||
pdf_errors.push(msg);
|
errors.push(msg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1d. CrossRef API 回退(需要 DOI)
|
// 1d. CrossRef API 回退
|
||||||
if let Some(doi_str) = doi {
|
if let Some(doi_str) = doi {
|
||||||
match self.download_crossref_pdf(doi_str, &pdf_dest).await {
|
match self.download_crossref_pdf(doi_str, dest).await {
|
||||||
Ok(_) => {
|
Ok(_) => return Ok(dest.to_path_buf()),
|
||||||
pdf_res = Ok(pdf_dest.clone());
|
|
||||||
break 'pdf;
|
|
||||||
}
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let msg = format!("CrossRef下载失败: {}", e);
|
let msg = format!("CrossRef下载失败: {}", e);
|
||||||
warn!("{}", msg);
|
warn!("{}", msg);
|
||||||
pdf_errors.push(msg);
|
errors.push(msg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1e. ADS SCAN 扫描版文献直接合并下载 PDF(主要针对早期/不可下载直接 PDF 的文献)
|
// 1e. ADS SCAN 扫描版
|
||||||
let scan_url = format!("https://articles.adsabs.harvard.edu/cgi-bin/nph-iarticle_query?bibcode={}&db_key=AST&data_type=PDF_HIGH", bibcode);
|
let scan_url = format!(
|
||||||
match self
|
"https://articles.adsabs.harvard.edu/cgi-bin/nph-iarticle_query?bibcode={}&db_key=AST&data_type=PDF_HIGH",
|
||||||
.download_pdf_direct(&scan_url, &pdf_dest, "ADS_SCAN")
|
bibcode
|
||||||
.await
|
);
|
||||||
{
|
match self.download_pdf_direct(&scan_url, dest, "ADS_SCAN").await {
|
||||||
Ok(_) => {
|
Ok(_) => return Ok(dest.to_path_buf()),
|
||||||
pdf_res = Ok(pdf_dest.clone());
|
|
||||||
}
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let msg = format!("ADS_SCAN下载失败: {}", e);
|
let msg = format!("ADS_SCAN下载失败: {}", e);
|
||||||
warn!("{}", msg);
|
warn!("{}", msg);
|
||||||
pdf_errors.push(msg);
|
errors.push(msg);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if pdf_res.is_err() && !pdf_errors.is_empty() {
|
if errors.is_empty() {
|
||||||
pdf_res = Err(pdf_errors.join("; "));
|
Err("未尝试任何下载通道".to_string())
|
||||||
|
} else {
|
||||||
|
Err(errors.join("; "))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── HTML 下载 ──────────────────────────────────────────
|
/// HTML 多级回退下载:PUB_HTML → EPRINT_HTML
|
||||||
|
async fn try_download_html(&self, bibcode: &str, dest: &Path) -> Result<PathBuf, String> {
|
||||||
|
let base = "https://ui.adsabs.harvard.edu/link_gateway";
|
||||||
|
let mut errors = Vec::new();
|
||||||
|
|
||||||
info!("[下载] 开始 HTML 下载: {}", bibcode);
|
info!("[下载] 开始 HTML 下载: {}", bibcode);
|
||||||
|
|
||||||
'html: {
|
|
||||||
// 2a. ADS PUB_HTML 网关
|
// 2a. ADS PUB_HTML 网关
|
||||||
let gw = format!("{}/{}/PUB_HTML", base, bibcode);
|
let gw = format!("{}/{}/PUB_HTML", base, bibcode);
|
||||||
match self.resolve_ads_gateway(&gw).await {
|
match self.resolve_ads_gateway(&gw).await {
|
||||||
Ok(resolved) => {
|
Ok(resolved) => {
|
||||||
let result = if resolved.contains("link.springer.com")
|
let result =
|
||||||
|| resolved.contains("nature.com")
|
if resolved.contains("link.springer.com") || resolved.contains("nature.com") {
|
||||||
{
|
|
||||||
// Springer/Nature 专属 HTML 策略
|
|
||||||
let doi_part = resolved
|
let doi_part = resolved
|
||||||
.trim_start_matches("https://link.springer.com/article/")
|
.trim_start_matches("https://link.springer.com/article/")
|
||||||
.trim_start_matches("https://www.nature.com/articles/")
|
.trim_start_matches("https://www.nature.com/articles/")
|
||||||
.trim_end_matches('/');
|
.trim_end_matches('/');
|
||||||
self.download_springer_html(doi_part, &html_dest).await
|
self.download_springer_html(doi_part, dest).await
|
||||||
} else if let Some(arxiv_id) = extract_arxiv_id_from_url(&resolved) {
|
} else if let Some(arxiv_id) = extract_arxiv_id_from_url(&resolved) {
|
||||||
// ADS 网关指向 arXiv abs 页面 → 优先官方 HTML,ar5iv 兜底
|
self.download_arxiv_html_with_fallback(&arxiv_id, dest)
|
||||||
self.download_arxiv_html_with_fallback(&arxiv_id, &html_dest)
|
|
||||||
.await
|
.await
|
||||||
} else {
|
} else {
|
||||||
self.download_html_direct(&resolved, &html_dest, "PUB_HTML")
|
self.download_html_direct(&resolved, dest, "PUB_HTML").await
|
||||||
.await
|
|
||||||
};
|
};
|
||||||
match result {
|
match result {
|
||||||
Ok(_) => {
|
Ok(_) => return Ok(dest.to_path_buf()),
|
||||||
html_res = Ok(html_dest.clone());
|
|
||||||
break 'html;
|
|
||||||
}
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let msg = format!("PUB_HTML下载失败: {}", e);
|
let msg = format!("PUB_HTML下载失败: {}", e);
|
||||||
warn!("{}", msg);
|
warn!("{}", msg);
|
||||||
html_errors.push(msg);
|
errors.push(msg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let msg = format!("PUB_HTML网关解析失败: {}", e);
|
let msg = format!("PUB_HTML网关解析失败: {}", e);
|
||||||
warn!("{}", msg);
|
warn!("{}", msg);
|
||||||
html_errors.push(msg);
|
errors.push(msg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2b. ADS EPRINT_HTML 网关(大多数天文论文有 arXiv eprint)
|
// 2b. ADS EPRINT_HTML 网关
|
||||||
let gw = format!("{}/{}/EPRINT_HTML", base, bibcode);
|
let gw = format!("{}/{}/EPRINT_HTML", base, bibcode);
|
||||||
match self.resolve_ads_gateway(&gw).await {
|
match self.resolve_ads_gateway(&gw).await {
|
||||||
Ok(resolved) => {
|
Ok(resolved) => {
|
||||||
let result = if let Some(arxiv_id) = extract_arxiv_id_from_url(&resolved) {
|
let result = if let Some(arxiv_id) = extract_arxiv_id_from_url(&resolved) {
|
||||||
self.download_arxiv_html_with_fallback(&arxiv_id, &html_dest)
|
self.download_arxiv_html_with_fallback(&arxiv_id, dest)
|
||||||
.await
|
.await
|
||||||
} else {
|
} else {
|
||||||
self.download_html_direct(&resolved, &html_dest, "EPRINT_HTML")
|
self.download_html_direct(&resolved, dest, "EPRINT_HTML")
|
||||||
.await
|
.await
|
||||||
};
|
};
|
||||||
match result {
|
match result {
|
||||||
Ok(_) => {
|
Ok(_) => return Ok(dest.to_path_buf()),
|
||||||
html_res = Ok(html_dest.clone());
|
|
||||||
}
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let msg = format!("EPRINT_HTML下载失败: {}", e);
|
let msg = format!("EPRINT_HTML下载失败: {}", e);
|
||||||
warn!("{}", msg);
|
warn!("{}", msg);
|
||||||
html_errors.push(msg);
|
errors.push(msg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let msg = format!("EPRINT_HTML网关解析失败: {}", e);
|
let msg = format!("EPRINT_HTML网关解析失败: {}", e);
|
||||||
warn!("{}", msg);
|
warn!("{}", msg);
|
||||||
html_errors.push(msg);
|
errors.push(msg);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if html_res.is_err() && !html_errors.is_empty() {
|
if errors.is_empty() {
|
||||||
html_res = Err(html_errors.join("; "));
|
Err("未尝试任何下载通道".to_string())
|
||||||
|
} else {
|
||||||
|
Err(errors.join("; "))
|
||||||
}
|
}
|
||||||
|
|
||||||
(pdf_res, html_res)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 高层文献下载服务,处理数据库记录、多级回退下载、数据库路径写回
|
/// 高层文献下载服务,处理数据库记录、多级回退下载、数据库路径写回
|
||||||
|
|||||||
@ -27,7 +27,7 @@ impl Downloader {
|
|||||||
info!("[Obscura 后备通道] 启动下载: {}", url);
|
info!("[Obscura 后备通道] 启动下载: {}", url);
|
||||||
|
|
||||||
if let Some(parent) = dest_path.parent() {
|
if let Some(parent) = dest_path.parent() {
|
||||||
std::fs::create_dir_all(parent)?;
|
tokio::fs::create_dir_all(parent).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "obscura-inprocess")]
|
#[cfg(feature = "obscura-inprocess")]
|
||||||
@ -190,7 +190,7 @@ impl Downloader {
|
|||||||
use futures_util::StreamExt;
|
use futures_util::StreamExt;
|
||||||
|
|
||||||
if let Some(parent) = target_path.parent() {
|
if let Some(parent) = target_path.parent() {
|
||||||
std::fs::create_dir_all(parent)?;
|
tokio::fs::create_dir_all(parent).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut file = tokio::fs::File::create(target_path)
|
let mut file = tokio::fs::File::create(target_path)
|
||||||
|
|||||||
@ -56,7 +56,11 @@ pub fn init_logging() -> anyhow::Result<Vec<WorkerGuard>> {
|
|||||||
let log_dir = env::var("LOG_DIR").unwrap_or_else(|_| "logs".to_string());
|
let log_dir = env::var("LOG_DIR").unwrap_or_else(|_| "logs".to_string());
|
||||||
fs::create_dir_all(&log_dir).unwrap_or(());
|
fs::create_dir_all(&log_dir).unwrap_or(());
|
||||||
|
|
||||||
let file_appender = rolling::daily(log_dir, "astro_research.log");
|
let file_appender = rolling::RollingFileAppender::builder()
|
||||||
|
.rotation(rolling::Rotation::DAILY)
|
||||||
|
.filename_prefix("astro_research")
|
||||||
|
.filename_suffix("log")
|
||||||
|
.build(log_dir)?;
|
||||||
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
|
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
|
||||||
guards.push(guard);
|
guards.push(guard);
|
||||||
|
|
||||||
|
|||||||
@ -4,12 +4,11 @@
|
|||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
/// 保存研究笔记为本地 Markdown 文件
|
/// 保存研究笔记为本地 Markdown 文件
|
||||||
pub fn save_note_service(
|
pub async fn save_note_service(
|
||||||
library_dir: &Path,
|
library_dir: &Path,
|
||||||
title: &str,
|
title: &str,
|
||||||
content: &str,
|
content: &str,
|
||||||
) -> anyhow::Result<(String, PathBuf, usize)> {
|
) -> anyhow::Result<(String, PathBuf, usize)> {
|
||||||
// 文件名安全化
|
|
||||||
let safe_title: String = title
|
let safe_title: String = title
|
||||||
.chars()
|
.chars()
|
||||||
.map(|c| {
|
.map(|c| {
|
||||||
@ -23,7 +22,7 @@ pub fn save_note_service(
|
|||||||
.replace(' ', "_");
|
.replace(' ', "_");
|
||||||
|
|
||||||
let notes_dir = library_dir.join("notes");
|
let notes_dir = library_dir.join("notes");
|
||||||
std::fs::create_dir_all(¬es_dir)?;
|
tokio::fs::create_dir_all(¬es_dir).await?;
|
||||||
|
|
||||||
let filename = format!("{}.md", safe_title);
|
let filename = format!("{}.md", safe_title);
|
||||||
let filepath = notes_dir.join(&filename);
|
let filepath = notes_dir.join(&filename);
|
||||||
@ -35,7 +34,7 @@ pub fn save_note_service(
|
|||||||
content
|
content
|
||||||
);
|
);
|
||||||
|
|
||||||
std::fs::write(&filepath, &full_content)?;
|
tokio::fs::write(&filepath, &full_content).await?;
|
||||||
|
|
||||||
Ok((filename, filepath, full_content.len()))
|
Ok((filename, filepath, full_content.len()))
|
||||||
}
|
}
|
||||||
@ -103,13 +102,13 @@ pub async fn delete_highlight_note(db: &sqlx::SqlitePool, id: i64) -> Result<boo
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
#[test]
|
#[tokio::test]
|
||||||
fn test_save_note_service() {
|
async fn test_save_note_service() {
|
||||||
let tmp_dir = std::env::temp_dir().join("astro_note_test");
|
let tmp_dir = std::env::temp_dir().join("astro_note_test");
|
||||||
let title = "Test Note Title 123!";
|
let title = "Test Note Title 123!";
|
||||||
let content = "Hello world from note service test";
|
let content = "Hello world from note service test";
|
||||||
|
|
||||||
let res = save_note_service(&tmp_dir, title, content);
|
let res = save_note_service(&tmp_dir, title, content).await;
|
||||||
assert!(res.is_ok());
|
assert!(res.is_ok());
|
||||||
let (filename, filepath, size) = res.unwrap();
|
let (filename, filepath, size) = res.unwrap();
|
||||||
assert_eq!(filename, "Test_Note_Title_123_.md");
|
assert_eq!(filename, "Test_Note_Title_123_.md");
|
||||||
|
|||||||
@ -198,26 +198,39 @@ pub struct CachedPage {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// 列出已缓存产物(服务端分页 + 筛选 + 排序)
|
/// 列出已缓存产物(服务端分页 + 筛选 + 排序)
|
||||||
|
///
|
||||||
|
/// 所有筛选值通过 SQL 参数绑定(`?`)传入,杜绝 SQL 注入与 LIKE 通配符(`%`/`_`)误匹配。
|
||||||
|
/// 排序方向用白名单(只接受 created/source/product 三个字面量)。
|
||||||
pub async fn list_cached_paged(pool: &SqlitePool, params: &ListCachedParams) -> Result<CachedPage> {
|
pub async fn list_cached_paged(pool: &SqlitePool, params: &ListCachedParams) -> Result<CachedPage> {
|
||||||
let limit = params.limit.unwrap_or(50).clamp(1, 200);
|
let limit = params.limit.unwrap_or(50).clamp(1, 200);
|
||||||
let offset = params.offset.unwrap_or(0).max(0);
|
let offset = params.offset.unwrap_or(0).max(0);
|
||||||
|
|
||||||
// 动态构造 SQL(参数绑定防注入;列名/排序方向用白名单)
|
// 动态构造 WHERE 子句(用 ? 占位符)+ 同步收集绑定值
|
||||||
let mut where_clauses: Vec<String> = Vec::new();
|
// bind_values 顺序必须与占位符出现顺序一致:source → product → q → limit → offset
|
||||||
|
let mut where_parts: Vec<&'static str> = Vec::new();
|
||||||
|
let mut bind_source: Option<String> = None;
|
||||||
|
let mut bind_product: Option<String> = None;
|
||||||
|
let mut bind_q: Option<String> = None; // 已包成 '%q%',且 LIKE 通配符已转义
|
||||||
|
|
||||||
if let Some(s) = ¶ms.source {
|
if let Some(s) = ¶ms.source {
|
||||||
where_clauses.push(format!("source = '{}'", s.replace('\'', "''")));
|
where_parts.push("source = ?");
|
||||||
|
bind_source = Some(s.clone());
|
||||||
}
|
}
|
||||||
if let Some(p) = ¶ms.product {
|
if let Some(p) = ¶ms.product {
|
||||||
where_clauses.push(format!("product = '{}'", p.replace('\'', "''")));
|
where_parts.push("product = ?");
|
||||||
|
bind_product = Some(p.clone());
|
||||||
}
|
}
|
||||||
if let Some(q) = ¶ms.q {
|
if let Some(q) = ¶ms.q {
|
||||||
let q = q.replace('\'', "''");
|
// 转义 LIKE 通配符,避免用户输入的 %/_ 被解释为模式
|
||||||
where_clauses.push(format!("source_id LIKE '%{}%'", q));
|
let escaped = q.replace('%', "\\%").replace('_', "\\_");
|
||||||
|
where_parts.push("source_id LIKE ? ESCAPE '\\'");
|
||||||
|
bind_q = Some(format!("%{}%", escaped));
|
||||||
}
|
}
|
||||||
let where_sql = if where_clauses.is_empty() {
|
|
||||||
|
let where_sql = if where_parts.is_empty() {
|
||||||
String::new()
|
String::new()
|
||||||
} else {
|
} else {
|
||||||
format!(" WHERE {}", where_clauses.join(" AND "))
|
format!(" WHERE {}", where_parts.join(" AND "))
|
||||||
};
|
};
|
||||||
|
|
||||||
let order_sql = match params.sort.as_deref() {
|
let order_sql = match params.sort.as_deref() {
|
||||||
@ -226,22 +239,41 @@ pub async fn list_cached_paged(pool: &SqlitePool, params: &ListCachedParams) ->
|
|||||||
_ => " ORDER BY created_at DESC",
|
_ => " ORDER BY created_at DESC",
|
||||||
};
|
};
|
||||||
|
|
||||||
// count
|
// count(绑定 source/product/q,不绑 limit/offset)
|
||||||
let count_sql = format!("SELECT COUNT(*) as cnt FROM observation_cache{}", where_sql);
|
let count_sql = format!("SELECT COUNT(*) FROM observation_cache{}", where_sql);
|
||||||
let total: i64 = sqlx::query_scalar(&count_sql)
|
let mut count_q = sqlx::query_scalar::<_, i64>(&count_sql);
|
||||||
|
if let Some(v) = &bind_source {
|
||||||
|
count_q = count_q.bind(v);
|
||||||
|
}
|
||||||
|
if let Some(v) = &bind_product {
|
||||||
|
count_q = count_q.bind(v);
|
||||||
|
}
|
||||||
|
if let Some(v) = &bind_q {
|
||||||
|
count_q = count_q.bind(v);
|
||||||
|
}
|
||||||
|
let total = count_q
|
||||||
.fetch_one(pool)
|
.fetch_one(pool)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| anyhow!("统计 observation_cache 总数失败: {}", e))?;
|
.map_err(|e| anyhow!("统计 observation_cache 总数失败: {}", e))?;
|
||||||
|
|
||||||
// items
|
// items(绑定 source/product/q/limit/offset)
|
||||||
let items_sql = format!(
|
let items_sql = format!(
|
||||||
"SELECT source, product, source_id, ra, dec, artifacts_json, meta_json, created_at \
|
"SELECT source, product, source_id, ra, dec, artifacts_json, meta_json, created_at \
|
||||||
FROM observation_cache{}{} LIMIT ? OFFSET ?",
|
FROM observation_cache{}{} LIMIT ? OFFSET ?",
|
||||||
where_sql, order_sql
|
where_sql, order_sql
|
||||||
);
|
);
|
||||||
let items = sqlx::query_as::<_, ObservationCacheRow>(&items_sql)
|
let mut items_q = sqlx::query_as::<_, ObservationCacheRow>(&items_sql);
|
||||||
.bind(limit)
|
if let Some(v) = &bind_source {
|
||||||
.bind(offset)
|
items_q = items_q.bind(v);
|
||||||
|
}
|
||||||
|
if let Some(v) = &bind_product {
|
||||||
|
items_q = items_q.bind(v);
|
||||||
|
}
|
||||||
|
if let Some(v) = &bind_q {
|
||||||
|
items_q = items_q.bind(v);
|
||||||
|
}
|
||||||
|
items_q = items_q.bind(limit).bind(offset);
|
||||||
|
let items = items_q
|
||||||
.fetch_all(pool)
|
.fetch_all(pool)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| anyhow!("查询 observation_cache 分页列表失败: {}", e))?;
|
.map_err(|e| anyhow!("查询 observation_cache 分页列表失败: {}", e))?;
|
||||||
@ -259,26 +291,27 @@ pub fn file_url_from_path(rel_path: &str) -> String {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// 落盘字节到 library_dir/<rel_path>,返回字节数
|
/// 落盘字节到 library_dir/<rel_path>,返回字节数
|
||||||
pub fn persist_bytes(library_dir: &Path, rel_path: &str, bytes: &[u8]) -> Result<usize> {
|
pub async fn persist_bytes(library_dir: &Path, rel_path: &str, bytes: &[u8]) -> Result<usize> {
|
||||||
let abs_path = library_dir.join(rel_path);
|
let abs_path = library_dir.join(rel_path);
|
||||||
if let Some(parent) = abs_path.parent() {
|
if let Some(parent) = abs_path.parent() {
|
||||||
std::fs::create_dir_all(parent)
|
tokio::fs::create_dir_all(parent)
|
||||||
|
.await
|
||||||
.map_err(|e| anyhow!("创建观测数据目录失败 {:?}: {}", parent, e))?;
|
.map_err(|e| anyhow!("创建观测数据目录失败 {:?}: {}", parent, e))?;
|
||||||
}
|
}
|
||||||
std::fs::write(&abs_path, bytes)
|
tokio::fs::write(&abs_path, bytes)
|
||||||
|
.await
|
||||||
.map_err(|e| anyhow!("写入观测数据文件失败 {:?}: {}", abs_path, e))?;
|
.map_err(|e| anyhow!("写入观测数据文件失败 {:?}: {}", abs_path, e))?;
|
||||||
Ok(bytes.len())
|
Ok(bytes.len())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 校验缓存产物中至少有一个文件真实存在;存在则返回其总大小
|
/// 校验缓存产物中至少有一个文件真实存在;存在则返回其总大小
|
||||||
pub fn cached_files_total_size(library_dir: &Path, artifacts: &[Artifact]) -> Option<usize> {
|
pub async fn cached_files_total_size(library_dir: &Path, artifacts: &[Artifact]) -> Option<usize> {
|
||||||
let mut total = 0;
|
let mut total = 0;
|
||||||
for a in artifacts {
|
for a in artifacts {
|
||||||
let abs_path = library_dir.join(&a.file_path);
|
let abs_path = library_dir.join(&a.file_path);
|
||||||
match std::fs::metadata(&abs_path) {
|
match tokio::fs::metadata(&abs_path).await {
|
||||||
Ok(m) => total += m.len() as usize,
|
Ok(m) => total += m.len() as usize,
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
// 任一文件缺失即视为缓存不完整
|
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -292,7 +325,10 @@ pub fn cached_files_total_size(library_dir: &Path, artifacts: &[Artifact]) -> Op
|
|||||||
|
|
||||||
/// 记录缓存写入失败但不中断流程
|
/// 记录缓存写入失败但不中断流程
|
||||||
pub fn log_write_failure(source: &str, product: &str, e: anyhow::Error) {
|
pub fn log_write_failure(source: &str, product: &str, e: anyhow::Error) {
|
||||||
error!("[Observation] 写入 observation_cache 失败 ({}/{}): {}", source, product, e);
|
error!(
|
||||||
|
"[Observation] 写入 observation_cache 失败 ({}/{}): {}",
|
||||||
|
source, product, e
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@ -22,11 +22,11 @@ use tracing::{info, warn};
|
|||||||
|
|
||||||
// ── 版本枚举 ──
|
// ── 版本枚举 ──
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||||
pub enum DesiRelease { Edr, Dr1 }
|
pub enum DesiRelease {
|
||||||
|
Edr,
|
||||||
impl Default for DesiRelease {
|
#[default]
|
||||||
fn default() -> Self { Self::Dr1 }
|
Dr1,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DesiRelease {
|
impl DesiRelease {
|
||||||
@ -34,16 +34,28 @@ impl DesiRelease {
|
|||||||
&["edr", "dr1"]
|
&["edr", "dr1"]
|
||||||
}
|
}
|
||||||
pub fn dr_segment(&self) -> &'static str {
|
pub fn dr_segment(&self) -> &'static str {
|
||||||
match self { Self::Edr => "edr", Self::Dr1 => "dr1" }
|
match self {
|
||||||
|
Self::Edr => "edr",
|
||||||
|
Self::Dr1 => "dr1",
|
||||||
|
}
|
||||||
}
|
}
|
||||||
pub fn specredux_ver(&self) -> &'static str {
|
pub fn specredux_ver(&self) -> &'static str {
|
||||||
match self { Self::Edr => "fuji", Self::Dr1 => "iron" }
|
match self {
|
||||||
|
Self::Edr => "fuji",
|
||||||
|
Self::Dr1 => "iron",
|
||||||
|
}
|
||||||
}
|
}
|
||||||
pub fn datalab_table(&self) -> &'static str {
|
pub fn datalab_table(&self) -> &'static str {
|
||||||
match self { Self::Edr => "desi_edr.zpix", Self::Dr1 => "desi_dr1.zpix" }
|
match self {
|
||||||
|
Self::Edr => "desi_edr.zpix",
|
||||||
|
Self::Dr1 => "desi_dr1.zpix",
|
||||||
|
}
|
||||||
}
|
}
|
||||||
pub fn display(&self) -> &'static str {
|
pub fn display(&self) -> &'static str {
|
||||||
match self { Self::Edr => "EDR", Self::Dr1 => "DR1" }
|
match self {
|
||||||
|
Self::Edr => "EDR",
|
||||||
|
Self::Dr1 => "DR1",
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -52,7 +64,13 @@ pub fn parse_desi_release(s: Option<&str>) -> Result<DesiRelease> {
|
|||||||
None => DesiRelease::Dr1,
|
None => DesiRelease::Dr1,
|
||||||
Some("edr") => DesiRelease::Edr,
|
Some("edr") => DesiRelease::Edr,
|
||||||
Some("dr1") => DesiRelease::Dr1,
|
Some("dr1") => DesiRelease::Dr1,
|
||||||
Some(other) => return Err(anyhow!("不支持的 DESI release '{}',可选: {:?}", other, DesiRelease::valid_values())),
|
Some(other) => {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"不支持的 DESI release '{}',可选: {:?}",
|
||||||
|
other,
|
||||||
|
DesiRelease::valid_values()
|
||||||
|
))
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -114,8 +132,12 @@ impl ObservationFetcher for DesiSpectrumFetcher {
|
|||||||
async fn cone_search_raw(
|
async fn cone_search_raw(
|
||||||
&self,
|
&self,
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
ra: f64, dec: f64, radius_deg: f64,
|
ra: f64,
|
||||||
release: Option<&str>, _subtype: Option<&str>,
|
dec: f64,
|
||||||
|
radius_deg: f64,
|
||||||
|
release: Option<&str>,
|
||||||
|
_subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
) -> Result<Vec<Candidate>> {
|
) -> Result<Vec<Candidate>> {
|
||||||
validate_cone_params(ra, dec, radius_deg)?;
|
validate_cone_params(ra, dec, radius_deg)?;
|
||||||
let rel = parse_desi_release(release)?;
|
let rel = parse_desi_release(release)?;
|
||||||
@ -128,6 +150,7 @@ impl ObservationFetcher for DesiSpectrumFetcher {
|
|||||||
identifier: &str,
|
identifier: &str,
|
||||||
_release: Option<&str>,
|
_release: Option<&str>,
|
||||||
_subtype: Option<&str>,
|
_subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
) -> Result<Candidate> {
|
) -> Result<Candidate> {
|
||||||
// 格式 "{survey}-{program}-{healpix}"
|
// 格式 "{survey}-{program}-{healpix}"
|
||||||
let parts: Vec<&str> = identifier.split('-').collect();
|
let parts: Vec<&str> = identifier.split('-').collect();
|
||||||
@ -137,7 +160,28 @@ impl ObservationFetcher for DesiSpectrumFetcher {
|
|||||||
identifier
|
identifier
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let healpix: i64 = parts[2].parse()
|
let survey = parts[0];
|
||||||
|
let program = parts[1];
|
||||||
|
// 白名单校验:survey/program 会拼进文件系统路径和下载 URL,
|
||||||
|
// 必须防止路径穿越(如 "../etc")。仅允许 DESI 实际使用的取值。
|
||||||
|
const VALID_SURVEYS: &[&str] = &["main", "sv1", "sv2", "sv3", "sv", "backup", "cumulative"];
|
||||||
|
const VALID_PROGRAMS: &[&str] = &["dark", "bright", "backup", "other"];
|
||||||
|
if !VALID_SURVEYS.contains(&survey) {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"DESI survey '{}' 不在允许列表 {:?} 中",
|
||||||
|
survey,
|
||||||
|
VALID_SURVEYS
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !VALID_PROGRAMS.contains(&program) {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"DESI program '{}' 不在允许列表 {:?} 中",
|
||||||
|
program,
|
||||||
|
VALID_PROGRAMS
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let healpix: i64 = parts[2]
|
||||||
|
.parse()
|
||||||
.map_err(|_| anyhow!("healpix 非数字: '{}'", parts[2]))?;
|
.map_err(|_| anyhow!("healpix 非数字: '{}'", parts[2]))?;
|
||||||
if healpix < 0 {
|
if healpix < 0 {
|
||||||
return Err(anyhow!("healpix 应为非负整数,当前: {}", healpix));
|
return Err(anyhow!("healpix 应为非负整数,当前: {}", healpix));
|
||||||
@ -145,10 +189,12 @@ impl ObservationFetcher for DesiSpectrumFetcher {
|
|||||||
Ok(Candidate {
|
Ok(Candidate {
|
||||||
source: Source::Desi,
|
source: Source::Desi,
|
||||||
source_id: identifier.to_string(),
|
source_id: identifier.to_string(),
|
||||||
label: format!("{} {} healpix {}", parts[0], parts[1], healpix),
|
label: format!("{} {} healpix {}", survey, program, healpix),
|
||||||
ra: None, dec: None, distance: None,
|
ra: None,
|
||||||
|
dec: None,
|
||||||
|
distance: None,
|
||||||
raw: Some(serde_json::json!({
|
raw: Some(serde_json::json!({
|
||||||
"survey": parts[0], "program": parts[1], "healpix": healpix
|
"survey": survey, "program": program, "healpix": healpix
|
||||||
})),
|
})),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -159,13 +205,26 @@ impl ObservationFetcher for DesiSpectrumFetcher {
|
|||||||
candidate: &Candidate,
|
candidate: &Candidate,
|
||||||
release: Option<&str>,
|
release: Option<&str>,
|
||||||
_subtype: Option<&str>,
|
_subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
force: bool,
|
force: bool,
|
||||||
) -> Result<ObservationProduct> {
|
) -> Result<ObservationProduct> {
|
||||||
let survey = candidate.raw.as_ref().and_then(|v| v.get("survey")).and_then(|v| v.as_str())
|
let survey = candidate
|
||||||
|
.raw
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("survey"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
.ok_or_else(|| anyhow!("DESI Candidate 缺 survey"))?;
|
.ok_or_else(|| anyhow!("DESI Candidate 缺 survey"))?;
|
||||||
let program = candidate.raw.as_ref().and_then(|v| v.get("program")).and_then(|v| v.as_str())
|
let program = candidate
|
||||||
|
.raw
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("program"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
.ok_or_else(|| anyhow!("DESI Candidate 缺 program"))?;
|
.ok_or_else(|| anyhow!("DESI Candidate 缺 program"))?;
|
||||||
let healpix = candidate.raw.as_ref().and_then(|v| v.get("healpix")).and_then(|v| v.as_i64())
|
let healpix = candidate
|
||||||
|
.raw
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("healpix"))
|
||||||
|
.and_then(|v| v.as_i64())
|
||||||
.ok_or_else(|| anyhow!("DESI Candidate 缺 healpix"))?;
|
.ok_or_else(|| anyhow!("DESI Candidate 缺 healpix"))?;
|
||||||
let rel = parse_desi_release(release)?;
|
let rel = parse_desi_release(release)?;
|
||||||
let product = ProductSpec::with_subtype(ProductType::Spectrum, "coadd");
|
let product = ProductSpec::with_subtype(ProductType::Spectrum, "coadd");
|
||||||
@ -175,12 +234,21 @@ impl ObservationFetcher for DesiSpectrumFetcher {
|
|||||||
|
|
||||||
// 1) 缓存命中
|
// 1) 缓存命中
|
||||||
if !force {
|
if !force {
|
||||||
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Desi, &product, &cache_key).await? {
|
if let Some((artifacts, meta)) =
|
||||||
if cached_files_total_size(&state.config.library_dir, &artifacts).is_some() {
|
fetch_observation_cache(&state.db, Source::Desi, &product, &cache_key).await?
|
||||||
|
{
|
||||||
|
if cached_files_total_size(&state.config.library_dir, &artifacts)
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
info!("[DESI] coadd 缓存命中 ({})", cache_key);
|
info!("[DESI] coadd 缓存命中 ({})", cache_key);
|
||||||
return Ok(ObservationProduct {
|
return Ok(ObservationProduct {
|
||||||
source: Source::Desi, product,
|
source: Source::Desi,
|
||||||
source_id: cache_key, source_label, artifacts, source_meta: meta,
|
product,
|
||||||
|
source_id: cache_key,
|
||||||
|
source_label,
|
||||||
|
artifacts,
|
||||||
|
source_meta: meta,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
warn!("[DESI] 缓存记录存在但文件缺失,重新下载 ({})", cache_key);
|
warn!("[DESI] 缓存记录存在但文件缺失,重新下载 ({})", cache_key);
|
||||||
@ -189,34 +257,57 @@ impl ObservationFetcher for DesiSpectrumFetcher {
|
|||||||
|
|
||||||
// 2) 下载 coadd FITS(无压缩)
|
// 2) 下载 coadd FITS(无压缩)
|
||||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
let fits_bytes = state.desi.download_coadd(survey, program, healpix, rel).await?;
|
let fits_bytes = state
|
||||||
|
.desi
|
||||||
|
.download_coadd(survey, program, healpix, rel)
|
||||||
|
.await?;
|
||||||
let rel_path = format!(
|
let rel_path = format!(
|
||||||
"Telescope/desi/{dr}/{survey}-{program}/{healpix}.fits",
|
"Telescope/desi/{dr}/{survey}-{program}/{healpix}.fits",
|
||||||
dr = dr, survey = survey, program = program, healpix = healpix
|
dr = dr,
|
||||||
|
survey = survey,
|
||||||
|
program = program,
|
||||||
|
healpix = healpix
|
||||||
|
);
|
||||||
|
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes).await?;
|
||||||
|
info!(
|
||||||
|
"[DESI] coadd 已保存 {} → {} ({}B)",
|
||||||
|
cache_key, rel_path, size
|
||||||
);
|
);
|
||||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes)?;
|
|
||||||
info!("[DESI] coadd 已保存 {} → {} ({}B)", cache_key, rel_path, size);
|
|
||||||
|
|
||||||
let meta = serde_json::json!({"survey": survey, "program": program, "healpix": healpix, "dr": dr});
|
let meta =
|
||||||
|
serde_json::json!({"survey": survey, "program": program, "healpix": healpix, "dr": dr});
|
||||||
let artifact = Artifact {
|
let artifact = Artifact {
|
||||||
band: None, original_name: None,
|
band: None,
|
||||||
|
original_name: None,
|
||||||
file_path: rel_path.clone(),
|
file_path: rel_path.clone(),
|
||||||
file_url: file_url_from_path(&rel_path),
|
file_url: file_url_from_path(&rel_path),
|
||||||
file_format: "fits".to_string(),
|
file_format: "fits".to_string(),
|
||||||
size_bytes: size, cached: false,
|
size_bytes: size,
|
||||||
|
cached: false,
|
||||||
};
|
};
|
||||||
let meta_str = meta.to_string();
|
let meta_str = meta.to_string();
|
||||||
if let Err(e) = write_observation_cache(
|
if let Err(e) = write_observation_cache(
|
||||||
&state.db, Source::Desi, &product, &cache_key, None, None,
|
&state.db,
|
||||||
std::slice::from_ref(&artifact), Some(&meta_str),
|
Source::Desi,
|
||||||
).await {
|
&product,
|
||||||
|
&cache_key,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
std::slice::from_ref(&artifact),
|
||||||
|
Some(&meta_str),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
log_write_failure(Source::Desi.as_str(), "spectrum", e);
|
log_write_failure(Source::Desi.as_str(), "spectrum", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(ObservationProduct {
|
Ok(ObservationProduct {
|
||||||
source: Source::Desi, product,
|
source: Source::Desi,
|
||||||
source_id: cache_key, source_label,
|
product,
|
||||||
artifacts: vec![artifact], source_meta: Some(meta),
|
source_id: cache_key,
|
||||||
|
source_label,
|
||||||
|
artifacts: vec![artifact],
|
||||||
|
source_meta: Some(meta),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -226,7 +317,9 @@ fn row_to_candidate(row: &DesiSpectrumRow) -> Candidate {
|
|||||||
source: Source::Desi,
|
source: Source::Desi,
|
||||||
source_id: format!("{}-{}-{}", row.survey, row.program, row.healpix),
|
source_id: format!("{}-{}-{}", row.survey, row.program, row.healpix),
|
||||||
label: format!("{} {} healpix {}", row.survey, row.program, row.healpix),
|
label: format!("{} {} healpix {}", row.survey, row.program, row.healpix),
|
||||||
ra: row.ra, dec: row.dec, distance: None,
|
ra: row.ra,
|
||||||
|
dec: row.dec,
|
||||||
|
distance: None,
|
||||||
raw: Some(serde_json::json!({
|
raw: Some(serde_json::json!({
|
||||||
"survey": row.survey, "program": row.program, "healpix": row.healpix
|
"survey": row.survey, "program": row.program, "healpix": row.healpix
|
||||||
})),
|
})),
|
||||||
|
|||||||
@ -30,6 +30,7 @@ pub async fn search_observation(
|
|||||||
dec: f64,
|
dec: f64,
|
||||||
radius_deg: f64,
|
radius_deg: f64,
|
||||||
release: Option<&str>,
|
release: Option<&str>,
|
||||||
|
version: Option<&str>,
|
||||||
) -> Result<Vec<Candidate>> {
|
) -> Result<Vec<Candidate>> {
|
||||||
let subtype = product.subtype.as_deref();
|
let subtype = product.subtype.as_deref();
|
||||||
let fetcher = registry.get(source, product.product, subtype)?;
|
let fetcher = registry.get(source, product.product, subtype)?;
|
||||||
@ -38,7 +39,7 @@ pub async fn search_observation(
|
|||||||
source, product.product, ra, dec, radius_deg
|
source, product.product, ra, dec, radius_deg
|
||||||
);
|
);
|
||||||
let matched = fetcher
|
let matched = fetcher
|
||||||
.cone_search_cached(state, ra, dec, radius_deg, release, subtype)
|
.cone_search_cached(state, ra, dec, radius_deg, release, subtype, version)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(matched)
|
Ok(matched)
|
||||||
}
|
}
|
||||||
@ -54,19 +55,48 @@ pub async fn download_observation(
|
|||||||
) -> Result<ObservationBatch> {
|
) -> Result<ObservationBatch> {
|
||||||
match request {
|
match request {
|
||||||
ObservationRequest::ByCoordinates {
|
ObservationRequest::ByCoordinates {
|
||||||
source, product, ra, dec, radius_deg, strategy, release,
|
source,
|
||||||
|
product,
|
||||||
|
ra,
|
||||||
|
dec,
|
||||||
|
radius_deg,
|
||||||
|
strategy,
|
||||||
|
release,
|
||||||
|
version,
|
||||||
} => {
|
} => {
|
||||||
download_by_coordinates(
|
download_by_coordinates(
|
||||||
state, registry, *source, product, *ra, *dec, *radius_deg,
|
state,
|
||||||
*strategy, release.as_deref(), force,
|
registry,
|
||||||
).await
|
*source,
|
||||||
|
product,
|
||||||
|
*ra,
|
||||||
|
*dec,
|
||||||
|
*radius_deg,
|
||||||
|
*strategy,
|
||||||
|
release.as_deref(),
|
||||||
|
version.as_deref(),
|
||||||
|
force,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
ObservationRequest::ByIdentifier {
|
ObservationRequest::ByIdentifier {
|
||||||
source, product, identifiers, release,
|
source,
|
||||||
|
product,
|
||||||
|
identifiers,
|
||||||
|
release,
|
||||||
|
version,
|
||||||
} => {
|
} => {
|
||||||
download_by_identifiers(
|
download_by_identifiers(
|
||||||
state, registry, *source, product, identifiers, release.as_deref(), force,
|
state,
|
||||||
).await
|
registry,
|
||||||
|
*source,
|
||||||
|
product,
|
||||||
|
identifiers,
|
||||||
|
release.as_deref(),
|
||||||
|
version.as_deref(),
|
||||||
|
force,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -77,9 +107,12 @@ async fn download_by_coordinates(
|
|||||||
registry: &ObservationRegistry,
|
registry: &ObservationRegistry,
|
||||||
source: crate::services::observation::types::Source,
|
source: crate::services::observation::types::Source,
|
||||||
product: &ProductSpec,
|
product: &ProductSpec,
|
||||||
ra: f64, dec: f64, radius_deg: f64,
|
ra: f64,
|
||||||
|
dec: f64,
|
||||||
|
radius_deg: f64,
|
||||||
strategy: FindStrategy,
|
strategy: FindStrategy,
|
||||||
release: Option<&str>,
|
release: Option<&str>,
|
||||||
|
version: Option<&str>,
|
||||||
force: bool,
|
force: bool,
|
||||||
) -> Result<ObservationBatch> {
|
) -> Result<ObservationBatch> {
|
||||||
let subtype = product.subtype.as_deref();
|
let subtype = product.subtype.as_deref();
|
||||||
@ -89,14 +122,21 @@ async fn download_by_coordinates(
|
|||||||
source, product.product, ra, dec, radius_deg, strategy
|
source, product.product, ra, dec, radius_deg, strategy
|
||||||
);
|
);
|
||||||
|
|
||||||
let matched = fetcher.cone_search_cached(state, ra, dec, radius_deg, release, subtype).await?;
|
let matched = fetcher
|
||||||
|
.cone_search_cached(state, ra, dec, radius_deg, release, subtype, version)
|
||||||
|
.await?;
|
||||||
let matched_count = matched.len();
|
let matched_count = matched.len();
|
||||||
|
|
||||||
if matched_count == 0 {
|
if matched_count == 0 {
|
||||||
return Ok(ObservationBatch {
|
return Ok(ObservationBatch {
|
||||||
source, product: product.clone(),
|
source,
|
||||||
ra: Some(ra), dec: Some(dec), radius_deg: Some(radius_deg),
|
product: product.clone(),
|
||||||
matched_count: 0, products: Vec::new(), failures: Vec::new(),
|
ra: Some(ra),
|
||||||
|
dec: Some(dec),
|
||||||
|
radius_deg: Some(radius_deg),
|
||||||
|
matched_count: 0,
|
||||||
|
products: Vec::new(),
|
||||||
|
failures: Vec::new(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -109,7 +149,10 @@ async fn download_by_coordinates(
|
|||||||
let mut failures = Vec::new();
|
let mut failures = Vec::new();
|
||||||
for idx in targets {
|
for idx in targets {
|
||||||
let candidate = &matched[idx];
|
let candidate = &matched[idx];
|
||||||
match fetcher.fetch(state, candidate, release, subtype, force).await {
|
match fetcher
|
||||||
|
.fetch(state, candidate, release, subtype, version, force)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(p) => products.push(p),
|
Ok(p) => products.push(p),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let label = candidate.label.clone();
|
let label = candidate.label.clone();
|
||||||
@ -123,9 +166,14 @@ async fn download_by_coordinates(
|
|||||||
}
|
}
|
||||||
|
|
||||||
Ok(ObservationBatch {
|
Ok(ObservationBatch {
|
||||||
source, product: product.clone(),
|
source,
|
||||||
ra: Some(ra), dec: Some(dec), radius_deg: Some(radius_deg),
|
product: product.clone(),
|
||||||
matched_count, products, failures,
|
ra: Some(ra),
|
||||||
|
dec: Some(dec),
|
||||||
|
radius_deg: Some(radius_deg),
|
||||||
|
matched_count,
|
||||||
|
products,
|
||||||
|
failures,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -137,20 +185,34 @@ async fn download_by_identifiers(
|
|||||||
product: &ProductSpec,
|
product: &ProductSpec,
|
||||||
identifiers: &[String],
|
identifiers: &[String],
|
||||||
release: Option<&str>,
|
release: Option<&str>,
|
||||||
|
version: Option<&str>,
|
||||||
force: bool,
|
force: bool,
|
||||||
) -> Result<ObservationBatch> {
|
) -> Result<ObservationBatch> {
|
||||||
let subtype = product.subtype.as_deref();
|
let subtype = product.subtype.as_deref();
|
||||||
let fetcher = registry.get(source, product.product, subtype)?;
|
let fetcher = registry.get(source, product.product, subtype)?;
|
||||||
|
if identifiers.is_empty() {
|
||||||
|
return Err(anyhow::anyhow!(
|
||||||
|
"标识符列表不能为空(source={:?} product={:?})",
|
||||||
|
source,
|
||||||
|
product.product
|
||||||
|
));
|
||||||
|
}
|
||||||
info!(
|
info!(
|
||||||
"[Observation] by_id source={:?} product={:?} count={} release={:?}",
|
"[Observation] by_id source={:?} product={:?} count={} release={:?}",
|
||||||
source, product.product, identifiers.len(), release
|
source,
|
||||||
|
product.product,
|
||||||
|
identifiers.len(),
|
||||||
|
release
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut products = Vec::with_capacity(identifiers.len());
|
let mut products = Vec::with_capacity(identifiers.len());
|
||||||
let mut failures = Vec::new();
|
let mut failures = Vec::new();
|
||||||
for id in identifiers {
|
for id in identifiers {
|
||||||
// 先把标识符解析为 Candidate(验证格式 + 提取字段)
|
// 先把标识符解析为 Candidate(验证格式 + 提取字段)
|
||||||
let candidate = match fetcher.resolve_identifier(id, release, subtype).await {
|
let candidate = match fetcher
|
||||||
|
.resolve_identifier(id, release, subtype, version)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(c) => c,
|
Ok(c) => c,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
warn!("[Observation] 标识符解析失败 {}: {}", id, e);
|
warn!("[Observation] 标识符解析失败 {}: {}", id, e);
|
||||||
@ -161,7 +223,10 @@ async fn download_by_identifiers(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
match fetcher.fetch(state, &candidate, release, subtype, force).await {
|
match fetcher
|
||||||
|
.fetch(state, &candidate, release, subtype, version, force)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(p) => products.push(p),
|
Ok(p) => products.push(p),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
warn!("[Observation] 下载失败 {}: {}", id, e);
|
warn!("[Observation] 下载失败 {}: {}", id, e);
|
||||||
@ -174,10 +239,14 @@ async fn download_by_identifiers(
|
|||||||
}
|
}
|
||||||
|
|
||||||
Ok(ObservationBatch {
|
Ok(ObservationBatch {
|
||||||
source, product: product.clone(),
|
source,
|
||||||
ra: None, dec: None, radius_deg: None,
|
product: product.clone(),
|
||||||
|
ra: None,
|
||||||
|
dec: None,
|
||||||
|
radius_deg: None,
|
||||||
matched_count: products.len() + failures.len(),
|
matched_count: products.len() + failures.len(),
|
||||||
products, failures,
|
products,
|
||||||
|
failures,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -189,10 +258,15 @@ pub async fn download_one(
|
|||||||
product: ProductSpec,
|
product: ProductSpec,
|
||||||
identifier: &str,
|
identifier: &str,
|
||||||
release: Option<&str>,
|
release: Option<&str>,
|
||||||
|
version: Option<&str>,
|
||||||
force: bool,
|
force: bool,
|
||||||
) -> Result<ObservationProduct> {
|
) -> Result<ObservationProduct> {
|
||||||
let subtype = product.subtype.as_deref();
|
let subtype = product.subtype.as_deref();
|
||||||
let fetcher = registry.get(source, product.product, subtype)?;
|
let fetcher = registry.get(source, product.product, subtype)?;
|
||||||
let candidate = fetcher.resolve_identifier(identifier, release, subtype).await?;
|
let candidate = fetcher
|
||||||
fetcher.fetch(state, &candidate, release, subtype, force).await
|
.resolve_identifier(identifier, release, subtype, version)
|
||||||
|
.await?;
|
||||||
|
fetcher
|
||||||
|
.fetch(state, &candidate, release, subtype, version, force)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|||||||
@ -76,6 +76,33 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 该 release 支持的子版本列表(仅 LAMOST 有意义),返回空表示不区分子版本。
|
||||||
|
/// `release` 为 None 时返回默认 release 的子版本列表。
|
||||||
|
fn versions_for_release(&self, _release: Option<&str>) -> Vec<String> {
|
||||||
|
Vec::new()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 该 release 的默认子版本(None 表示无子版本概念或用默认 release)
|
||||||
|
fn default_version_for_release(&self, _release: Option<&str>) -> Option<String> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 需要登录认证才能访问的 release 列表(如 LAMOST Internal DR),空表示全部公开。
|
||||||
|
///
|
||||||
|
/// 前端据此对这些 release 做禁用/灰显处理。
|
||||||
|
fn releases_requiring_auth(&self) -> Vec<String> {
|
||||||
|
Vec::new()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 每个 release 实际支持的 subtype 列表(用于表达 release×subtype 交叉约束)。
|
||||||
|
///
|
||||||
|
/// 默认返回空 HashMap,表示所有 release 都支持 `subtypes()` 的全部值。
|
||||||
|
/// 仅当某些 release 不支持某些 subtype 时才需覆写(如 LAMOST DR5/6 不支持 MRS)。
|
||||||
|
/// key=release 字符串,value=该 release 支持的 subtype 列表。
|
||||||
|
fn subtypes_per_release(&self) -> std::collections::HashMap<String, Vec<String>> {
|
||||||
|
std::collections::HashMap::new()
|
||||||
|
}
|
||||||
|
|
||||||
/// 坐标模式检索半径的建议上限(度),用于前端表单提示与服务端警告
|
/// 坐标模式检索半径的建议上限(度),用于前端表单提示与服务端警告
|
||||||
///
|
///
|
||||||
/// 各源主表规模差异极大(Gaia/SDSS/DESI 主表上亿行,大范围查询易超时),
|
/// 各源主表规模差异极大(Gaia/SDSS/DESI 主表上亿行,大范围查询易超时),
|
||||||
@ -109,6 +136,8 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
|||||||
// ── 检索(各源实现) ──
|
// ── 检索(各源实现) ──
|
||||||
|
|
||||||
/// 纯 cone 检索(不查缓存)—— 各源实现自己的端点调用与行映射
|
/// 纯 cone 检索(不查缓存)—— 各源实现自己的端点调用与行映射
|
||||||
|
///
|
||||||
|
/// `version` 为该 release 的子版本(仅 LAMOST 有意义,其他源可忽略)。
|
||||||
async fn cone_search_raw(
|
async fn cone_search_raw(
|
||||||
&self,
|
&self,
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
@ -117,6 +146,7 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
|||||||
radius_deg: f64,
|
radius_deg: f64,
|
||||||
release: Option<&str>,
|
release: Option<&str>,
|
||||||
subtype: Option<&str>,
|
subtype: Option<&str>,
|
||||||
|
version: Option<&str>,
|
||||||
) -> Result<Vec<Candidate>>;
|
) -> Result<Vec<Candidate>>;
|
||||||
|
|
||||||
/// 标识符模式:验证并归一化为 Candidate(不发起下载)
|
/// 标识符模式:验证并归一化为 Candidate(不发起下载)
|
||||||
@ -125,6 +155,7 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
|||||||
identifier: &str,
|
identifier: &str,
|
||||||
release: Option<&str>,
|
release: Option<&str>,
|
||||||
subtype: Option<&str>,
|
subtype: Option<&str>,
|
||||||
|
version: Option<&str>,
|
||||||
) -> Result<Candidate>;
|
) -> Result<Candidate>;
|
||||||
|
|
||||||
// ── 下载(各源实现) ──
|
// ── 下载(各源实现) ──
|
||||||
@ -136,6 +167,7 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
|||||||
candidate: &Candidate,
|
candidate: &Candidate,
|
||||||
release: Option<&str>,
|
release: Option<&str>,
|
||||||
subtype: Option<&str>,
|
subtype: Option<&str>,
|
||||||
|
version: Option<&str>,
|
||||||
force: bool,
|
force: bool,
|
||||||
) -> Result<ObservationProduct>;
|
) -> Result<ObservationProduct>;
|
||||||
|
|
||||||
@ -159,8 +191,9 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
|||||||
radius_deg: f64,
|
radius_deg: f64,
|
||||||
release: Option<&str>,
|
release: Option<&str>,
|
||||||
subtype: Option<&str>,
|
subtype: Option<&str>,
|
||||||
|
version: Option<&str>,
|
||||||
) -> Result<Vec<Candidate>> {
|
) -> Result<Vec<Candidate>> {
|
||||||
let query_hash = self.cone_cache_hash(ra, dec, radius_deg, release, subtype);
|
let query_hash = self.cone_cache_hash(ra, dec, radius_deg, release, subtype, version);
|
||||||
|
|
||||||
if let Some(cached) = fetch_cone_cache(&state.db, &query_hash).await? {
|
if let Some(cached) = fetch_cone_cache(&state.db, &query_hash).await? {
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
@ -174,10 +207,12 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
|||||||
|
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||||
let result = self
|
let result = self
|
||||||
.cone_search_raw(state, ra, dec, radius_deg, release, subtype)
|
.cone_search_raw(state, ra, dec, radius_deg, release, subtype, version)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
if let Err(e) = write_cone_cache(&state.db, &query_hash, &self.cone_cache_prefix(), &result).await {
|
if let Err(e) =
|
||||||
|
write_cone_cache(&state.db, &query_hash, &self.cone_cache_prefix(), &result).await
|
||||||
|
{
|
||||||
error!(
|
error!(
|
||||||
"[Observation] 写入 cone 缓存失败 ({:?}/{:?}): {}",
|
"[Observation] 写入 cone 缓存失败 ({:?}/{:?}): {}",
|
||||||
self.key().0,
|
self.key().0,
|
||||||
@ -188,7 +223,7 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
|||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 计算 cone 缓存的 sha1 hash(prefix + 坐标 + release + subtype)
|
/// 计算 cone 缓存的 sha1 hash(prefix + 坐标 + release + subtype + version)
|
||||||
fn cone_cache_hash(
|
fn cone_cache_hash(
|
||||||
&self,
|
&self,
|
||||||
ra: f64,
|
ra: f64,
|
||||||
@ -196,18 +231,42 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
|
|||||||
radius_deg: f64,
|
radius_deg: f64,
|
||||||
release: Option<&str>,
|
release: Option<&str>,
|
||||||
subtype: Option<&str>,
|
subtype: Option<&str>,
|
||||||
|
version: Option<&str>,
|
||||||
) -> String {
|
) -> String {
|
||||||
|
// 各字段加长度前缀 + 字段标记字节,避免不同组合拼接后产生相同字节流。
|
||||||
|
// 例如:release="dr1"+subtype=None 与 release="dr"+subtype=Some("1")
|
||||||
|
// 不加界定会哈希到相同字节序列,导致缓存错误命中。
|
||||||
let mut h = Sha1::new();
|
let mut h = Sha1::new();
|
||||||
h.update(self.cone_cache_prefix().as_bytes());
|
let prefix = self.cone_cache_prefix();
|
||||||
|
h.update((prefix.len() as u64).to_le_bytes());
|
||||||
|
h.update(prefix.as_bytes());
|
||||||
h.update(ra.to_le_bytes());
|
h.update(ra.to_le_bytes());
|
||||||
h.update(dec.to_le_bytes());
|
h.update(dec.to_le_bytes());
|
||||||
h.update(radius_deg.to_le_bytes());
|
h.update(radius_deg.to_le_bytes());
|
||||||
if let Some(r) = release {
|
match release {
|
||||||
|
Some(r) => {
|
||||||
|
h.update([1u8]); // presence marker
|
||||||
|
h.update((r.len() as u64).to_le_bytes());
|
||||||
h.update(r.as_bytes());
|
h.update(r.as_bytes());
|
||||||
}
|
}
|
||||||
if let Some(st) = subtype {
|
None => h.update([0u8]),
|
||||||
|
}
|
||||||
|
match subtype {
|
||||||
|
Some(st) => {
|
||||||
|
h.update([1u8]); // presence marker
|
||||||
|
h.update((st.len() as u64).to_le_bytes());
|
||||||
h.update(st.as_bytes());
|
h.update(st.as_bytes());
|
||||||
}
|
}
|
||||||
|
None => h.update([0u8]),
|
||||||
|
}
|
||||||
|
match version {
|
||||||
|
Some(v) => {
|
||||||
|
h.update([1u8]); // presence marker
|
||||||
|
h.update((v.len() as u64).to_le_bytes());
|
||||||
|
h.update(v.as_bytes());
|
||||||
|
}
|
||||||
|
None => h.update([0u8]),
|
||||||
|
}
|
||||||
format!("{:x}", h.finalize())
|
format!("{:x}", h.finalize())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -224,10 +283,7 @@ struct ConeCacheRow {
|
|||||||
expires_at: Option<chrono::DateTime<chrono::Utc>>,
|
expires_at: Option<chrono::DateTime<chrono::Utc>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn fetch_cone_cache(
|
async fn fetch_cone_cache(pool: &SqlitePool, query_hash: &str) -> Result<Option<Vec<Candidate>>> {
|
||||||
pool: &SqlitePool,
|
|
||||||
query_hash: &str,
|
|
||||||
) -> Result<Option<Vec<Candidate>>> {
|
|
||||||
let row = sqlx::query_as::<_, ConeCacheRow>(
|
let row = sqlx::query_as::<_, ConeCacheRow>(
|
||||||
"SELECT result_json, expires_at FROM vizier_query_cache WHERE query_hash = ? LIMIT 1",
|
"SELECT result_json, expires_at FROM vizier_query_cache WHERE query_hash = ? LIMIT 1",
|
||||||
)
|
)
|
||||||
@ -335,12 +391,22 @@ mod tests {
|
|||||||
fn test_pick_nearest_without_distance_uses_coords() {
|
fn test_pick_nearest_without_distance_uses_coords() {
|
||||||
let rows = vec![
|
let rows = vec![
|
||||||
Candidate {
|
Candidate {
|
||||||
source: Source::Lamost, source_id: String::new(), label: String::new(),
|
source: Source::Lamost,
|
||||||
ra: Some(10.5), dec: Some(41.0), distance: None, raw: None,
|
source_id: String::new(),
|
||||||
|
label: String::new(),
|
||||||
|
ra: Some(10.5),
|
||||||
|
dec: Some(41.0),
|
||||||
|
distance: None,
|
||||||
|
raw: None,
|
||||||
},
|
},
|
||||||
Candidate {
|
Candidate {
|
||||||
source: Source::Lamost, source_id: String::new(), label: String::new(),
|
source: Source::Lamost,
|
||||||
ra: Some(10.51), dec: Some(41.01), distance: None, raw: None,
|
source_id: String::new(),
|
||||||
|
label: String::new(),
|
||||||
|
ra: Some(10.51),
|
||||||
|
dec: Some(41.01),
|
||||||
|
distance: None,
|
||||||
|
raw: None,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
assert_eq!(pick_nearest(&rows, 10.5, 41.0), 0);
|
assert_eq!(pick_nearest(&rows, 10.5, 41.0), 0);
|
||||||
|
|||||||
@ -28,21 +28,34 @@ use tracing::{info, warn};
|
|||||||
// ── 版本与解析辅助 ──
|
// ── 版本与解析辅助 ──
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||||
pub enum GaiaRelease { #[default] Dr3 }
|
pub enum GaiaRelease {
|
||||||
|
#[default]
|
||||||
|
Dr3,
|
||||||
|
}
|
||||||
|
|
||||||
impl GaiaRelease {
|
impl GaiaRelease {
|
||||||
pub fn valid_values() -> &'static [&'static str] {
|
pub fn valid_values() -> &'static [&'static str] {
|
||||||
&["dr3"]
|
&["dr3"]
|
||||||
}
|
}
|
||||||
pub fn dr_str(&self) -> &'static str { "dr3" }
|
pub fn dr_str(&self) -> &'static str {
|
||||||
pub fn tap_table(&self) -> &'static str { "gaiadr3.gaia_source" }
|
"dr3"
|
||||||
pub fn display(&self) -> &'static str { "DR3" }
|
}
|
||||||
|
pub fn tap_table(&self) -> &'static str {
|
||||||
|
"gaiadr3.gaia_source"
|
||||||
|
}
|
||||||
|
pub fn display(&self) -> &'static str {
|
||||||
|
"DR3"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn parse_gaia_release(s: Option<&str>) -> Result<GaiaRelease> {
|
pub fn parse_gaia_release(s: Option<&str>) -> Result<GaiaRelease> {
|
||||||
match s {
|
match s {
|
||||||
None | Some("dr3") => Ok(GaiaRelease::Dr3),
|
None | Some("dr3") => Ok(GaiaRelease::Dr3),
|
||||||
Some(other) => Err(anyhow!("不支持的 Gaia release '{}',可选: {:?}", other, GaiaRelease::valid_values())),
|
Some(other) => Err(anyhow!(
|
||||||
|
"不支持的 Gaia release '{}',可选: {:?}",
|
||||||
|
other,
|
||||||
|
GaiaRelease::valid_values()
|
||||||
|
)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -53,7 +66,11 @@ pub fn parse_gaia_retrieval_type_str(s: Option<&str>) -> Result<GaiaRetrievalTyp
|
|||||||
Some("XP_SAMPLED") => Ok(GaiaRetrievalType::XpSampled),
|
Some("XP_SAMPLED") => Ok(GaiaRetrievalType::XpSampled),
|
||||||
Some("EPOCH_PHOTOMETRY") => Ok(GaiaRetrievalType::EpochPhotometry),
|
Some("EPOCH_PHOTOMETRY") => Ok(GaiaRetrievalType::EpochPhotometry),
|
||||||
Some("RVS") => Ok(GaiaRetrievalType::Rvs),
|
Some("RVS") => Ok(GaiaRetrievalType::Rvs),
|
||||||
Some(other) => Err(anyhow!("不支持的 Gaia subtype '{}',可选: {:?}", other, GaiaRetrievalType::valid_values())),
|
Some(other) => Err(anyhow!(
|
||||||
|
"不支持的 Gaia subtype '{}',可选: {:?}",
|
||||||
|
other,
|
||||||
|
GaiaRetrievalType::valid_values()
|
||||||
|
)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -64,7 +81,13 @@ pub fn parse_gaia_retrieval_type(s: &str) -> Result<GaiaRetrievalType> {
|
|||||||
"XP_SAMPLED" => XpSampled,
|
"XP_SAMPLED" => XpSampled,
|
||||||
"EPOCH_PHOTOMETRY" => EpochPhotometry,
|
"EPOCH_PHOTOMETRY" => EpochPhotometry,
|
||||||
"RVS" => Rvs,
|
"RVS" => Rvs,
|
||||||
other => return Err(anyhow!("不支持的 Gaia retrieval_type '{}',可选: {:?}", other, GaiaRetrievalType::valid_values())),
|
other => {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"不支持的 Gaia retrieval_type '{}',可选: {:?}",
|
||||||
|
other,
|
||||||
|
GaiaRetrievalType::valid_values()
|
||||||
|
))
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -120,29 +143,52 @@ async fn download_gaia_product(
|
|||||||
|
|
||||||
// 1) 缓存命中
|
// 1) 缓存命中
|
||||||
if !force {
|
if !force {
|
||||||
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Gaia, &product, &cache_key).await? {
|
if let Some((artifacts, meta)) =
|
||||||
if cached_files_total_size(&state.config.library_dir, &artifacts).is_some() {
|
fetch_observation_cache(&state.db, Source::Gaia, &product, &cache_key).await?
|
||||||
info!("[Gaia] 缓存命中 (source_id={}, type={})", source_id, rt_param);
|
{
|
||||||
let artifacts = artifacts.into_iter().map(|mut a| {
|
if cached_files_total_size(&state.config.library_dir, &artifacts)
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
|
info!(
|
||||||
|
"[Gaia] 缓存命中 (source_id={}, type={})",
|
||||||
|
source_id, rt_param
|
||||||
|
);
|
||||||
|
let artifacts = artifacts
|
||||||
|
.into_iter()
|
||||||
|
.map(|mut a| {
|
||||||
if a.size_bytes == 0 {
|
if a.size_bytes == 0 {
|
||||||
if let Ok(m) = std::fs::metadata(state.config.library_dir.join(&a.file_path)) {
|
if let Ok(m) =
|
||||||
|
std::fs::metadata(state.config.library_dir.join(&a.file_path))
|
||||||
|
{
|
||||||
a.size_bytes = m.len() as usize;
|
a.size_bytes = m.len() as usize;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
a
|
a
|
||||||
}).collect();
|
})
|
||||||
|
.collect();
|
||||||
return Ok(ObservationProduct {
|
return Ok(ObservationProduct {
|
||||||
source: Source::Gaia, product,
|
source: Source::Gaia,
|
||||||
source_id: cache_key, source_label, artifacts, source_meta: meta,
|
product,
|
||||||
|
source_id: cache_key,
|
||||||
|
source_label,
|
||||||
|
artifacts,
|
||||||
|
source_meta: meta,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
warn!("[Gaia] 缓存记录存在但文件缺失,重新下载 (source_id={})", source_id);
|
warn!(
|
||||||
|
"[Gaia] 缓存记录存在但文件缺失,重新下载 (source_id={})",
|
||||||
|
source_id
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2) DataLink 下载 ZIP + 解包取**全部**文件(修复 EPOCH_PHOTOMETRY 多文件丢失)
|
// 2) DataLink 下载 ZIP + 解包取**全部**文件(修复 EPOCH_PHOTOMETRY 多文件丢失)
|
||||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
let zip_bytes = state.gaia.download_products(&[source_id.to_string()], retrieval_type, "fits").await?;
|
let zip_bytes = state
|
||||||
|
.gaia
|
||||||
|
.download_products(&[source_id.to_string()], retrieval_type, "fits")
|
||||||
|
.await?;
|
||||||
let extracted = extract_all_zip_entries(&zip_bytes, source_id)?;
|
let extracted = extract_all_zip_entries(&zip_bytes, source_id)?;
|
||||||
|
|
||||||
// 3) 落盘
|
// 3) 落盘
|
||||||
@ -152,13 +198,21 @@ async fn download_gaia_product(
|
|||||||
for ext_file in extracted {
|
for ext_file in extracted {
|
||||||
let rel_path = if product_type == ProductType::LightCurve {
|
let rel_path = if product_type == ProductType::LightCurve {
|
||||||
let safe_name = ext_file.name.replace('/', "_");
|
let safe_name = ext_file.name.replace('/', "_");
|
||||||
format!("Telescope/gaia/lightcurve/{rt}/{sid}/{name}",
|
format!(
|
||||||
rt = rt_lower, sid = source_id, name = safe_name)
|
"Telescope/gaia/lightcurve/{rt}/{sid}/{name}",
|
||||||
|
rt = rt_lower,
|
||||||
|
sid = source_id,
|
||||||
|
name = safe_name
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
format!("Telescope/gaia/spectrum/{rt}/{sid}.{ext}",
|
format!(
|
||||||
rt = rt_lower, sid = source_id, ext = ext_file.ext)
|
"Telescope/gaia/spectrum/{rt}/{sid}.{ext}",
|
||||||
|
rt = rt_lower,
|
||||||
|
sid = source_id,
|
||||||
|
ext = ext_file.ext
|
||||||
|
)
|
||||||
};
|
};
|
||||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &ext_file.bytes)?;
|
let size = persist_bytes(&state.config.library_dir, &rel_path, &ext_file.bytes).await?;
|
||||||
info!("[Gaia] 已保存 {} ({}B) → {}", ext_file.name, size, rel_path);
|
info!("[Gaia] 已保存 {} ({}B) → {}", ext_file.name, size, rel_path);
|
||||||
artifacts.push(Artifact {
|
artifacts.push(Artifact {
|
||||||
band: parse_band_from_name(&ext_file.name),
|
band: parse_band_from_name(&ext_file.name),
|
||||||
@ -174,22 +228,37 @@ async fn download_gaia_product(
|
|||||||
a.file_url = file_url_from_path(&a.file_path);
|
a.file_url = file_url_from_path(&a.file_path);
|
||||||
}
|
}
|
||||||
if artifacts.is_empty() {
|
if artifacts.is_empty() {
|
||||||
return Err(anyhow!("Gaia DataLink ZIP 内无可下载文件条目 (source_id={}, type={})", source_id, rt_param));
|
return Err(anyhow!(
|
||||||
|
"Gaia DataLink ZIP 内无可下载文件条目 (source_id={}, type={})",
|
||||||
|
source_id,
|
||||||
|
rt_param
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
let meta = serde_json::json!({"source_id": source_id, "retrieval_type": rt_param, "dr": dr});
|
let meta = serde_json::json!({"source_id": source_id, "retrieval_type": rt_param, "dr": dr});
|
||||||
let meta_str = meta.to_string();
|
let meta_str = meta.to_string();
|
||||||
if let Err(e) = write_observation_cache(
|
if let Err(e) = write_observation_cache(
|
||||||
&state.db, Source::Gaia, &product, &cache_key, None, None,
|
&state.db,
|
||||||
&artifacts, Some(&meta_str),
|
Source::Gaia,
|
||||||
).await {
|
&product,
|
||||||
|
&cache_key,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
&artifacts,
|
||||||
|
Some(&meta_str),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
log_write_failure(Source::Gaia.as_str(), product.product.as_str(), e);
|
log_write_failure(Source::Gaia.as_str(), product.product.as_str(), e);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(ObservationProduct {
|
Ok(ObservationProduct {
|
||||||
source: Source::Gaia, product,
|
source: Source::Gaia,
|
||||||
source_id: cache_key, source_label,
|
product,
|
||||||
artifacts, source_meta: Some(meta),
|
source_id: cache_key,
|
||||||
|
source_label,
|
||||||
|
artifacts,
|
||||||
|
source_meta: Some(meta),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -208,30 +277,48 @@ struct ExtractedFile {
|
|||||||
/// 旧实现 `extract_first_zip_entry` 只取首个文件,丢失了 BP/RP 波段——本函数修复之。
|
/// 旧实现 `extract_first_zip_entry` 只取首个文件,丢失了 BP/RP 波段——本函数修复之。
|
||||||
fn extract_all_zip_entries(zip_bytes: &[u8], source_id: &str) -> Result<Vec<ExtractedFile>> {
|
fn extract_all_zip_entries(zip_bytes: &[u8], source_id: &str) -> Result<Vec<ExtractedFile>> {
|
||||||
let cursor = Cursor::new(zip_bytes);
|
let cursor = Cursor::new(zip_bytes);
|
||||||
let mut archive = zip::ZipArchive::new(cursor).map_err(|e| anyhow!("Gaia ZIP 解析失败: {}", e))?;
|
let mut archive =
|
||||||
|
zip::ZipArchive::new(cursor).map_err(|e| anyhow!("Gaia ZIP 解析失败: {}", e))?;
|
||||||
|
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
for i in 0..archive.len() {
|
for i in 0..archive.len() {
|
||||||
let mut file = archive.by_index(i)
|
let mut file = archive
|
||||||
|
.by_index(i)
|
||||||
.map_err(|e| anyhow!("读取 Gaia ZIP 条目 {} 失败: {}", i, e))?;
|
.map_err(|e| anyhow!("读取 Gaia ZIP 条目 {} 失败: {}", i, e))?;
|
||||||
if file.is_dir() { continue; }
|
if file.is_dir() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let name = file.name().to_string();
|
let name = file.name().to_string();
|
||||||
let mut buf = Vec::new();
|
let mut buf = Vec::new();
|
||||||
file.read_to_end(&mut buf)
|
file.read_to_end(&mut buf)
|
||||||
.map_err(|e| anyhow!("解压 Gaia ZIP 条目失败: {}", e))?;
|
.map_err(|e| anyhow!("解压 Gaia ZIP 条目失败: {}", e))?;
|
||||||
if buf.is_empty() { continue; }
|
if buf.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
let lower = name.to_lowercase();
|
let lower = name.to_lowercase();
|
||||||
let ext = if lower.ends_with(".fits") { "fits" }
|
let ext = if lower.ends_with(".fits") {
|
||||||
else if lower.ends_with(".vot") || lower.ends_with(".xml") { "vot" }
|
"fits"
|
||||||
else if lower.ends_with(".csv") { "csv" }
|
} else if lower.ends_with(".vot") || lower.ends_with(".xml") {
|
||||||
else { "fits" };
|
"vot"
|
||||||
|
} else if lower.ends_with(".csv") {
|
||||||
|
"csv"
|
||||||
|
} else {
|
||||||
|
"fits"
|
||||||
|
};
|
||||||
|
|
||||||
out.push(ExtractedFile { name, ext, bytes: buf });
|
out.push(ExtractedFile {
|
||||||
|
name,
|
||||||
|
ext,
|
||||||
|
bytes: buf,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if out.is_empty() {
|
if out.is_empty() {
|
||||||
return Err(anyhow!("Gaia ZIP 内无可下载文件条目 (source_id={})", source_id));
|
return Err(anyhow!(
|
||||||
|
"Gaia ZIP 内无可下载文件条目 (source_id={})",
|
||||||
|
source_id
|
||||||
|
));
|
||||||
}
|
}
|
||||||
Ok(out)
|
Ok(out)
|
||||||
}
|
}
|
||||||
@ -284,12 +371,19 @@ impl ObservationFetcher for GaiaSpectrumFetcher {
|
|||||||
async fn cone_search_raw(
|
async fn cone_search_raw(
|
||||||
&self,
|
&self,
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
ra: f64, dec: f64, radius_deg: f64,
|
ra: f64,
|
||||||
_release: Option<&str>, _subtype: Option<&str>,
|
dec: f64,
|
||||||
|
radius_deg: f64,
|
||||||
|
_release: Option<&str>,
|
||||||
|
_subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
) -> Result<Vec<Candidate>> {
|
) -> Result<Vec<Candidate>> {
|
||||||
validate_cone_params(ra, dec, radius_deg)?;
|
validate_cone_params(ra, dec, radius_deg)?;
|
||||||
let rel = parse_gaia_release(_release)?;
|
let rel = parse_gaia_release(_release)?;
|
||||||
let result = state.gaia.cone_search(ra, dec, radius_deg, 50, true, rel).await?;
|
let result = state
|
||||||
|
.gaia
|
||||||
|
.cone_search(ra, dec, radius_deg, 50, true, rel)
|
||||||
|
.await?;
|
||||||
Ok(result.rows.iter().map(row_to_candidate).collect())
|
Ok(result.rows.iter().map(row_to_candidate).collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -298,6 +392,7 @@ impl ObservationFetcher for GaiaSpectrumFetcher {
|
|||||||
identifier: &str,
|
identifier: &str,
|
||||||
_release: Option<&str>,
|
_release: Option<&str>,
|
||||||
subtype: Option<&str>,
|
subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
) -> Result<Candidate> {
|
) -> Result<Candidate> {
|
||||||
let (rt_param, gaia_sid) = match identifier.split_once('|') {
|
let (rt_param, gaia_sid) = match identifier.split_once('|') {
|
||||||
Some((rt, s)) => (rt, s),
|
Some((rt, s)) => (rt, s),
|
||||||
@ -315,7 +410,9 @@ impl ObservationFetcher for GaiaSpectrumFetcher {
|
|||||||
source: Source::Gaia,
|
source: Source::Gaia,
|
||||||
source_id: gaia_sid.to_string(),
|
source_id: gaia_sid.to_string(),
|
||||||
label: format!("Gaia {}", gaia_sid),
|
label: format!("Gaia {}", gaia_sid),
|
||||||
ra: None, dec: None, distance: None,
|
ra: None,
|
||||||
|
dec: None,
|
||||||
|
distance: None,
|
||||||
raw: Some(serde_json::json!({"source_id": gaia_sid, "retrieval_type": rt.as_param()})),
|
raw: Some(serde_json::json!({"source_id": gaia_sid, "retrieval_type": rt.as_param()})),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -326,13 +423,18 @@ impl ObservationFetcher for GaiaSpectrumFetcher {
|
|||||||
candidate: &Candidate,
|
candidate: &Candidate,
|
||||||
_release: Option<&str>,
|
_release: Option<&str>,
|
||||||
subtype: Option<&str>,
|
subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
force: bool,
|
force: bool,
|
||||||
) -> Result<ObservationProduct> {
|
) -> Result<ObservationProduct> {
|
||||||
let source_id = candidate.raw.as_ref()
|
let source_id = candidate
|
||||||
|
.raw
|
||||||
|
.as_ref()
|
||||||
.and_then(|v| v.get("source_id"))
|
.and_then(|v| v.get("source_id"))
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.unwrap_or(&candidate.source_id);
|
.unwrap_or(&candidate.source_id);
|
||||||
let rt = if let Some(rt_str) = candidate.raw.as_ref()
|
let rt = if let Some(rt_str) = candidate
|
||||||
|
.raw
|
||||||
|
.as_ref()
|
||||||
.and_then(|v| v.get("retrieval_type"))
|
.and_then(|v| v.get("retrieval_type"))
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
{
|
{
|
||||||
@ -381,13 +483,20 @@ impl ObservationFetcher for GaiaLightCurveFetcher {
|
|||||||
async fn cone_search_raw(
|
async fn cone_search_raw(
|
||||||
&self,
|
&self,
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
ra: f64, dec: f64, radius_deg: f64,
|
ra: f64,
|
||||||
_release: Option<&str>, _subtype: Option<&str>,
|
dec: f64,
|
||||||
|
radius_deg: f64,
|
||||||
|
_release: Option<&str>,
|
||||||
|
_subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
) -> Result<Vec<Candidate>> {
|
) -> Result<Vec<Candidate>> {
|
||||||
// 光变 cone 复用 Gaia 主表检索(不强制 xp_only,因为光变对所有源都有)
|
// 光变 cone 复用 Gaia 主表检索(不强制 xp_only,因为光变对所有源都有)
|
||||||
validate_cone_params(ra, dec, radius_deg)?;
|
validate_cone_params(ra, dec, radius_deg)?;
|
||||||
let rel = parse_gaia_release(_release)?;
|
let rel = parse_gaia_release(_release)?;
|
||||||
let result = state.gaia.cone_search(ra, dec, radius_deg, 50, false, rel).await?;
|
let result = state
|
||||||
|
.gaia
|
||||||
|
.cone_search(ra, dec, radius_deg, 50, false, rel)
|
||||||
|
.await?;
|
||||||
Ok(result.rows.iter().map(row_to_candidate).collect())
|
Ok(result.rows.iter().map(row_to_candidate).collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -396,6 +505,7 @@ impl ObservationFetcher for GaiaLightCurveFetcher {
|
|||||||
identifier: &str,
|
identifier: &str,
|
||||||
_release: Option<&str>,
|
_release: Option<&str>,
|
||||||
_subtype: Option<&str>,
|
_subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
) -> Result<Candidate> {
|
) -> Result<Candidate> {
|
||||||
let sid = identifier.trim();
|
let sid = identifier.trim();
|
||||||
if sid.is_empty() {
|
if sid.is_empty() {
|
||||||
@ -405,7 +515,9 @@ impl ObservationFetcher for GaiaLightCurveFetcher {
|
|||||||
source: Source::Gaia,
|
source: Source::Gaia,
|
||||||
source_id: sid.to_string(),
|
source_id: sid.to_string(),
|
||||||
label: format!("Gaia {}", sid),
|
label: format!("Gaia {}", sid),
|
||||||
ra: None, dec: None, distance: None,
|
ra: None,
|
||||||
|
dec: None,
|
||||||
|
distance: None,
|
||||||
raw: Some(serde_json::json!({"source_id": sid, "retrieval_type": "EPOCH_PHOTOMETRY"})),
|
raw: Some(serde_json::json!({"source_id": sid, "retrieval_type": "EPOCH_PHOTOMETRY"})),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -416,13 +528,23 @@ impl ObservationFetcher for GaiaLightCurveFetcher {
|
|||||||
candidate: &Candidate,
|
candidate: &Candidate,
|
||||||
_release: Option<&str>,
|
_release: Option<&str>,
|
||||||
_subtype: Option<&str>,
|
_subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
force: bool,
|
force: bool,
|
||||||
) -> Result<ObservationProduct> {
|
) -> Result<ObservationProduct> {
|
||||||
let source_id = candidate.raw.as_ref()
|
let source_id = candidate
|
||||||
|
.raw
|
||||||
|
.as_ref()
|
||||||
.and_then(|v| v.get("source_id"))
|
.and_then(|v| v.get("source_id"))
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.unwrap_or(&candidate.source_id);
|
.unwrap_or(&candidate.source_id);
|
||||||
download_gaia_product(state, source_id, GaiaRetrievalType::EpochPhotometry, ProductType::LightCurve, force).await
|
download_gaia_product(
|
||||||
|
state,
|
||||||
|
source_id,
|
||||||
|
GaiaRetrievalType::EpochPhotometry,
|
||||||
|
ProductType::LightCurve,
|
||||||
|
force,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -442,14 +564,26 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_parse_band_from_name_epoch() {
|
fn test_parse_band_from_name_epoch() {
|
||||||
assert_eq!(parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_G.fits").as_deref(), Some("G"));
|
assert_eq!(
|
||||||
assert_eq!(parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_BP.fits").as_deref(), Some("BP"));
|
parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_G.fits").as_deref(),
|
||||||
assert_eq!(parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_RP.fits").as_deref(), Some("RP"));
|
Some("G")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_BP.fits").as_deref(),
|
||||||
|
Some("BP")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_RP.fits").as_deref(),
|
||||||
|
Some("RP")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_parse_band_from_name_xp() {
|
fn test_parse_band_from_name_xp() {
|
||||||
assert_eq!(parse_band_from_name("65214031805717376_XP_CONTINUOUS.fits"), None);
|
assert_eq!(
|
||||||
|
parse_band_from_name("65214031805717376_XP_CONTINUOUS.fits"),
|
||||||
|
None
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -474,17 +608,26 @@ mod tests {
|
|||||||
{
|
{
|
||||||
let mut writer = zip::ZipWriter::new(std::io::Cursor::new(&mut buf));
|
let mut writer = zip::ZipWriter::new(std::io::Cursor::new(&mut buf));
|
||||||
let opts = zip::write::SimpleFileOptions::default();
|
let opts = zip::write::SimpleFileOptions::default();
|
||||||
writer.start_file("123_EPOCH_PHOTOMETRY_G.fits", opts).unwrap();
|
writer
|
||||||
|
.start_file("123_EPOCH_PHOTOMETRY_G.fits", opts)
|
||||||
|
.unwrap();
|
||||||
writer.write_all(b"G_DATA").unwrap();
|
writer.write_all(b"G_DATA").unwrap();
|
||||||
writer.start_file("123_EPOCH_PHOTOMETRY_BP.fits", opts).unwrap();
|
writer
|
||||||
|
.start_file("123_EPOCH_PHOTOMETRY_BP.fits", opts)
|
||||||
|
.unwrap();
|
||||||
writer.write_all(b"BP_DATA").unwrap();
|
writer.write_all(b"BP_DATA").unwrap();
|
||||||
writer.start_file("123_EPOCH_PHOTOMETRY_RP.fits", opts).unwrap();
|
writer
|
||||||
|
.start_file("123_EPOCH_PHOTOMETRY_RP.fits", opts)
|
||||||
|
.unwrap();
|
||||||
writer.write_all(b"RP_DATA").unwrap();
|
writer.write_all(b"RP_DATA").unwrap();
|
||||||
writer.finish().unwrap();
|
writer.finish().unwrap();
|
||||||
}
|
}
|
||||||
let files = extract_all_zip_entries(&buf, "123").unwrap();
|
let files = extract_all_zip_entries(&buf, "123").unwrap();
|
||||||
assert_eq!(files.len(), 3, "EPOCH_PHOTOMETRY 应返回全部 3 个波段文件");
|
assert_eq!(files.len(), 3, "EPOCH_PHOTOMETRY 应返回全部 3 个波段文件");
|
||||||
let bands: Vec<_> = files.iter().map(|f| parse_band_from_name(&f.name)).collect();
|
let bands: Vec<_> = files
|
||||||
|
.iter()
|
||||||
|
.map(|f| parse_band_from_name(&f.name))
|
||||||
|
.collect();
|
||||||
assert!(bands.contains(&Some("G".into())));
|
assert!(bands.contains(&Some("G".into())));
|
||||||
assert!(bands.contains(&Some("BP".into())));
|
assert!(bands.contains(&Some("BP".into())));
|
||||||
assert!(bands.contains(&Some("RP".into())));
|
assert!(bands.contains(&Some("RP".into())));
|
||||||
|
|||||||
@ -23,55 +23,116 @@ use tracing::{info, warn};
|
|||||||
|
|
||||||
// ── 版本与分辨率枚举(领域知识) ──
|
// ── 版本与分辨率枚举(领域知识) ──
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||||
pub enum LamostRelease {
|
pub enum LamostRelease {
|
||||||
Dr5, Dr6, Dr7, Dr8, Dr9, Dr10, Dr11,
|
Dr5,
|
||||||
}
|
Dr6,
|
||||||
|
Dr7,
|
||||||
impl Default for LamostRelease {
|
Dr8,
|
||||||
fn default() -> Self { Self::Dr10 }
|
Dr9,
|
||||||
|
#[default]
|
||||||
|
Dr10,
|
||||||
|
Dr11,
|
||||||
|
Dr12,
|
||||||
|
Dr13,
|
||||||
|
Dr14,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LamostRelease {
|
impl LamostRelease {
|
||||||
pub fn valid_values() -> &'static [&'static str] {
|
pub fn valid_values() -> &'static [&'static str] {
|
||||||
&["dr5", "dr6", "dr7", "dr8", "dr9", "dr10", "dr11"]
|
&[
|
||||||
|
"dr5", "dr6", "dr7", "dr8", "dr9", "dr10", "dr11", "dr12", "dr13", "dr14",
|
||||||
|
]
|
||||||
}
|
}
|
||||||
pub fn path_segment(&self) -> &'static str {
|
pub fn path_segment(&self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Self::Dr5 => "dr5", Self::Dr6 => "dr6", Self::Dr7 => "dr7",
|
Self::Dr5 => "dr5",
|
||||||
Self::Dr8 => "dr8", Self::Dr9 => "dr9", Self::Dr10 => "dr10", Self::Dr11 => "dr11",
|
Self::Dr6 => "dr6",
|
||||||
|
Self::Dr7 => "dr7",
|
||||||
|
Self::Dr8 => "dr8",
|
||||||
|
Self::Dr9 => "dr9",
|
||||||
|
Self::Dr10 => "dr10",
|
||||||
|
Self::Dr11 => "dr11",
|
||||||
|
Self::Dr12 => "dr12",
|
||||||
|
Self::Dr13 => "dr13",
|
||||||
|
Self::Dr14 => "dr14",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub fn version_segment(&self) -> &'static str {
|
/// 该 DR 支持的全部子版本(URL 路径段),按时间从早到晚排序
|
||||||
|
///
|
||||||
|
/// 数据来源:LAMOST 官网 lmusers 页面的发布版本表格 + 端点实测。
|
||||||
|
/// v0 通常是早期/内部预览版(数据量少或为空),v2.0 为最终公开版。
|
||||||
|
pub fn versions(&self) -> &'static [&'static str] {
|
||||||
|
match self {
|
||||||
|
Self::Dr5 | Self::Dr6 => &["v0", "v2"],
|
||||||
|
Self::Dr7 => &["v0", "v1.2"],
|
||||||
|
Self::Dr8 | Self::Dr9 => &["v0", "v1.0", "v2.0"],
|
||||||
|
Self::Dr10 => &["v0", "v1.0", "v2.0"],
|
||||||
|
Self::Dr11 | Self::Dr12 | Self::Dr13 | Self::Dr14 => &["v0", "v1.0", "v1.1", "v2.0"],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/// 该 DR 的默认(最新公开)子版本
|
||||||
|
pub fn default_version(&self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Self::Dr5 | Self::Dr6 => "v2",
|
Self::Dr5 | Self::Dr6 => "v2",
|
||||||
Self::Dr7 => "v1.2",
|
Self::Dr7 => "v1.2",
|
||||||
Self::Dr8 | Self::Dr9 | Self::Dr10 | Self::Dr11 => "v2.0",
|
// v2.0 为各 DR 的最终/最新公开版
|
||||||
|
Self::Dr8
|
||||||
|
| Self::Dr9
|
||||||
|
| Self::Dr10
|
||||||
|
| Self::Dr11
|
||||||
|
| Self::Dr12
|
||||||
|
| Self::Dr13
|
||||||
|
| Self::Dr14 => "v2.0",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub fn display(&self) -> &'static str {
|
pub fn display(&self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Self::Dr5 => "DR5", Self::Dr6 => "DR6", Self::Dr7 => "DR7",
|
Self::Dr5 => "DR5",
|
||||||
Self::Dr8 => "DR8", Self::Dr9 => "DR9", Self::Dr10 => "DR10", Self::Dr11 => "DR11",
|
Self::Dr6 => "DR6",
|
||||||
|
Self::Dr7 => "DR7",
|
||||||
|
Self::Dr8 => "DR8",
|
||||||
|
Self::Dr9 => "DR9",
|
||||||
|
Self::Dr10 => "DR10",
|
||||||
|
Self::Dr11 => "DR11",
|
||||||
|
Self::Dr12 => "DR12",
|
||||||
|
Self::Dr13 => "DR13",
|
||||||
|
Self::Dr14 => "DR14",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub fn supports_mrs(&self) -> bool {
|
pub fn supports_mrs(&self) -> bool {
|
||||||
!matches!(self, Self::Dr5 | Self::Dr6)
|
!matches!(self, Self::Dr5 | Self::Dr6)
|
||||||
}
|
}
|
||||||
|
/// 是否为公开发布(无需登录即可访问 cone search / FITS 下载)。
|
||||||
|
///
|
||||||
|
/// LAMOST 的 Internal 数据发布需经 oauth.china-vo.org 完成统一身份认证后才能访问。
|
||||||
|
/// DR12-14 当前为 Internal,公开 cone search 端点会被 302 重定向到登录页。
|
||||||
|
pub fn is_public(&self) -> bool {
|
||||||
|
!matches!(self, Self::Dr12 | Self::Dr13 | Self::Dr14)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub enum LamostResolution { Lrs, Mrs }
|
pub enum LamostResolution {
|
||||||
|
Lrs,
|
||||||
|
Mrs,
|
||||||
|
}
|
||||||
|
|
||||||
impl LamostResolution {
|
impl LamostResolution {
|
||||||
pub fn valid_values() -> &'static [&'static str] {
|
pub fn valid_values() -> &'static [&'static str] {
|
||||||
&["lrs", "mrs"]
|
&["lrs", "mrs"]
|
||||||
}
|
}
|
||||||
pub fn as_str(&self) -> &'static str {
|
pub fn as_str(&self) -> &'static str {
|
||||||
match self { Self::Lrs => "lrs", Self::Mrs => "mrs" }
|
match self {
|
||||||
|
Self::Lrs => "lrs",
|
||||||
|
Self::Mrs => "mrs",
|
||||||
|
}
|
||||||
}
|
}
|
||||||
pub fn display(&self) -> &'static str {
|
pub fn display(&self) -> &'static str {
|
||||||
match self { Self::Lrs => "LRS", Self::Mrs => "MRS" }
|
match self {
|
||||||
|
Self::Lrs => "LRS",
|
||||||
|
Self::Mrs => "MRS",
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -85,16 +146,56 @@ pub fn parse_lamost_release(s: Option<&str>) -> Result<LamostRelease> {
|
|||||||
Some("dr9") => LamostRelease::Dr9,
|
Some("dr9") => LamostRelease::Dr9,
|
||||||
Some("dr10") => LamostRelease::Dr10,
|
Some("dr10") => LamostRelease::Dr10,
|
||||||
Some("dr11") => LamostRelease::Dr11,
|
Some("dr11") => LamostRelease::Dr11,
|
||||||
Some(other) => return Err(anyhow!("不支持的 LAMOST release '{}',可选: {:?}", other, LamostRelease::valid_values())),
|
Some("dr12") => LamostRelease::Dr12,
|
||||||
|
Some("dr13") => LamostRelease::Dr13,
|
||||||
|
Some("dr14") => LamostRelease::Dr14,
|
||||||
|
Some(other) => {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"不支持的 LAMOST release '{}',可选: {:?}",
|
||||||
|
other,
|
||||||
|
LamostRelease::valid_values()
|
||||||
|
))
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 解析 LAMOST 子版本参数。None 时返回该 DR 的默认(最新公开)子版本。
|
||||||
|
pub fn parse_lamost_version(release: LamostRelease, version: Option<&str>) -> Result<String> {
|
||||||
|
let valid = release.versions();
|
||||||
|
match version.map(|x| x.trim().to_lowercase()) {
|
||||||
|
None => Ok(release.default_version().to_string()),
|
||||||
|
Some(v) => {
|
||||||
|
if valid.iter().any(|x| x.eq_ignore_ascii_case(&v)) {
|
||||||
|
// 规范化为官网写法(保留原大小写约定,如 v1.0/v2.0)
|
||||||
|
Ok(valid
|
||||||
|
.iter()
|
||||||
|
.find(|x| x.eq_ignore_ascii_case(&v))
|
||||||
|
.unwrap()
|
||||||
|
.to_string())
|
||||||
|
} else {
|
||||||
|
Err(anyhow!(
|
||||||
|
"不支持的 LAMOST 子版本 '{}'({:?} 的可选版本: {:?})",
|
||||||
|
v,
|
||||||
|
release,
|
||||||
|
valid
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn parse_lamost_resolution(subtype: Option<&str>) -> Result<LamostResolution> {
|
pub fn parse_lamost_resolution(subtype: Option<&str>) -> Result<LamostResolution> {
|
||||||
Ok(match subtype.map(|x| x.to_lowercase()).as_deref() {
|
Ok(match subtype.map(|x| x.to_lowercase()).as_deref() {
|
||||||
None => LamostResolution::Lrs,
|
None => LamostResolution::Lrs,
|
||||||
Some("lrs") | Some("low") => LamostResolution::Lrs,
|
Some("lrs") | Some("low") => LamostResolution::Lrs,
|
||||||
Some("mrs") | Some("medium") => LamostResolution::Mrs,
|
Some("mrs") | Some("medium") => LamostResolution::Mrs,
|
||||||
Some(other) => return Err(anyhow!("不支持的 LAMOST subtype '{}',可选: {:?}", other, LamostResolution::valid_values())),
|
Some(other) => {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"不支持的 LAMOST subtype '{}',可选: {:?}",
|
||||||
|
other,
|
||||||
|
LamostResolution::valid_values()
|
||||||
|
))
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -143,6 +244,49 @@ impl ObservationFetcher for LamostSpectrumFetcher {
|
|||||||
Some("dr10")
|
Some("dr10")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn versions_for_release(&self, release: Option<&str>) -> Vec<String> {
|
||||||
|
match parse_lamost_release(release) {
|
||||||
|
Ok(rel) => rel.versions().iter().map(|s| s.to_string()).collect(),
|
||||||
|
Err(_) => Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_version_for_release(&self, release: Option<&str>) -> Option<String> {
|
||||||
|
parse_lamost_release(release)
|
||||||
|
.ok()
|
||||||
|
.map(|rel| rel.default_version().to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Internal 数据发布(DR12-14)需经 oauth.china-vo.org 登录,公开 cone search 不可用
|
||||||
|
fn releases_requiring_auth(&self) -> Vec<String> {
|
||||||
|
LamostRelease::valid_values()
|
||||||
|
.iter()
|
||||||
|
.filter_map(|s| {
|
||||||
|
parse_lamost_release(Some(s))
|
||||||
|
.ok()
|
||||||
|
.filter(|rel| !rel.is_public())
|
||||||
|
.map(|_| s.to_string())
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 表达 release×subtype 交叉约束:DR5/DR6 仅有 LRS(无 MRS),DR7+ 有 LRS+MRS。
|
||||||
|
/// 前端据此在选 DR5/DR6 时隐藏/禁用 MRS 子类型选项。
|
||||||
|
fn subtypes_per_release(&self) -> std::collections::HashMap<String, Vec<String>> {
|
||||||
|
LamostRelease::valid_values()
|
||||||
|
.iter()
|
||||||
|
.map(|s| {
|
||||||
|
let rel = parse_lamost_release(Some(s)).unwrap();
|
||||||
|
let subs: Vec<String> = if rel.supports_mrs() {
|
||||||
|
vec!["lrs".to_string(), "mrs".to_string()]
|
||||||
|
} else {
|
||||||
|
vec!["lrs".to_string()]
|
||||||
|
};
|
||||||
|
(s.to_string(), subs)
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
fn suggested_max_radius_deg(&self) -> f64 {
|
fn suggested_max_radius_deg(&self) -> f64 {
|
||||||
// 与 validate_cone_params 的建议范围一致(≤5° 仅警告不拒绝)
|
// 与 validate_cone_params 的建议范围一致(≤5° 仅警告不拒绝)
|
||||||
5.0
|
5.0
|
||||||
@ -155,31 +299,70 @@ impl ObservationFetcher for LamostSpectrumFetcher {
|
|||||||
async fn cone_search_raw(
|
async fn cone_search_raw(
|
||||||
&self,
|
&self,
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
ra: f64, dec: f64, radius_deg: f64,
|
ra: f64,
|
||||||
release: Option<&str>, subtype: Option<&str>,
|
dec: f64,
|
||||||
|
radius_deg: f64,
|
||||||
|
release: Option<&str>,
|
||||||
|
subtype: Option<&str>,
|
||||||
|
version: Option<&str>,
|
||||||
) -> Result<Vec<Candidate>> {
|
) -> Result<Vec<Candidate>> {
|
||||||
validate_cone_params(ra, dec, radius_deg)?;
|
validate_cone_params(ra, dec, radius_deg)?;
|
||||||
let rel = parse_lamost_release(release)?;
|
let rel = parse_lamost_release(release)?;
|
||||||
let res = parse_lamost_resolution(subtype)?;
|
let res = parse_lamost_resolution(subtype)?;
|
||||||
if res == LamostResolution::Mrs && !rel.supports_mrs() {
|
let ver = parse_lamost_version(rel, version)?;
|
||||||
return Err(anyhow!("MRS(中分辨率)从 DR7 起才支持,{:?} 无 MRS 数据", rel));
|
if !rel.is_public() {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"{:?} 当前为 Internal 数据发布,公开 Cone Search 不可用(需经 oauth.china-vo.org 登录认证)",
|
||||||
|
rel
|
||||||
|
));
|
||||||
}
|
}
|
||||||
let result = state.lamost.cone_search(ra, dec, radius_deg, rel, res).await?;
|
if res == LamostResolution::Mrs && !rel.supports_mrs() {
|
||||||
Ok(result.rows.iter().map(|r| row_to_candidate(r, res.as_str())).collect())
|
return Err(anyhow!(
|
||||||
|
"MRS(中分辨率)从 DR7 起才支持,{:?} 无 MRS 数据",
|
||||||
|
rel
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let result = state
|
||||||
|
.lamost
|
||||||
|
.cone_search(ra, dec, radius_deg, rel, &ver, res)
|
||||||
|
.await?;
|
||||||
|
Ok(result
|
||||||
|
.rows
|
||||||
|
.iter()
|
||||||
|
.map(|r| row_to_candidate(r, res.as_str()))
|
||||||
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn resolve_identifier(
|
async fn resolve_identifier(
|
||||||
&self,
|
&self,
|
||||||
identifier: &str,
|
identifier: &str,
|
||||||
_release: Option<&str>,
|
release: Option<&str>,
|
||||||
subtype: Option<&str>,
|
subtype: Option<&str>,
|
||||||
|
version: Option<&str>,
|
||||||
) -> Result<Candidate> {
|
) -> Result<Candidate> {
|
||||||
let obsid: i64 = identifier.trim().parse()
|
let obsid: i64 = identifier
|
||||||
|
.trim()
|
||||||
|
.parse()
|
||||||
.map_err(|_| anyhow!("LAMOST source_id 应为纯数字 obsid,得到 '{}'", identifier))?;
|
.map_err(|_| anyhow!("LAMOST source_id 应为纯数字 obsid,得到 '{}'", identifier))?;
|
||||||
if obsid <= 0 {
|
if obsid <= 0 {
|
||||||
return Err(anyhow!("obsid 应为正整数,当前: {}", obsid));
|
return Err(anyhow!("obsid 应为正整数,当前: {}", obsid));
|
||||||
}
|
}
|
||||||
let res = parse_lamost_resolution(subtype)?.as_str().to_string();
|
let rel = parse_lamost_release(release)?;
|
||||||
|
let res = parse_lamost_resolution(subtype)?;
|
||||||
|
let ver = parse_lamost_version(rel, version)?;
|
||||||
|
if !rel.is_public() {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"{:?} 当前为 Internal 数据发布,公开访问不可用(需经 oauth.china-vo.org 登录认证)",
|
||||||
|
rel
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// 与 cone_search_raw 一致的交叉约束:MRS 仅从 DR7 起支持
|
||||||
|
if res == LamostResolution::Mrs && !rel.supports_mrs() {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"MRS(中分辨率)从 DR7 起才支持,{:?} 无 MRS 数据",
|
||||||
|
rel
|
||||||
|
));
|
||||||
|
}
|
||||||
Ok(Candidate {
|
Ok(Candidate {
|
||||||
source: Source::Lamost,
|
source: Source::Lamost,
|
||||||
source_id: obsid.to_string(),
|
source_id: obsid.to_string(),
|
||||||
@ -187,7 +370,9 @@ impl ObservationFetcher for LamostSpectrumFetcher {
|
|||||||
ra: None,
|
ra: None,
|
||||||
dec: None,
|
dec: None,
|
||||||
distance: None,
|
distance: None,
|
||||||
raw: Some(serde_json::json!({"obsid": obsid, "resolution": res})),
|
raw: Some(
|
||||||
|
serde_json::json!({"obsid": obsid, "resolution": res.as_str(), "version": ver}),
|
||||||
|
),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -197,68 +382,116 @@ impl ObservationFetcher for LamostSpectrumFetcher {
|
|||||||
candidate: &Candidate,
|
candidate: &Candidate,
|
||||||
release: Option<&str>,
|
release: Option<&str>,
|
||||||
subtype: Option<&str>,
|
subtype: Option<&str>,
|
||||||
|
version: Option<&str>,
|
||||||
force: bool,
|
force: bool,
|
||||||
) -> Result<ObservationProduct> {
|
) -> Result<ObservationProduct> {
|
||||||
let obsid = candidate.raw.as_ref()
|
let obsid = candidate
|
||||||
|
.raw
|
||||||
|
.as_ref()
|
||||||
.and_then(|v| v.get("obsid"))
|
.and_then(|v| v.get("obsid"))
|
||||||
.and_then(|v| v.as_i64())
|
.and_then(|v| v.as_i64())
|
||||||
.ok_or_else(|| anyhow!("LAMOST Candidate 缺 obsid 字段"))?;
|
.ok_or_else(|| anyhow!("LAMOST Candidate 缺 obsid 字段"))?;
|
||||||
let rel = parse_lamost_release(release)?;
|
let rel = parse_lamost_release(release)?;
|
||||||
let res = parse_lamost_resolution(subtype)?;
|
let res = parse_lamost_resolution(subtype)?;
|
||||||
|
let ver = parse_lamost_version(rel, version)?;
|
||||||
|
if !rel.is_public() {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"{:?} 当前为 Internal 数据发布,公开访问不可用(需经 oauth.china-vo.org 登录认证)",
|
||||||
|
rel
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// 防御性交叉约束(resolve_identifier 已查过;直接调 fetch 时再兜一次)
|
||||||
|
if res == LamostResolution::Mrs && !rel.supports_mrs() {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"MRS(中分辨率)从 DR7 起才支持,{:?} 无 MRS 数据",
|
||||||
|
rel
|
||||||
|
));
|
||||||
|
}
|
||||||
let product = ProductSpec::with_subtype(ProductType::Spectrum, res.as_str());
|
let product = ProductSpec::with_subtype(ProductType::Spectrum, res.as_str());
|
||||||
let dr = rel.path_segment();
|
let dr = rel.path_segment();
|
||||||
let ver = rel.version_segment();
|
|
||||||
let res_str = res.as_str();
|
let res_str = res.as_str();
|
||||||
let cache_key = format!("{}|{}|{}|{}", dr, ver, res_str, obsid);
|
let cache_key = format!("{}|{}|{}|{}", dr, ver, res_str, obsid);
|
||||||
let source_label = format!("obsid {}", obsid);
|
let source_label = format!("obsid {}", obsid);
|
||||||
|
|
||||||
// 1) 缓存命中
|
// 1) 缓存命中
|
||||||
if !force {
|
if !force {
|
||||||
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Lamost, &product, &cache_key).await? {
|
if let Some((artifacts, meta)) =
|
||||||
if let Some(_) = cached_files_total_size(&state.config.library_dir, &artifacts) {
|
fetch_observation_cache(&state.db, Source::Lamost, &product, &cache_key).await?
|
||||||
|
{
|
||||||
|
if cached_files_total_size(&state.config.library_dir, &artifacts)
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
info!("[LAMOST] 光谱缓存命中 (obsid={})", obsid);
|
info!("[LAMOST] 光谱缓存命中 (obsid={})", obsid);
|
||||||
return Ok(ObservationProduct {
|
return Ok(ObservationProduct {
|
||||||
source: Source::Lamost, product,
|
source: Source::Lamost,
|
||||||
source_id: cache_key, source_label, artifacts, source_meta: meta,
|
product,
|
||||||
|
source_id: cache_key,
|
||||||
|
source_label,
|
||||||
|
artifacts,
|
||||||
|
source_meta: meta,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
warn!("[LAMOST] 缓存记录存在但文件缺失,重新下载 (obsid={})", obsid);
|
warn!(
|
||||||
|
"[LAMOST] 缓存记录存在但文件缺失,重新下载 (obsid={})",
|
||||||
|
obsid
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2) 下载 + 解压
|
// 2) 下载 + 解压
|
||||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
let gz_bytes = state.lamost.download_fits(obsid, rel).await?;
|
let gz_bytes = state.lamost.download_fits(obsid, rel, &ver).await?;
|
||||||
let fits_bytes = decompress_if_gzip(&gz_bytes)?;
|
let fits_bytes = decompress_if_gzip(&gz_bytes)?;
|
||||||
|
|
||||||
// 3) 落盘
|
// 3) 落盘
|
||||||
let rel_path = format!(
|
let rel_path = format!(
|
||||||
"Telescope/lamost/spectrum/{res}/{dr}/{ver}/{obsid}.fits",
|
"Telescope/lamost/spectrum/{res}/{dr}/{ver}/{obsid}.fits",
|
||||||
res = res_str, dr = dr, ver = ver, obsid = obsid
|
res = res_str,
|
||||||
|
dr = dr,
|
||||||
|
ver = ver,
|
||||||
|
obsid = obsid
|
||||||
|
);
|
||||||
|
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes).await?;
|
||||||
|
info!(
|
||||||
|
"[LAMOST] FITS 已保存 obsid={} → {} ({}B)",
|
||||||
|
obsid, rel_path, size
|
||||||
);
|
);
|
||||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes)?;
|
|
||||||
info!("[LAMOST] FITS 已保存 obsid={} → {} ({}B)", obsid, rel_path, size);
|
|
||||||
|
|
||||||
let meta = serde_json::json!({"obsid": obsid, "dr": dr, "version": ver, "resolution": res_str});
|
let meta =
|
||||||
|
serde_json::json!({"obsid": obsid, "dr": dr, "version": ver, "resolution": res_str});
|
||||||
let artifact = Artifact {
|
let artifact = Artifact {
|
||||||
band: None, original_name: None,
|
band: None,
|
||||||
|
original_name: None,
|
||||||
file_path: rel_path.clone(),
|
file_path: rel_path.clone(),
|
||||||
file_url: file_url_from_path(&rel_path),
|
file_url: file_url_from_path(&rel_path),
|
||||||
file_format: "fits".to_string(),
|
file_format: "fits".to_string(),
|
||||||
size_bytes: size, cached: false,
|
size_bytes: size,
|
||||||
|
cached: false,
|
||||||
};
|
};
|
||||||
let meta_str = meta.to_string();
|
let meta_str = meta.to_string();
|
||||||
if let Err(e) = write_observation_cache(
|
if let Err(e) = write_observation_cache(
|
||||||
&state.db, Source::Lamost, &product, &cache_key, None, None,
|
&state.db,
|
||||||
std::slice::from_ref(&artifact), Some(&meta_str),
|
Source::Lamost,
|
||||||
).await {
|
&product,
|
||||||
|
&cache_key,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
std::slice::from_ref(&artifact),
|
||||||
|
Some(&meta_str),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
log_write_failure(Source::Lamost.as_str(), "spectrum", e);
|
log_write_failure(Source::Lamost.as_str(), "spectrum", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(ObservationProduct {
|
Ok(ObservationProduct {
|
||||||
source: Source::Lamost, product,
|
source: Source::Lamost,
|
||||||
source_id: cache_key, source_label,
|
product,
|
||||||
artifacts: vec![artifact], source_meta: Some(meta),
|
source_id: cache_key,
|
||||||
|
source_label,
|
||||||
|
artifacts: vec![artifact],
|
||||||
|
source_meta: Some(meta),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -280,7 +513,9 @@ fn decompress_if_gzip(bytes: &[u8]) -> Result<Vec<u8>> {
|
|||||||
if bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b {
|
if bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b {
|
||||||
let mut decoder = flate2::read::GzDecoder::new(bytes);
|
let mut decoder = flate2::read::GzDecoder::new(bytes);
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
decoder.read_to_end(&mut out).map_err(|e| anyhow!("gzip 解压失败: {}", e))?;
|
decoder
|
||||||
|
.read_to_end(&mut out)
|
||||||
|
.map_err(|e| anyhow!("gzip 解压失败: {}", e))?;
|
||||||
Ok(out)
|
Ok(out)
|
||||||
} else {
|
} else {
|
||||||
Ok(bytes.to_vec())
|
Ok(bytes.to_vec())
|
||||||
@ -323,9 +558,18 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_row_to_candidate() {
|
fn test_row_to_candidate() {
|
||||||
let row = LamostSpectrumRow {
|
let row = LamostSpectrumRow {
|
||||||
obsid: 438809089, designation: None, ra_obs: Some(10.5), dec_obs: Some(41.0),
|
obsid: 438809089,
|
||||||
z: None, class: None, subclass: None,
|
designation: None,
|
||||||
sn_u: None, sn_g: None, sn_r: None, sn_i: None, sn_z: None,
|
ra_obs: Some(10.5),
|
||||||
|
dec_obs: Some(41.0),
|
||||||
|
z: None,
|
||||||
|
class: None,
|
||||||
|
subclass: None,
|
||||||
|
sn_u: None,
|
||||||
|
sn_g: None,
|
||||||
|
sn_r: None,
|
||||||
|
sn_i: None,
|
||||||
|
sn_z: None,
|
||||||
};
|
};
|
||||||
let c = row_to_candidate(&row, "lrs");
|
let c = row_to_candidate(&row, "lrs");
|
||||||
assert_eq!(c.source_id, "438809089");
|
assert_eq!(c.source_id, "438809089");
|
||||||
|
|||||||
@ -64,32 +64,38 @@ impl ObservationRegistry {
|
|||||||
product: ProductType,
|
product: ProductType,
|
||||||
subtype: Option<&str>,
|
subtype: Option<&str>,
|
||||||
) -> Result<Arc<dyn ObservationFetcher>> {
|
) -> Result<Arc<dyn ObservationFetcher>> {
|
||||||
let candidates = self
|
let candidates = self.fetchers.get(&(source, product)).ok_or_else(|| {
|
||||||
.fetchers
|
anyhow!(
|
||||||
.get(&(source, product))
|
|
||||||
.ok_or_else(|| anyhow!(
|
|
||||||
"{:?} 暂不支持 {:?} 产品。支持的组合:{}",
|
"{:?} 暂不支持 {:?} 产品。支持的组合:{}",
|
||||||
source,
|
source,
|
||||||
product,
|
product,
|
||||||
self.list_supported_combinations()
|
self.list_supported_combinations()
|
||||||
))?;
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
// subtype 精确匹配
|
// subtype 精确匹配
|
||||||
if let Some(st) = subtype {
|
if let Some(st) = subtype {
|
||||||
let st_lower = st.to_lowercase();
|
let st_lower = st.to_lowercase();
|
||||||
for f in candidates {
|
for f in candidates {
|
||||||
if f.subtypes().iter().any(|s| s.eq_ignore_ascii_case(&st_lower)) {
|
if f.subtypes()
|
||||||
|
.iter()
|
||||||
|
.any(|s| s.eq_ignore_ascii_case(&st_lower))
|
||||||
|
{
|
||||||
return Ok(f.clone());
|
return Ok(f.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// 未找到匹配的 subtype — 报错
|
// 未找到匹配的 subtype — 报错
|
||||||
let valid: Vec<&str> = candidates.iter()
|
let valid: Vec<&str> = candidates
|
||||||
|
.iter()
|
||||||
.flat_map(|f| f.subtypes())
|
.flat_map(|f| f.subtypes())
|
||||||
.copied()
|
.copied()
|
||||||
.collect();
|
.collect();
|
||||||
return Err(anyhow!(
|
return Err(anyhow!(
|
||||||
"{:?} 的 {:?} 不支持 subtype '{}',可选: {:?}",
|
"{:?} 的 {:?} 不支持 subtype '{}',可选: {:?}",
|
||||||
source, product, st, valid
|
source,
|
||||||
|
product,
|
||||||
|
st,
|
||||||
|
valid
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -134,6 +140,16 @@ impl ObservationRegistry {
|
|||||||
subtypes: f.subtypes().iter().map(|s| s.to_string()).collect(),
|
subtypes: f.subtypes().iter().map(|s| s.to_string()).collect(),
|
||||||
releases: f.releases().iter().map(|s| s.to_string()).collect(),
|
releases: f.releases().iter().map(|s| s.to_string()).collect(),
|
||||||
default_release: f.default_release().map(|s| s.to_string()),
|
default_release: f.default_release().map(|s| s.to_string()),
|
||||||
|
// 每个 release 对应的子版本列表(仅 LAMOST 有内容)
|
||||||
|
release_versions: f
|
||||||
|
.releases()
|
||||||
|
.iter()
|
||||||
|
.map(|r| (r.to_string(), f.versions_for_release(Some(r))))
|
||||||
|
.collect(),
|
||||||
|
// 需要登录认证的 release(LAMOST Internal DR12-14)
|
||||||
|
releases_requiring_auth: f.releases_requiring_auth(),
|
||||||
|
// 每个 release 实际支持的 subtype(表达交叉约束,如 LAMOST DR5/6 无 MRS)
|
||||||
|
subtypes_per_release: f.subtypes_per_release().into_iter().collect(),
|
||||||
suggested_max_radius_deg: f.suggested_max_radius_deg(),
|
suggested_max_radius_deg: f.suggested_max_radius_deg(),
|
||||||
hard_max_radius_deg: f.hard_max_radius_deg(),
|
hard_max_radius_deg: f.hard_max_radius_deg(),
|
||||||
identifier_format: f.identifier_format().map(|s| s.to_string()),
|
identifier_format: f.identifier_format().map(|s| s.to_string()),
|
||||||
@ -163,6 +179,15 @@ pub struct CapabilitySpec {
|
|||||||
pub releases: Vec<String>,
|
pub releases: Vec<String>,
|
||||||
/// 默认版本(release 参数为空时使用),None 表示无版本概念
|
/// 默认版本(release 参数为空时使用),None 表示无版本概念
|
||||||
pub default_release: Option<String>,
|
pub default_release: Option<String>,
|
||||||
|
/// 每个 release 对应的子版本列表(仅 LAMOST 有内容;key=release 如 "dr11",value=["v0","v1.0",...])
|
||||||
|
#[serde(default)]
|
||||||
|
pub release_versions: std::collections::BTreeMap<String, Vec<String>>,
|
||||||
|
/// 需要登录认证才能访问的 release 列表(如 LAMOST Internal DR12-14),空表示全部公开
|
||||||
|
#[serde(default)]
|
||||||
|
pub releases_requiring_auth: Vec<String>,
|
||||||
|
/// 每个 release 实际支持的 subtype(表达交叉约束,如 LAMOST DR5/6 无 MRS);空表示所有 release 都支持全部 subtypes
|
||||||
|
#[serde(default)]
|
||||||
|
pub subtypes_per_release: std::collections::BTreeMap<String, Vec<String>>,
|
||||||
/// 坐标模式检索半径的建议上限(度),超出仅警告不拒绝
|
/// 坐标模式检索半径的建议上限(度),超出仅警告不拒绝
|
||||||
pub suggested_max_radius_deg: f64,
|
pub suggested_max_radius_deg: f64,
|
||||||
/// 坐标模式检索半径的硬性上限(度),超出会返回错误
|
/// 坐标模式检索半径的硬性上限(度),超出会返回错误
|
||||||
@ -186,8 +211,12 @@ mod tests {
|
|||||||
assert!(reg.get(Source::Gaia, ProductType::LightCurve, None).is_ok());
|
assert!(reg.get(Source::Gaia, ProductType::LightCurve, None).is_ok());
|
||||||
// SDSS spectrum (specobj 默认 + apogee/aspcap subtype)
|
// SDSS spectrum (specobj 默认 + apogee/aspcap subtype)
|
||||||
assert!(reg.get(Source::Sdss, ProductType::Spectrum, None).is_ok());
|
assert!(reg.get(Source::Sdss, ProductType::Spectrum, None).is_ok());
|
||||||
assert!(reg.get(Source::Sdss, ProductType::Spectrum, Some("apstar")).is_ok());
|
assert!(reg
|
||||||
assert!(reg.get(Source::Sdss, ProductType::Spectrum, Some("aspcap")).is_ok());
|
.get(Source::Sdss, ProductType::Spectrum, Some("apstar"))
|
||||||
|
.is_ok());
|
||||||
|
assert!(reg
|
||||||
|
.get(Source::Sdss, ProductType::Spectrum, Some("aspcap"))
|
||||||
|
.is_ok());
|
||||||
// DESI spectrum
|
// DESI spectrum
|
||||||
assert!(reg.get(Source::Desi, ProductType::Spectrum, None).is_ok());
|
assert!(reg.get(Source::Desi, ProductType::Spectrum, None).is_ok());
|
||||||
}
|
}
|
||||||
@ -195,7 +224,9 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_unsupported_combination_error() {
|
fn test_unsupported_combination_error() {
|
||||||
let reg = ObservationRegistry::default();
|
let reg = ObservationRegistry::default();
|
||||||
let err = reg.get(Source::Lamost, ProductType::LightCurve, None).unwrap_err();
|
let err = reg
|
||||||
|
.get(Source::Lamost, ProductType::LightCurve, None)
|
||||||
|
.unwrap_err();
|
||||||
assert!(format!("{}", err).contains("暂不支持"));
|
assert!(format!("{}", err).contains("暂不支持"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -203,7 +234,9 @@ mod tests {
|
|||||||
fn test_sdss_subtype_routing() {
|
fn test_sdss_subtype_routing() {
|
||||||
let reg = ObservationRegistry::default();
|
let reg = ObservationRegistry::default();
|
||||||
// subtype=apstar 应路由到 SdssApogeeFetcher(其 subtypes 含 "apstar")
|
// subtype=apstar 应路由到 SdssApogeeFetcher(其 subtypes 含 "apstar")
|
||||||
let apstar = reg.get(Source::Sdss, ProductType::Spectrum, Some("apstar")).unwrap();
|
let apstar = reg
|
||||||
|
.get(Source::Sdss, ProductType::Spectrum, Some("apstar"))
|
||||||
|
.unwrap();
|
||||||
assert!(apstar.subtypes().contains(&"apstar"));
|
assert!(apstar.subtypes().contains(&"apstar"));
|
||||||
// 默认(无 subtype)应路由到 SdssSpectrumFetcher
|
// 默认(无 subtype)应路由到 SdssSpectrumFetcher
|
||||||
let specobj = reg.get(Source::Sdss, ProductType::Spectrum, None).unwrap();
|
let specobj = reg.get(Source::Sdss, ProductType::Spectrum, None).unwrap();
|
||||||
|
|||||||
@ -23,11 +23,13 @@ use tracing::{info, warn};
|
|||||||
|
|
||||||
// ── 版本枚举 ──
|
// ── 版本枚举 ──
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||||
pub enum SdssRelease { Dr16, Dr17, Dr18, Dr19 }
|
pub enum SdssRelease {
|
||||||
|
Dr16,
|
||||||
impl Default for SdssRelease {
|
#[default]
|
||||||
fn default() -> Self { Self::Dr17 }
|
Dr17,
|
||||||
|
Dr18,
|
||||||
|
Dr19,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SdssRelease {
|
impl SdssRelease {
|
||||||
@ -36,8 +38,10 @@ impl SdssRelease {
|
|||||||
}
|
}
|
||||||
pub fn sas_dr_segment(&self) -> &'static str {
|
pub fn sas_dr_segment(&self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Self::Dr16 => "dr16", Self::Dr17 => "dr17",
|
Self::Dr16 => "dr16",
|
||||||
Self::Dr18 => "dr18", Self::Dr19 => "dr19",
|
Self::Dr17 => "dr17",
|
||||||
|
Self::Dr18 => "dr18",
|
||||||
|
Self::Dr19 => "dr19",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub fn datalab_table(&self) -> Option<&'static str> {
|
pub fn datalab_table(&self) -> Option<&'static str> {
|
||||||
@ -55,8 +59,10 @@ impl SdssRelease {
|
|||||||
}
|
}
|
||||||
pub fn display(&self) -> &'static str {
|
pub fn display(&self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Self::Dr16 => "DR16", Self::Dr17 => "DR17",
|
Self::Dr16 => "DR16",
|
||||||
Self::Dr18 => "DR18", Self::Dr19 => "DR19",
|
Self::Dr17 => "DR17",
|
||||||
|
Self::Dr18 => "DR18",
|
||||||
|
Self::Dr19 => "DR19",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -68,7 +74,13 @@ pub fn parse_sdss_release(s: Option<&str>) -> Result<SdssRelease> {
|
|||||||
Some("dr17") => SdssRelease::Dr17,
|
Some("dr17") => SdssRelease::Dr17,
|
||||||
Some("dr18") => SdssRelease::Dr18,
|
Some("dr18") => SdssRelease::Dr18,
|
||||||
Some("dr19") => SdssRelease::Dr19,
|
Some("dr19") => SdssRelease::Dr19,
|
||||||
Some(other) => return Err(anyhow!("不支持的 SDSS release '{}',可选: {:?}", other, SdssRelease::valid_values())),
|
Some(other) => {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"不支持的 SDSS release '{}',可选: {:?}",
|
||||||
|
other,
|
||||||
|
SdssRelease::valid_values()
|
||||||
|
))
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -131,8 +143,12 @@ impl ObservationFetcher for SdssSpectrumFetcher {
|
|||||||
async fn cone_search_raw(
|
async fn cone_search_raw(
|
||||||
&self,
|
&self,
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
ra: f64, dec: f64, radius_deg: f64,
|
ra: f64,
|
||||||
release: Option<&str>, _subtype: Option<&str>,
|
dec: f64,
|
||||||
|
radius_deg: f64,
|
||||||
|
release: Option<&str>,
|
||||||
|
_subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
) -> Result<Vec<Candidate>> {
|
) -> Result<Vec<Candidate>> {
|
||||||
validate_cone_params(ra, dec, radius_deg)?;
|
validate_cone_params(ra, dec, radius_deg)?;
|
||||||
let rel = parse_sdss_release(release)?;
|
let rel = parse_sdss_release(release)?;
|
||||||
@ -145,6 +161,7 @@ impl ObservationFetcher for SdssSpectrumFetcher {
|
|||||||
identifier: &str,
|
identifier: &str,
|
||||||
_release: Option<&str>,
|
_release: Option<&str>,
|
||||||
_subtype: Option<&str>,
|
_subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
) -> Result<Candidate> {
|
) -> Result<Candidate> {
|
||||||
let parts: Vec<&str> = identifier.split('-').collect();
|
let parts: Vec<&str> = identifier.split('-').collect();
|
||||||
if parts.len() != 4 {
|
if parts.len() != 4 {
|
||||||
@ -153,14 +170,22 @@ impl ObservationFetcher for SdssSpectrumFetcher {
|
|||||||
identifier
|
identifier
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let plate: i64 = parts[1].parse().map_err(|_| anyhow!("plate 非数字: '{}'", parts[1]))?;
|
let plate: i64 = parts[1]
|
||||||
let mjd: i64 = parts[2].parse().map_err(|_| anyhow!("mjd 非数字: '{}'", parts[2]))?;
|
.parse()
|
||||||
let fiberid: i64 = parts[3].parse().map_err(|_| anyhow!("fiberid 非数字: '{}'", parts[3]))?;
|
.map_err(|_| anyhow!("plate 非数字: '{}'", parts[1]))?;
|
||||||
|
let mjd: i64 = parts[2]
|
||||||
|
.parse()
|
||||||
|
.map_err(|_| anyhow!("mjd 非数字: '{}'", parts[2]))?;
|
||||||
|
let fiberid: i64 = parts[3]
|
||||||
|
.parse()
|
||||||
|
.map_err(|_| anyhow!("fiberid 非数字: '{}'", parts[3]))?;
|
||||||
Ok(Candidate {
|
Ok(Candidate {
|
||||||
source: Source::Sdss,
|
source: Source::Sdss,
|
||||||
source_id: identifier.to_string(),
|
source_id: identifier.to_string(),
|
||||||
label: format!("plate {} mjd {} fiber {}", plate, mjd, fiberid),
|
label: format!("plate {} mjd {} fiber {}", plate, mjd, fiberid),
|
||||||
ra: None, dec: None, distance: None,
|
ra: None,
|
||||||
|
dec: None,
|
||||||
|
distance: None,
|
||||||
raw: Some(serde_json::json!({
|
raw: Some(serde_json::json!({
|
||||||
"plate": plate, "mjd": mjd, "fiberid": fiberid, "run2d": parts[0]
|
"plate": plate, "mjd": mjd, "fiberid": fiberid, "run2d": parts[0]
|
||||||
})),
|
})),
|
||||||
@ -173,15 +198,32 @@ impl ObservationFetcher for SdssSpectrumFetcher {
|
|||||||
candidate: &Candidate,
|
candidate: &Candidate,
|
||||||
release: Option<&str>,
|
release: Option<&str>,
|
||||||
_subtype: Option<&str>,
|
_subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
force: bool,
|
force: bool,
|
||||||
) -> Result<ObservationProduct> {
|
) -> Result<ObservationProduct> {
|
||||||
let plate = candidate.raw.as_ref().and_then(|v| v.get("plate")).and_then(|v| v.as_i64())
|
let plate = candidate
|
||||||
|
.raw
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("plate"))
|
||||||
|
.and_then(|v| v.as_i64())
|
||||||
.ok_or_else(|| anyhow!("SDSS Candidate 缺 plate"))?;
|
.ok_or_else(|| anyhow!("SDSS Candidate 缺 plate"))?;
|
||||||
let mjd = candidate.raw.as_ref().and_then(|v| v.get("mjd")).and_then(|v| v.as_i64())
|
let mjd = candidate
|
||||||
|
.raw
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("mjd"))
|
||||||
|
.and_then(|v| v.as_i64())
|
||||||
.ok_or_else(|| anyhow!("SDSS Candidate 缺 mjd"))?;
|
.ok_or_else(|| anyhow!("SDSS Candidate 缺 mjd"))?;
|
||||||
let fiberid = candidate.raw.as_ref().and_then(|v| v.get("fiberid")).and_then(|v| v.as_i64())
|
let fiberid = candidate
|
||||||
|
.raw
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("fiberid"))
|
||||||
|
.and_then(|v| v.as_i64())
|
||||||
.ok_or_else(|| anyhow!("SDSS Candidate 缺 fiberid"))?;
|
.ok_or_else(|| anyhow!("SDSS Candidate 缺 fiberid"))?;
|
||||||
let run2d = candidate.raw.as_ref().and_then(|v| v.get("run2d")).and_then(|v| v.as_str())
|
let run2d = candidate
|
||||||
|
.raw
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("run2d"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
.ok_or_else(|| anyhow!("SDSS Candidate 缺 run2d"))?;
|
.ok_or_else(|| anyhow!("SDSS Candidate 缺 run2d"))?;
|
||||||
let rel = parse_sdss_release(release)?;
|
let rel = parse_sdss_release(release)?;
|
||||||
let product = ProductSpec::with_subtype(ProductType::Spectrum, "spec");
|
let product = ProductSpec::with_subtype(ProductType::Spectrum, "spec");
|
||||||
@ -191,12 +233,21 @@ impl ObservationFetcher for SdssSpectrumFetcher {
|
|||||||
|
|
||||||
// 1) 缓存命中
|
// 1) 缓存命中
|
||||||
if !force {
|
if !force {
|
||||||
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Sdss, &product, &cache_key).await? {
|
if let Some((artifacts, meta)) =
|
||||||
if cached_files_total_size(&state.config.library_dir, &artifacts).is_some() {
|
fetch_observation_cache(&state.db, Source::Sdss, &product, &cache_key).await?
|
||||||
|
{
|
||||||
|
if cached_files_total_size(&state.config.library_dir, &artifacts)
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
info!("[SDSS] 光谱缓存命中 ({})", cache_key);
|
info!("[SDSS] 光谱缓存命中 ({})", cache_key);
|
||||||
return Ok(ObservationProduct {
|
return Ok(ObservationProduct {
|
||||||
source: Source::Sdss, product,
|
source: Source::Sdss,
|
||||||
source_id: cache_key, source_label, artifacts, source_meta: meta,
|
product,
|
||||||
|
source_id: cache_key,
|
||||||
|
source_label,
|
||||||
|
artifacts,
|
||||||
|
source_meta: meta,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
warn!("[SDSS] 缓存记录存在但文件缺失,重新下载 ({})", cache_key);
|
warn!("[SDSS] 缓存记录存在但文件缺失,重新下载 ({})", cache_key);
|
||||||
@ -205,34 +256,57 @@ impl ObservationFetcher for SdssSpectrumFetcher {
|
|||||||
|
|
||||||
// 2) SAS 下载(未压缩 FITS)+ 落盘
|
// 2) SAS 下载(未压缩 FITS)+ 落盘
|
||||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
let fits_bytes = state.sdss.download_fits(plate, mjd, fiberid, run2d, rel).await?;
|
let fits_bytes = state
|
||||||
|
.sdss
|
||||||
|
.download_fits(plate, mjd, fiberid, run2d, rel)
|
||||||
|
.await?;
|
||||||
let rel_path = format!(
|
let rel_path = format!(
|
||||||
"Telescope/sdss/{dr}/spec/{run2d}-{plate}-{mjd}-{fiber:04}.fits",
|
"Telescope/sdss/{dr}/spec/{run2d}-{plate}-{mjd}-{fiber:04}.fits",
|
||||||
dr = dr, run2d = run2d, plate = plate, mjd = mjd, fiber = fiberid
|
dr = dr,
|
||||||
|
run2d = run2d,
|
||||||
|
plate = plate,
|
||||||
|
mjd = mjd,
|
||||||
|
fiber = fiberid
|
||||||
|
);
|
||||||
|
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes).await?;
|
||||||
|
info!(
|
||||||
|
"[SDSS] FITS 已保存 {} → {} ({}B)",
|
||||||
|
cache_key, rel_path, size
|
||||||
);
|
);
|
||||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes)?;
|
|
||||||
info!("[SDSS] FITS 已保存 {} → {} ({}B)", cache_key, rel_path, size);
|
|
||||||
|
|
||||||
let meta = serde_json::json!({"plate": plate, "mjd": mjd, "fiberid": fiberid, "run2d": run2d, "dr": dr});
|
let meta = serde_json::json!({"plate": plate, "mjd": mjd, "fiberid": fiberid, "run2d": run2d, "dr": dr});
|
||||||
let artifact = Artifact {
|
let artifact = Artifact {
|
||||||
band: None, original_name: None,
|
band: None,
|
||||||
|
original_name: None,
|
||||||
file_path: rel_path.clone(),
|
file_path: rel_path.clone(),
|
||||||
file_url: file_url_from_path(&rel_path),
|
file_url: file_url_from_path(&rel_path),
|
||||||
file_format: "fits".to_string(),
|
file_format: "fits".to_string(),
|
||||||
size_bytes: size, cached: false,
|
size_bytes: size,
|
||||||
|
cached: false,
|
||||||
};
|
};
|
||||||
let meta_str = meta.to_string();
|
let meta_str = meta.to_string();
|
||||||
if let Err(e) = write_observation_cache(
|
if let Err(e) = write_observation_cache(
|
||||||
&state.db, Source::Sdss, &product, &cache_key, None, None,
|
&state.db,
|
||||||
std::slice::from_ref(&artifact), Some(&meta_str),
|
Source::Sdss,
|
||||||
).await {
|
&product,
|
||||||
|
&cache_key,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
std::slice::from_ref(&artifact),
|
||||||
|
Some(&meta_str),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
log_write_failure(Source::Sdss.as_str(), "spectrum", e);
|
log_write_failure(Source::Sdss.as_str(), "spectrum", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(ObservationProduct {
|
Ok(ObservationProduct {
|
||||||
source: Source::Sdss, product,
|
source: Source::Sdss,
|
||||||
source_id: cache_key, source_label,
|
product,
|
||||||
artifacts: vec![artifact], source_meta: Some(meta),
|
source_id: cache_key,
|
||||||
|
source_label,
|
||||||
|
artifacts: vec![artifact],
|
||||||
|
source_meta: Some(meta),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -242,7 +316,9 @@ fn row_to_candidate(row: &SdssSpectrumRow) -> Candidate {
|
|||||||
source: Source::Sdss,
|
source: Source::Sdss,
|
||||||
source_id: format!("{}-{}-{}-{}", row.run2d, row.plate, row.mjd, row.fiberid),
|
source_id: format!("{}-{}-{}-{}", row.run2d, row.plate, row.mjd, row.fiberid),
|
||||||
label: format!("plate {} mjd {} fiber {}", row.plate, row.mjd, row.fiberid),
|
label: format!("plate {} mjd {} fiber {}", row.plate, row.mjd, row.fiberid),
|
||||||
ra: row.ra, dec: row.dec, distance: None,
|
ra: row.ra,
|
||||||
|
dec: row.dec,
|
||||||
|
distance: None,
|
||||||
raw: Some(serde_json::json!({
|
raw: Some(serde_json::json!({
|
||||||
"plate": row.plate, "mjd": row.mjd, "fiberid": row.fiberid, "run2d": row.run2d
|
"plate": row.plate, "mjd": row.mjd, "fiberid": row.fiberid, "run2d": row.run2d
|
||||||
})),
|
})),
|
||||||
@ -287,12 +363,19 @@ impl ObservationFetcher for SdssApogeeFetcher {
|
|||||||
async fn cone_search_raw(
|
async fn cone_search_raw(
|
||||||
&self,
|
&self,
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
ra: f64, dec: f64, radius_deg: f64,
|
ra: f64,
|
||||||
release: Option<&str>, _subtype: Option<&str>,
|
dec: f64,
|
||||||
|
radius_deg: f64,
|
||||||
|
release: Option<&str>,
|
||||||
|
_subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
) -> Result<Vec<Candidate>> {
|
) -> Result<Vec<Candidate>> {
|
||||||
validate_cone_params(ra, dec, radius_deg)?;
|
validate_cone_params(ra, dec, radius_deg)?;
|
||||||
let rel = parse_sdss_release(release)?;
|
let rel = parse_sdss_release(release)?;
|
||||||
let result = state.sdss.apogee_cone_search(ra, dec, radius_deg, 50, rel).await?;
|
let result = state
|
||||||
|
.sdss
|
||||||
|
.apogee_cone_search(ra, dec, radius_deg, 50, rel)
|
||||||
|
.await?;
|
||||||
Ok(result.rows.iter().map(apogee_row_to_candidate).collect())
|
Ok(result.rows.iter().map(apogee_row_to_candidate).collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -301,6 +384,7 @@ impl ObservationFetcher for SdssApogeeFetcher {
|
|||||||
identifier: &str,
|
identifier: &str,
|
||||||
_release: Option<&str>,
|
_release: Option<&str>,
|
||||||
subtype: Option<&str>,
|
subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
) -> Result<Candidate> {
|
) -> Result<Candidate> {
|
||||||
// 格式 "{telescope}|{field}|{apogee_id}"
|
// 格式 "{telescope}|{field}|{apogee_id}"
|
||||||
let parts: Vec<&str> = identifier.splitn(3, '|').collect();
|
let parts: Vec<&str> = identifier.splitn(3, '|').collect();
|
||||||
@ -315,7 +399,9 @@ impl ObservationFetcher for SdssApogeeFetcher {
|
|||||||
source: Source::Sdss,
|
source: Source::Sdss,
|
||||||
source_id: identifier.to_string(),
|
source_id: identifier.to_string(),
|
||||||
label: format!("{} {} {}", parts[0], dt, parts[2]),
|
label: format!("{} {} {}", parts[0], dt, parts[2]),
|
||||||
ra: None, dec: None, distance: None,
|
ra: None,
|
||||||
|
dec: None,
|
||||||
|
distance: None,
|
||||||
raw: Some(serde_json::json!({
|
raw: Some(serde_json::json!({
|
||||||
"apogee_id": parts[2], "telescope": parts[0], "field": parts[1], "data_type": dt
|
"apogee_id": parts[2], "telescope": parts[0], "field": parts[1], "data_type": dt
|
||||||
})),
|
})),
|
||||||
@ -328,16 +414,34 @@ impl ObservationFetcher for SdssApogeeFetcher {
|
|||||||
candidate: &Candidate,
|
candidate: &Candidate,
|
||||||
release: Option<&str>,
|
release: Option<&str>,
|
||||||
subtype: Option<&str>,
|
subtype: Option<&str>,
|
||||||
|
_version: Option<&str>,
|
||||||
force: bool,
|
force: bool,
|
||||||
) -> Result<ObservationProduct> {
|
) -> Result<ObservationProduct> {
|
||||||
let apogee_id = candidate.raw.as_ref().and_then(|v| v.get("apogee_id")).and_then(|v| v.as_str())
|
let apogee_id = candidate
|
||||||
|
.raw
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("apogee_id"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
.ok_or_else(|| anyhow!("APOGEE Candidate 缺 apogee_id"))?;
|
.ok_or_else(|| anyhow!("APOGEE Candidate 缺 apogee_id"))?;
|
||||||
let telescope = candidate.raw.as_ref().and_then(|v| v.get("telescope")).and_then(|v| v.as_str())
|
let telescope = candidate
|
||||||
|
.raw
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("telescope"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
.ok_or_else(|| anyhow!("APOGEE Candidate 缺 telescope"))?;
|
.ok_or_else(|| anyhow!("APOGEE Candidate 缺 telescope"))?;
|
||||||
let field = candidate.raw.as_ref().and_then(|v| v.get("field")).and_then(|v| v.as_str())
|
let field = candidate
|
||||||
|
.raw
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("field"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
.ok_or_else(|| anyhow!("APOGEE Candidate 缺 field"))?;
|
.ok_or_else(|| anyhow!("APOGEE Candidate 缺 field"))?;
|
||||||
let dt = candidate.raw.as_ref().and_then(|v| v.get("data_type")).and_then(|v| v.as_str())
|
let dt = candidate
|
||||||
.or(subtype).unwrap_or("apstar");
|
.raw
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("data_type"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.or(subtype)
|
||||||
|
.unwrap_or("apstar");
|
||||||
let rel = parse_sdss_release(release)?;
|
let rel = parse_sdss_release(release)?;
|
||||||
let product = ProductSpec::with_subtype(ProductType::Spectrum, dt);
|
let product = ProductSpec::with_subtype(ProductType::Spectrum, dt);
|
||||||
let dr = rel.sas_dr_segment();
|
let dr = rel.sas_dr_segment();
|
||||||
@ -346,12 +450,21 @@ impl ObservationFetcher for SdssApogeeFetcher {
|
|||||||
|
|
||||||
// 1) 缓存命中
|
// 1) 缓存命中
|
||||||
if !force {
|
if !force {
|
||||||
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Sdss, &product, &cache_key).await? {
|
if let Some((artifacts, meta)) =
|
||||||
if cached_files_total_size(&state.config.library_dir, &artifacts).is_some() {
|
fetch_observation_cache(&state.db, Source::Sdss, &product, &cache_key).await?
|
||||||
|
{
|
||||||
|
if cached_files_total_size(&state.config.library_dir, &artifacts)
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
info!("[APOGEE] 缓存命中 ({})", cache_key);
|
info!("[APOGEE] 缓存命中 ({})", cache_key);
|
||||||
return Ok(ObservationProduct {
|
return Ok(ObservationProduct {
|
||||||
source: Source::Sdss, product,
|
source: Source::Sdss,
|
||||||
source_id: cache_key, source_label, artifacts, source_meta: meta,
|
product,
|
||||||
|
source_id: cache_key,
|
||||||
|
source_label,
|
||||||
|
artifacts,
|
||||||
|
source_meta: meta,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
warn!("[APOGEE] 缓存记录存在但文件缺失,重新下载 ({})", cache_key);
|
warn!("[APOGEE] 缓存记录存在但文件缺失,重新下载 ({})", cache_key);
|
||||||
@ -360,10 +473,18 @@ impl ObservationFetcher for SdssApogeeFetcher {
|
|||||||
|
|
||||||
// 2) 下载 FITS(无压缩)
|
// 2) 下载 FITS(无压缩)
|
||||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
let fits_bytes = state.sdss.download_apogee_fits(apogee_id, telescope, field, rel, dt).await?;
|
let fits_bytes = state
|
||||||
let safe_id = apogee_id.replace('+', "_").replace('/', "_");
|
.sdss
|
||||||
let rel_path = format!("Telescope/sdss/{dr}/{dt}/{safe_id}.fits", dr = dr, dt = dt, safe_id = safe_id);
|
.download_apogee_fits(apogee_id, telescope, field, rel, dt)
|
||||||
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes)?;
|
.await?;
|
||||||
|
let safe_id = apogee_id.replace(['+', '/'], "_");
|
||||||
|
let rel_path = format!(
|
||||||
|
"Telescope/sdss/{dr}/{dt}/{safe_id}.fits",
|
||||||
|
dr = dr,
|
||||||
|
dt = dt,
|
||||||
|
safe_id = safe_id
|
||||||
|
);
|
||||||
|
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes).await?;
|
||||||
info!("[APOGEE] 已保存 {} → {} ({}B)", cache_key, rel_path, size);
|
info!("[APOGEE] 已保存 {} → {} ({}B)", cache_key, rel_path, size);
|
||||||
|
|
||||||
let meta = serde_json::json!({
|
let meta = serde_json::json!({
|
||||||
@ -371,24 +492,37 @@ impl ObservationFetcher for SdssApogeeFetcher {
|
|||||||
"data_type": dt, "dr": dr
|
"data_type": dt, "dr": dr
|
||||||
});
|
});
|
||||||
let artifact = Artifact {
|
let artifact = Artifact {
|
||||||
band: None, original_name: None,
|
band: None,
|
||||||
|
original_name: None,
|
||||||
file_path: rel_path.clone(),
|
file_path: rel_path.clone(),
|
||||||
file_url: file_url_from_path(&rel_path),
|
file_url: file_url_from_path(&rel_path),
|
||||||
file_format: "fits".to_string(),
|
file_format: "fits".to_string(),
|
||||||
size_bytes: size, cached: false,
|
size_bytes: size,
|
||||||
|
cached: false,
|
||||||
};
|
};
|
||||||
let meta_str = meta.to_string();
|
let meta_str = meta.to_string();
|
||||||
if let Err(e) = write_observation_cache(
|
if let Err(e) = write_observation_cache(
|
||||||
&state.db, Source::Sdss, &product, &cache_key, None, None,
|
&state.db,
|
||||||
std::slice::from_ref(&artifact), Some(&meta_str),
|
Source::Sdss,
|
||||||
).await {
|
&product,
|
||||||
|
&cache_key,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
std::slice::from_ref(&artifact),
|
||||||
|
Some(&meta_str),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
log_write_failure(Source::Sdss.as_str(), "apogee", e);
|
log_write_failure(Source::Sdss.as_str(), "apogee", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(ObservationProduct {
|
Ok(ObservationProduct {
|
||||||
source: Source::Sdss, product,
|
source: Source::Sdss,
|
||||||
source_id: cache_key, source_label,
|
product,
|
||||||
artifacts: vec![artifact], source_meta: Some(meta),
|
source_id: cache_key,
|
||||||
|
source_label,
|
||||||
|
artifacts: vec![artifact],
|
||||||
|
source_meta: Some(meta),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -398,7 +532,9 @@ fn apogee_row_to_candidate(row: &ApogeeStarRow) -> Candidate {
|
|||||||
source: Source::Sdss,
|
source: Source::Sdss,
|
||||||
source_id: format!("{}|{}|{}", row.telescope, row.field, row.apogee_id),
|
source_id: format!("{}|{}|{}", row.telescope, row.field, row.apogee_id),
|
||||||
label: format!("{} {}", row.telescope, row.apogee_id),
|
label: format!("{} {}", row.telescope, row.apogee_id),
|
||||||
ra: row.ra, dec: row.dec, distance: None,
|
ra: row.ra,
|
||||||
|
dec: row.dec,
|
||||||
|
distance: None,
|
||||||
raw: Some(serde_json::json!({
|
raw: Some(serde_json::json!({
|
||||||
"apogee_id": row.apogee_id, "telescope": row.telescope, "field": row.field
|
"apogee_id": row.apogee_id, "telescope": row.telescope, "field": row.field
|
||||||
})),
|
})),
|
||||||
|
|||||||
@ -39,6 +39,24 @@ impl Source {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 从字符串解析数据源(大小写不敏感)。新增源只需在此加一行 match arm,
|
||||||
|
/// 所有调用方(API handler、Agent tool)统一用本方法,不再各自手写 match。
|
||||||
|
pub fn from_str(s: &str) -> Result<Self, String> {
|
||||||
|
Ok(match s.to_lowercase().as_str() {
|
||||||
|
"lamost" => Self::Lamost,
|
||||||
|
"gaia" => Self::Gaia,
|
||||||
|
"sdss" => Self::Sdss,
|
||||||
|
"desi" => Self::Desi,
|
||||||
|
other => {
|
||||||
|
return Err(format!(
|
||||||
|
"不支持的 source '{}',可选: {:?}",
|
||||||
|
other,
|
||||||
|
Self::valid_values()
|
||||||
|
))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn display(&self) -> &'static str {
|
pub fn display(&self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Self::Lamost => "LAMOST",
|
Self::Lamost => "LAMOST",
|
||||||
@ -81,6 +99,24 @@ impl ProductType {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 从字符串解析产品类型(大小写不敏感,接受常见别名)。
|
||||||
|
/// 别名集中在此一处管理,所有调用方统一用本方法。
|
||||||
|
pub fn from_str(s: &str) -> Result<Self, String> {
|
||||||
|
Ok(match s.to_lowercase().as_str() {
|
||||||
|
"spectrum" | "spec" => Self::Spectrum,
|
||||||
|
"lightcurve" | "light_curve" | "lc" => Self::LightCurve,
|
||||||
|
"photometry" => Self::Photometry,
|
||||||
|
"image" => Self::Image,
|
||||||
|
other => {
|
||||||
|
return Err(format!(
|
||||||
|
"不支持的 product '{}',可选: {:?}",
|
||||||
|
other,
|
||||||
|
Self::valid_values()
|
||||||
|
))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn display(&self) -> &'static str {
|
pub fn display(&self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Self::Spectrum => "光谱",
|
Self::Spectrum => "光谱",
|
||||||
@ -113,11 +149,17 @@ pub struct ProductSpec {
|
|||||||
|
|
||||||
impl ProductSpec {
|
impl ProductSpec {
|
||||||
pub fn new(product: ProductType) -> Self {
|
pub fn new(product: ProductType) -> Self {
|
||||||
Self { product, subtype: None }
|
Self {
|
||||||
|
product,
|
||||||
|
subtype: None,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn with_subtype(product: ProductType, subtype: impl Into<String>) -> Self {
|
pub fn with_subtype(product: ProductType, subtype: impl Into<String>) -> Self {
|
||||||
Self { product, subtype: Some(subtype.into()) }
|
Self {
|
||||||
|
product,
|
||||||
|
subtype: Some(subtype.into()),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 落盘子目录:有 subtype 则拼 "product/subtype",否则只用 product
|
/// 落盘子目录:有 subtype 则拼 "product/subtype",否则只用 product
|
||||||
@ -227,6 +269,9 @@ pub enum ObservationRequest {
|
|||||||
strategy: FindStrategy,
|
strategy: FindStrategy,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
release: Option<String>,
|
release: Option<String>,
|
||||||
|
/// 数据发布的子版本(仅 LAMOST 有意义,如 "v2.0"/"v1.0");None 时用该 DR 的默认版本
|
||||||
|
#[serde(default)]
|
||||||
|
version: Option<String>,
|
||||||
},
|
},
|
||||||
ByIdentifier {
|
ByIdentifier {
|
||||||
source: Source,
|
source: Source,
|
||||||
@ -234,6 +279,8 @@ pub enum ObservationRequest {
|
|||||||
identifiers: Vec<String>,
|
identifiers: Vec<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
release: Option<String>,
|
release: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
version: Option<String>,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -253,7 +300,10 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_product_spec_path() {
|
fn test_product_spec_path() {
|
||||||
assert_eq!(ProductSpec::new(ProductType::Spectrum).path_segment(), "spectrum");
|
assert_eq!(
|
||||||
|
ProductSpec::new(ProductType::Spectrum).path_segment(),
|
||||||
|
"spectrum"
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
ProductSpec::with_subtype(ProductType::Spectrum, "lrs").path_segment(),
|
ProductSpec::with_subtype(ProductType::Spectrum, "lrs").path_segment(),
|
||||||
"spectrum/lrs"
|
"spectrum/lrs"
|
||||||
|
|||||||
@ -88,6 +88,12 @@ pub async fn upload_paper_file_service(
|
|||||||
.map_err(|e| anyhow::anyhow!("PDF 文件内容校验失败: {}", e))?;
|
.map_err(|e| anyhow::anyhow!("PDF 文件内容校验失败: {}", e))?;
|
||||||
let pdf_filename = format!("{}.pdf", bibcode);
|
let pdf_filename = format!("{}.pdf", bibcode);
|
||||||
let pdf_dest = library_dir.join("PDF").join(&pdf_filename);
|
let pdf_dest = library_dir.join("PDF").join(&pdf_filename);
|
||||||
|
if pdf_dest
|
||||||
|
.components()
|
||||||
|
.any(|c| matches!(c, std::path::Component::ParentDir))
|
||||||
|
{
|
||||||
|
anyhow::bail!("非法的文件写入路径,bibcode 包含路径穿越字符");
|
||||||
|
}
|
||||||
if let Some(parent) = pdf_dest.parent() {
|
if let Some(parent) = pdf_dest.parent() {
|
||||||
tokio::fs::create_dir_all(parent)
|
tokio::fs::create_dir_all(parent)
|
||||||
.await
|
.await
|
||||||
@ -104,6 +110,12 @@ pub async fn upload_paper_file_service(
|
|||||||
.map_err(|e| anyhow::anyhow!("HTML 文件内容校验失败: {}", e))?;
|
.map_err(|e| anyhow::anyhow!("HTML 文件内容校验失败: {}", e))?;
|
||||||
let html_filename = format!("{}.html", bibcode);
|
let html_filename = format!("{}.html", bibcode);
|
||||||
let html_dest = library_dir.join("HTML").join(&html_filename);
|
let html_dest = library_dir.join("HTML").join(&html_filename);
|
||||||
|
if html_dest
|
||||||
|
.components()
|
||||||
|
.any(|c| matches!(c, std::path::Component::ParentDir))
|
||||||
|
{
|
||||||
|
anyhow::bail!("非法的文件写入路径,bibcode 包含路径穿越字符");
|
||||||
|
}
|
||||||
if let Some(parent) = html_dest.parent() {
|
if let Some(parent) = html_dest.parent() {
|
||||||
tokio::fs::create_dir_all(parent)
|
tokio::fs::create_dir_all(parent)
|
||||||
.await
|
.await
|
||||||
|
|||||||
@ -85,10 +85,20 @@ pub async fn search_papers(
|
|||||||
let mut seen_arxiv: HashSet<String> = HashSet::new();
|
let mut seen_arxiv: HashSet<String> = HashSet::new();
|
||||||
|
|
||||||
for r in results {
|
for r in results {
|
||||||
let dup = (!r.bibcode.is_empty() && !seen_bibcodes.insert(r.bibcode.clone()))
|
// 先用引用检查是否重复,仅非重复项才 clone 入 HashSet
|
||||||
|| (!r.doi.is_empty() && !seen_dois.insert(r.doi.clone()))
|
let dup = (!r.bibcode.is_empty() && seen_bibcodes.contains(&r.bibcode))
|
||||||
|| (!r.arxiv_id.is_empty() && !seen_arxiv.insert(r.arxiv_id.clone()));
|
|| (!r.doi.is_empty() && seen_dois.contains(&r.doi))
|
||||||
|
|| (!r.arxiv_id.is_empty() && seen_arxiv.contains(&r.arxiv_id));
|
||||||
if !dup {
|
if !dup {
|
||||||
|
if !r.bibcode.is_empty() {
|
||||||
|
seen_bibcodes.insert(r.bibcode.clone());
|
||||||
|
}
|
||||||
|
if !r.doi.is_empty() {
|
||||||
|
seen_dois.insert(r.doi.clone());
|
||||||
|
}
|
||||||
|
if !r.arxiv_id.is_empty() {
|
||||||
|
seen_arxiv.insert(r.arxiv_id.clone());
|
||||||
|
}
|
||||||
unique_results.push(r);
|
unique_results.push(r);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -199,13 +209,15 @@ async fn batch_fetch_existing_papers(
|
|||||||
/// 会被替换为空格,确保查询作为纯文本搜索而非布尔表达式执行。
|
/// 会被替换为空格,确保查询作为纯文本搜索而非布尔表达式执行。
|
||||||
pub fn sanitize_fts5_query(query: &str) -> String {
|
pub fn sanitize_fts5_query(query: &str) -> String {
|
||||||
query
|
query
|
||||||
.replace('"', "\"\"")
|
.replace(['"', '\''], " ")
|
||||||
.replace('\'', "''")
|
|
||||||
.replace(['*', ':'], " ")
|
.replace(['*', ':'], " ")
|
||||||
.replace(" AND ", " ")
|
.replace(" AND ", " ")
|
||||||
.replace(" OR ", " ")
|
.replace(" OR ", " ")
|
||||||
.replace(" NOT ", " ")
|
.replace(" NOT ", " ")
|
||||||
.replace(" NEAR ", " ")
|
.replace(" NEAR ", " ")
|
||||||
|
.split_whitespace()
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" ")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 本地文献库 FTS5 全文检索。
|
/// 本地文献库 FTS5 全文检索。
|
||||||
|
|||||||
@ -74,23 +74,22 @@ impl Dictionary {
|
|||||||
return Vec::new();
|
return Vec::new();
|
||||||
}
|
}
|
||||||
|
|
||||||
// 基础分词清理:保留字母数字及连接符,其余视为空格以进行精确段落划分
|
// 基础分词清理 + 预计算 lowercase,减少后续重复分配
|
||||||
let clean_text = text
|
let clean_text: String = text
|
||||||
.chars()
|
.chars()
|
||||||
.map(|c| {
|
.map(|c| {
|
||||||
if c.is_alphanumeric() || c == '-' || c == '\'' || c == ' ' {
|
if c.is_alphanumeric() || c == '-' || c == '\'' || c == ' ' {
|
||||||
c
|
c.to_ascii_lowercase()
|
||||||
} else {
|
} else {
|
||||||
' '
|
' '
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.collect::<String>();
|
.collect();
|
||||||
|
|
||||||
let words: Vec<&str> = clean_text.split_whitespace().collect();
|
let words: Vec<&str> = clean_text.split_whitespace().collect();
|
||||||
let mut matched = HashSet::new();
|
let mut matched = HashSet::new();
|
||||||
let mut results = Vec::new();
|
let mut results = Vec::new();
|
||||||
|
|
||||||
// 天文学词条跨度最大限制(一般多词短语不超过 6 个英文单词)
|
|
||||||
let max_span = 6;
|
let max_span = 6;
|
||||||
let n = words.len();
|
let n = words.len();
|
||||||
let mut next_valid_index = 0;
|
let mut next_valid_index = 0;
|
||||||
@ -99,22 +98,19 @@ impl Dictionary {
|
|||||||
if i < next_valid_index {
|
if i < next_valid_index {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// 快速过滤:如果当前单词的 lowercase 不在 first_words 集合中,不可能匹配任何词条
|
// 已 lowercase 的首词直接查 first_words
|
||||||
let first_word_lower = words[i].to_lowercase();
|
if !self.first_words.contains(words[i]) {
|
||||||
if !self.first_words.contains(&first_word_lower) {
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
for len in (1..=max_span).rev() {
|
for len in (1..=max_span).rev() {
|
||||||
if i + len <= n {
|
if i + len <= n {
|
||||||
let phrase_slice = &words[i..i + len];
|
let phrase = words[i..i + len].join(" ");
|
||||||
let phrase = phrase_slice.join(" ").to_lowercase();
|
|
||||||
|
|
||||||
if self.terms.contains_key(&phrase) {
|
if self.terms.contains_key(&phrase) {
|
||||||
// 避免重复匹配更长名词的子词 (如已匹配 'active galactic nucleus' 就不重复提取其中的 'nucleus')
|
|
||||||
if !matched.contains(&phrase) {
|
if !matched.contains(&phrase) {
|
||||||
let chinese = self.terms.get(&phrase).unwrap().clone();
|
let chinese = self.terms.get(&phrase).unwrap().clone();
|
||||||
let original_phrase = phrase_slice.join(" ");
|
let original_phrase = words[i..i + len].join(" ");
|
||||||
|
|
||||||
results.push((original_phrase, chinese));
|
results.push((original_phrase, chinese));
|
||||||
matched.insert(phrase);
|
matched.insert(phrase);
|
||||||
@ -248,7 +244,7 @@ pub async fn translate_markdown(
|
|||||||
|
|
||||||
info!(
|
info!(
|
||||||
"正在请求大模型开展中英翻译。所选大模型: {}",
|
"正在请求大模型开展中英翻译。所选大模型: {}",
|
||||||
llm_client.model()
|
llm_client.model().await
|
||||||
);
|
);
|
||||||
let start_time = std::time::Instant::now();
|
let start_time = std::time::Instant::now();
|
||||||
|
|
||||||
@ -260,7 +256,7 @@ pub async fn translate_markdown(
|
|||||||
let duration = start_time.elapsed();
|
let duration = start_time.elapsed();
|
||||||
info!(
|
info!(
|
||||||
"LLM 翻译成功。所选大模型: {}, 耗时: {:?}, 译文字符数: {}",
|
"LLM 翻译成功。所选大模型: {}, 耗时: {:?}, 译文字符数: {}",
|
||||||
llm_client.model(),
|
llm_client.model().await,
|
||||||
duration,
|
duration,
|
||||||
translated.len()
|
translated.len()
|
||||||
);
|
);
|
||||||
|
|||||||
@ -30,7 +30,7 @@ pub async fn analyze_image_stream_service(
|
|||||||
// 流式输出:先发送状态头部(作为第一条 text_delta,保证持久化到工具结果中)
|
// 流式输出:先发送状态头部(作为第一条 text_delta,保证持久化到工具结果中)
|
||||||
let header = format!(
|
let header = format!(
|
||||||
"> 正在使用视觉模型({})分析图片: {}\n\n",
|
"> 正在使用视觉模型({})分析图片: {}\n\n",
|
||||||
vision_llm.model(),
|
vision_llm.model().await,
|
||||||
image_path
|
image_path
|
||||||
);
|
);
|
||||||
if let Some(tx) = sse_tx {
|
if let Some(tx) = sse_tx {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user