// src/services/observation/cache.rs // // 观测数据缓存层 + 落盘辅助 // // observation_cache 表设计: // - UNIQUE(source, product, source_id):同一源的同一标识符可同时缓存不同产品 // (如 Gaia 同一 source_id 的光谱与光变分别独立缓存) // - artifacts_json:一个逻辑产物可对应多个物理文件(Gaia 光变 G/BP/RP 三波段) use crate::services::observation::types::{Artifact, ProductSpec, Source}; use anyhow::{anyhow, Result}; use serde::{Deserialize, Serialize}; use sqlx::SqlitePool; use std::path::Path; use tracing::error; // ═══════════════════════════════════════════════════════════════ // 1. 缓存命中行类型 // ═══════════════════════════════════════════════════════════════ /// observation_cache 中单条 artifacts 的 JSON 结构 #[derive(Debug, Clone, Serialize, Deserialize)] struct ArtifactRecord { path: String, format: String, #[serde(default)] size: usize, #[serde(default, skip_serializing_if = "Option::is_none")] band: Option, #[serde(default, skip_serializing_if = "Option::is_none")] original_name: Option, } impl From<&ArtifactRecord> for Artifact { fn from(r: &ArtifactRecord) -> Self { Artifact { band: r.band.clone(), original_name: r.original_name.clone(), file_path: r.path.clone(), file_url: crate::services::observation::cache::file_url_from_path(&r.path), file_format: r.format.clone(), size_bytes: r.size, cached: true, } } } /// 缓存查询结果 #[derive(sqlx::FromRow)] struct CacheHitRow { artifacts_json: String, meta_json: Option, } /// observation_cache 列表查询行(对外暴露) #[derive(Debug, Clone, Serialize, sqlx::FromRow)] pub struct ObservationCacheRow { pub source: String, pub product: String, pub source_id: String, pub ra: Option, pub dec: Option, pub artifacts_json: String, pub meta_json: Option, pub created_at: String, } // ═══════════════════════════════════════════════════════════════ // 2. 缓存读写 // ═══════════════════════════════════════════════════════════════ /// 查询单条缓存(按 source + product + source_id),返回命中的 artifacts pub async fn fetch_observation_cache( pool: &SqlitePool, source: Source, product: &ProductSpec, source_id: &str, ) -> Result, Option)>> { let row = sqlx::query_as::<_, CacheHitRow>( "SELECT artifacts_json, meta_json FROM observation_cache \ WHERE source = ? AND product = ? AND source_id = ? LIMIT 1", ) .bind(source.as_str()) .bind(product.product.as_str()) .bind(source_id) .fetch_optional(pool) .await .map_err(|e| anyhow!("查询 observation_cache 失败: {}", e))?; match row { Some(r) => { let records: Vec = serde_json::from_str(&r.artifacts_json) .map_err(|e| anyhow!("反序列化 artifacts_json 失败: {}", e))?; let artifacts: Vec = records.iter().map(Artifact::from).collect(); let meta = r .meta_json .as_deref() .and_then(|s| serde_json::from_str(s).ok()); Ok(Some((artifacts, meta))) } None => Ok(None), } } /// 写入/更新缓存条目 pub async fn write_observation_cache( pool: &SqlitePool, source: Source, product: &ProductSpec, source_id: &str, ra: Option, dec: Option, artifacts: &[Artifact], meta_json: Option<&str>, ) -> Result<()> { let records: Vec = artifacts .iter() .map(|a| ArtifactRecord { path: a.file_path.clone(), format: a.file_format.clone(), size: a.size_bytes, band: a.band.clone(), original_name: a.original_name.clone(), }) .collect(); let artifacts_json = serde_json::to_string(&records)?; sqlx::query( "INSERT OR REPLACE INTO observation_cache \ (source, product, source_id, ra, dec, artifacts_json, meta_json, created_at) \ VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)", ) .bind(source.as_str()) .bind(product.product.as_str()) .bind(source_id) .bind(ra) .bind(dec) .bind(&artifacts_json) .bind(meta_json) .execute(pool) .await .map_err(|e| anyhow!("写入 observation_cache 失败: {}", e))?; Ok(()) } /// 列出某源某产品的全部已缓存产物 pub async fn list_cached( pool: &SqlitePool, source: Source, product: &ProductSpec, ) -> Result> { let rows = sqlx::query_as::<_, ObservationCacheRow>( "SELECT source, product, source_id, ra, dec, artifacts_json, meta_json, created_at \ FROM observation_cache WHERE source = ? AND product = ? \ ORDER BY created_at DESC LIMIT 500", ) .bind(source.as_str()) .bind(product.product.as_str()) .fetch_all(pool) .await .map_err(|e| anyhow!("查询 observation_cache 列表失败: {}", e))?; Ok(rows) } /// 列出全部已缓存产物(不限源/产品) pub async fn list_all_cached(pool: &SqlitePool) -> Result> { let rows = sqlx::query_as::<_, ObservationCacheRow>( "SELECT source, product, source_id, ra, dec, artifacts_json, meta_json, created_at \ FROM observation_cache ORDER BY created_at DESC LIMIT 500", ) .fetch_all(pool) .await .map_err(|e| anyhow!("查询 observation_cache 列表失败: {}", e))?; Ok(rows) } /// 缓存库列表筛选 + 排序参数(服务端分页) #[derive(Debug, Clone, Default)] pub struct ListCachedParams { /// 按数据源过滤(lamost/gaia/sdss/desi) pub source: Option, /// 按产品类型过滤(spectrum/lightcurve/photometry/image) pub product: Option, /// source_id 模糊匹配(LIKE %q%) pub q: Option, /// 排序:created(默认,DESC)/ source / product pub sort: Option, /// 每页条数(默认 50,上限 200) pub limit: Option, /// 偏移量 pub offset: Option, } /// 缓存库分页查询结果 #[derive(Debug, Clone, Serialize)] pub struct CachedPage { pub items: Vec, pub total: i64, } /// 列出已缓存产物(服务端分页 + 筛选 + 排序) pub async fn list_cached_paged(pool: &SqlitePool, params: &ListCachedParams) -> Result { let limit = params.limit.unwrap_or(50).clamp(1, 200); let offset = params.offset.unwrap_or(0).max(0); // 动态构造 SQL(参数绑定防注入;列名/排序方向用白名单) let mut where_clauses: Vec = Vec::new(); if let Some(s) = ¶ms.source { where_clauses.push(format!("source = '{}'", s.replace('\'', "''"))); } if let Some(p) = ¶ms.product { where_clauses.push(format!("product = '{}'", p.replace('\'', "''"))); } if let Some(q) = ¶ms.q { let q = q.replace('\'', "''"); where_clauses.push(format!("source_id LIKE '%{}%'", q)); } let where_sql = if where_clauses.is_empty() { String::new() } else { format!(" WHERE {}", where_clauses.join(" AND ")) }; let order_sql = match params.sort.as_deref() { Some("source") => " ORDER BY source ASC, created_at DESC", Some("product") => " ORDER BY product ASC, created_at DESC", _ => " ORDER BY created_at DESC", }; // count let count_sql = format!("SELECT COUNT(*) as cnt FROM observation_cache{}", where_sql); let total: i64 = sqlx::query_scalar(&count_sql) .fetch_one(pool) .await .map_err(|e| anyhow!("统计 observation_cache 总数失败: {}", e))?; // items let items_sql = format!( "SELECT source, product, source_id, ra, dec, artifacts_json, meta_json, created_at \ FROM observation_cache{}{} LIMIT ? OFFSET ?", where_sql, order_sql ); let items = sqlx::query_as::<_, ObservationCacheRow>(&items_sql) .bind(limit) .bind(offset) .fetch_all(pool) .await .map_err(|e| anyhow!("查询 observation_cache 分页列表失败: {}", e))?; Ok(CachedPage { items, total }) } // ═══════════════════════════════════════════════════════════════ // 3. 落盘 + URL 辅助 // ═══════════════════════════════════════════════════════════════ /// 经 /api/files 前缀生成访问 URL pub fn file_url_from_path(rel_path: &str) -> String { format!("/api/files/{}", rel_path) } /// 落盘字节到 library_dir/,返回字节数 pub fn persist_bytes(library_dir: &Path, rel_path: &str, bytes: &[u8]) -> Result { let abs_path = library_dir.join(rel_path); if let Some(parent) = abs_path.parent() { std::fs::create_dir_all(parent) .map_err(|e| anyhow!("创建观测数据目录失败 {:?}: {}", parent, e))?; } std::fs::write(&abs_path, bytes) .map_err(|e| anyhow!("写入观测数据文件失败 {:?}: {}", abs_path, e))?; Ok(bytes.len()) } /// 校验缓存产物中至少有一个文件真实存在;存在则返回其总大小 pub fn cached_files_total_size(library_dir: &Path, artifacts: &[Artifact]) -> Option { let mut total = 0; for a in artifacts { let abs_path = library_dir.join(&a.file_path); match std::fs::metadata(&abs_path) { Ok(m) => total += m.len() as usize, Err(_) => { // 任一文件缺失即视为缓存不完整 return None; } } } if total == 0 { None } else { Some(total) } } /// 记录缓存写入失败但不中断流程 pub fn log_write_failure(source: &str, product: &str, e: anyhow::Error) { error!("[Observation] 写入 observation_cache 失败 ({}/{}): {}", source, product, e); } #[cfg(test)] mod tests { use super::*; #[test] fn test_file_url_from_path() { assert_eq!( file_url_from_path("Telescope/lamost/spectrum/123.fits"), "/api/files/Telescope/lamost/spectrum/123.fits" ); } #[test] fn test_artifact_record_roundtrip() { let rec = ArtifactRecord { path: "Telescope/gaia/lightcurve/123/123_G.fits".into(), format: "fits".into(), size: 1024, band: Some("G".into()), original_name: Some("123_EPOCH_PHOTOMETRY_G.fits".into()), }; let json = serde_json::to_string(&rec).unwrap(); let back: ArtifactRecord = serde_json::from_str(&json).unwrap(); assert_eq!(back.path, rec.path); assert_eq!(back.band.as_deref(), Some("G")); } }