fmq
f2700031ce
收敛攻坚(81/400 失败点根因与实测,见 docs/failed81_cno_seed_popzer_dpsilg_2026_08_18.md): - runner: 新增 seed_step_stab 策略链——同物理族(同 Teff/logg/logHe)CNO 邻居种子 + DPSILG=3.0 λ 算子欠松弛 + POPZER=1E-10 微布居置零(联动 POPZR2/RADZER 同值 + NITZER=1),针对 20kK He 富大气 He I/II 电离前沿布居极限环 - runner: 三级链内延拓回退(主链与 nl_direct 全败后自动触发): · 固定 ladder 步进——plan_ladder_steps 按归一化间隔选轴,Δlogg≤0.25/ΔTeff≤2.5kK,≤4 步 · 自适应 Teff 延拓——步长 1250K 起、成功 ×1.5 恢复、失败二分至 25K 折叠墙,≤48 步 · C/N 丰度轴延拓——高温域(Teff>30kK)专用,严格同族种子沿 C(优先)/N 轴 0.2 dex 起步 延拓阶段 NITER 下限提至 300(慢收敛 waypoint 迭代饥饿误判修复) - runner: 稳定化多档回退(DPSILG/POPZER 三档互补,联合回收 41/65); 域门控 Teff≤30kK——高温高金属域实测旋钮致散(17 拍爆至 4e16),域外跳过 - 梯级种子持久化: 收敛中间模型登记 ladder_seeds 随上报落 server seeds 表 (任务失败也上传,合成名 _ladder 与真实网格点零冲突),簇内相邻失败点自动复用 调度与执行: - scheduler: 策略解析新增 seed_step_stab 臂——find_exact_family_seed_from_db 严格 同物理族判定(不做 global 退化,防 ladder 中间种子 ΔTeff≤5000K 误命中), 排除本点历史已用种子实现重试轮换;链在 stab 耗尽时轮换未试过同族邻居重派 - executor: seed_step_stab 补种子下载(漏列曾致 78 任务假失败,seed_nc 无 fort.8 崩溃); Teff>30kK 域外自动降级普通种子链 收敛判据: - conv_check: 热启动豁免——首拍 max_relc<1(种子已近解)时首末比 1e3 判据数学上 不可达,豁免后交五重物理硬门槛裁决(修复 nl_ladder 0.038→6e-4 物理全过被误杀); 冷启动仍受判据门控 workflow 生命周期: - workflows/tasks: 完成 flip 增加「未消费回退链」阻塞子句——failed 点策略链未耗尽 或链尾 seed_step_stab 尚有未试过同族种子时不得置 completed(修复最后活跃点 cold 失败上报抢先 flip、still_running 守卫拦截后续策略永不派发);按 failed_stage 归因 (synspec 失败行只看 synspec 链,防 stale 审计副本永久卡死)+ json_valid 脏行防护 统计与前端: - 统计新增权威口径 tlusty_converged(不按策略拆)与 seed_step_stab_converged 分项, 前端详情页色带/概览卡消费权威总数并新增稳定化青色段(修复 stab 收敛点漏计, 生产 9137/9216 差额);M14 迁移回填历史 completed 点的 tlusty_status 文档: - 新增 failed81 POPZER/DPSILG 制胜配方根因分析、Windows 节点经跳板 RDP 运维手册; failed400 增补 ladder 生产化实现与第二轮 121 残点实测矩阵
4780 lines
189 KiB
Rust
4780 lines
189 KiB
Rust
use crate::migrations;
|
||
use anyhow::{Context, Result};
|
||
use common::models::{
|
||
GridAxisValue, GridPointParams, GridPointStatus, NodeHeartbeatRequest, NodeInfo,
|
||
NodeRegisterRequest, ResumePolicy, TaskReport, TaskStatus,
|
||
};
|
||
use r2d2::Pool;
|
||
use r2d2_sqlite::SqliteConnectionManager;
|
||
use rusqlite::params;
|
||
use sha2::{Digest, Sha256};
|
||
use tracing::info;
|
||
|
||
/// 策略链弹栈快照（`pop_stage_strategy_for_fallback` 的返回值）。
|
||
///
|
||
/// 携带**派发时落库**的策略链与 policy（同一行快照），供回退决策与重试任务构造使用：
|
||
/// - `rest_strategies`：弹出失败首项后的剩余链；
|
||
/// - `popped`：被弹出的失败策略名；
|
||
/// - `policy`：失败阶段当时的执行策略（对齐设计 §4.2「不修改原有 policy，保持用户初始
|
||
/// 配置」——回退行为由派发时快照决定，不随运行期 YAML 编辑漂移）。
|
||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
pub struct FallbackSnapshot {
|
||
pub rest_strategies: Vec<String>,
|
||
pub popped: String,
|
||
pub policy: ResumePolicy,
|
||
}
|
||
|
||
/// 空/缺省的 workflow_name 归一到 legacy 兜底标记 `__legacy__`（H1 修复）。
|
||
///
|
||
/// 多工作流迁移把主库历史 grid_points/tasks 行回填为 `'__legacy__'`，但旧版在途任务
|
||
/// 的队列 payload 没有 workflow_name 字段（None）。若 claim/report/requeue 重置按空串
|
||
/// 定向更新主库，`WHERE workflow_name=''` 永远命中 0 行——旧任务正常结算但网格点
|
||
/// 永久卡死在 running/queued（无任何回收路径）。统一在此归一到 `'__legacy__'`，
|
||
/// 使这三个入口都能命中迁移后的 legacy 网格点行。
|
||
pub fn normalize_workflow_name(wf: Option<&str>) -> String {
|
||
match wf {
|
||
Some(w) if !w.is_empty() => w.to_string(),
|
||
_ => "__legacy__".to_string(),
|
||
}
|
||
}
|
||
|
||
/// grid_points 表的 CREATE 语句（单一真相源）。
|
||
///
|
||
/// 审查修复 #S4：原 init_tables 与 migrate_grid_points_for_workflow_partition 各持一份
|
||
/// CREATE TABLE 字面量，未来加列时极易漏改迁移函数那份，导致旧库迁移后缺列、直到某条
|
||
/// ALTER 触发才补上。提取为常量后两处复用同一份 schema 定义。
|
||
const GRID_POINTS_SCHEMA: &str = "CREATE TABLE grid_points (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
name TEXT NOT NULL,
|
||
workflow_name TEXT NOT NULL,
|
||
teff REAL NOT NULL,
|
||
logg REAL NOT NULL,
|
||
loghe REAL NOT NULL,
|
||
logc REAL NOT NULL,
|
||
logn REAL NOT NULL,
|
||
logo REAL NOT NULL,
|
||
cno_sum REAL NOT NULL,
|
||
wave INTEGER NOT NULL DEFAULT 0,
|
||
status TEXT NOT NULL DEFAULT 'pending',
|
||
attempt_count INTEGER NOT NULL DEFAULT 0,
|
||
-- P9（命名拆分）：阶段收敛归因列——TLUSTY 阶段以何策略收敛（cold_run/seed_step/策略名；
|
||
-- TLUSTY 禁用为 NULL）。整体归因由消费方派生（tlusty ?? synspec）。
|
||
tlusty_success_method TEXT,
|
||
-- P6（Phase 5a）：光谱收敛归因列（synspec 以什么策略收敛；TLUSTY-only 为 NULL）。
|
||
synspec_success_method TEXT,
|
||
-- P6（Phase 5b）：阶段状态列（NULL = 阶段不适用）。
|
||
-- 半失败点可查 tlusty_status='converged' + synspec_status='failed'。
|
||
tlusty_status TEXT,
|
||
synspec_status TEXT,
|
||
-- H1 修复（M11）：运行时回退把点打回 pending 时记录的「剩余策略链」（JSON 数组）。
|
||
-- 供调度路径识别「该点已失败过 cold_run、正在等种子」→ 重派时用剩余链而非完整 YAML 链，
|
||
-- 避免重跑已失败策略导致的无界失败重试活锁（见 scheduler.rs H1 注释）。NULL = 无标记。
|
||
pending_strategies TEXT,
|
||
-- 最近一次尝试的真实墙钟耗时（秒，Worker 回报值；旧数据为 None）。
|
||
last_elapsed_sec REAL,
|
||
-- 点级诊断快照（完整 ModelSummary JSON）。详情页 conv 诊断面板的数据源。
|
||
-- synspec-only 重跑时经 merge_point_summary 增量合并，保留 TLUSTY 字段不丢失。
|
||
summary_json TEXT
|
||
)";
|
||
|
||
#[derive(Debug)]
|
||
struct SqliteCustomizer;
|
||
|
||
/// 计算 token 的 SHA-256 hex hash。凭据表只存 hash，不存明文 token。
|
||
fn hash_token(token: &str) -> String {
|
||
let mut hasher = Sha256::new();
|
||
hasher.update(token.as_bytes());
|
||
hex::encode(hasher.finalize())
|
||
}
|
||
|
||
/// 恒定时间比对两个非空字符串（先 SHA-256 摘要再比较等长摘要，消除长度时序旁路）。
|
||
/// 用于 registration_secret 校验，避免通过比对耗时探得 secret 前缀。
|
||
fn ct_eq_option(a: &str, b: &str) -> bool {
|
||
use subtle::ConstantTimeEq;
|
||
let ha = {
|
||
let mut h = Sha256::new();
|
||
h.update(a.as_bytes());
|
||
h.finalize()
|
||
};
|
||
let hb = {
|
||
let mut h = Sha256::new();
|
||
h.update(b.as_bytes());
|
||
h.finalize()
|
||
};
|
||
ha.ct_eq(&hb).into()
|
||
}
|
||
|
||
/// 多工作流分区迁移：把旧版 grid_points 表（仅 name UNIQUE，无 workflow_name 列）
|
||
/// 重建为带 workflow_name 列、(workflow_name, name) 复合唯一的新结构。
|
||
///
|
||
/// 幂等：新库（CREATE TABLE 已含 workflow_name）经 PRAGMA 检测后直接跳过。
|
||
/// 旧库重建步骤：
|
||
/// 1. 把旧表重命名为 grid_points_legacy；
|
||
/// 2. 重建 grid_points（新 schema，已由 CREATE TABLE IF NOT EXISTS 建好——这里需先 DROP 再建）；
|
||
/// 3. 从 legacy 复制数据，workflow_name 回填 '__legacy__' 兜底；
|
||
/// 4. 删除 legacy 表。
|
||
///
|
||
/// 兜底标记 '__legacy__' 的意义：新工作流的查询恒带 WHERE workflow_name=<真实名>，
|
||
/// 不会命中 '__legacy__' 行；这些历史残留行既不干扰新调度，也保留下来供人工排查。
|
||
fn migrate_grid_points_for_workflow_partition(conn: &mut rusqlite::Connection) -> Result<()> {
|
||
// 若已是新结构（含 workflow_name 列），无需迁移。
|
||
if has_grid_points_column(conn, "workflow_name")? {
|
||
return Ok(());
|
||
}
|
||
|
||
tracing::info!("检测到旧版 grid_points 表（无 workflow_name 列），执行多工作流分区迁移...");
|
||
|
||
let tx = conn.transaction()?;
|
||
|
||
// 兼容：若存在遗留的迁移中间表（上次迁移被中断），先清理。
|
||
tx.execute("DROP TABLE IF EXISTS grid_points_legacy", [])?;
|
||
|
||
// 旧表改名 → 重建新表（按最新 CREATE TABLE 形态，复用 GRID_POINTS_SCHEMA 单一真相源）→ 回填数据 → 删旧表
|
||
tx.execute("ALTER TABLE grid_points RENAME TO grid_points_legacy", [])?;
|
||
tx.execute(GRID_POINTS_SCHEMA, [])?;
|
||
// 历史数据 workflow_name 兜底为 '__legacy__'，不污染新工作流查询。
|
||
tx.execute(
|
||
"INSERT INTO grid_points (name, workflow_name, teff, logg, loghe, logc, logn, logo, cno_sum, wave, status, attempt_count, tlusty_success_method)
|
||
SELECT name, '__legacy__', teff, logg, loghe, logc, logn, logo, cno_sum, wave, status, attempt_count, success_method
|
||
FROM grid_points_legacy",
|
||
[],
|
||
)?;
|
||
tx.execute("DROP TABLE grid_points_legacy", [])?;
|
||
|
||
tx.commit()?;
|
||
|
||
tracing::info!("grid_points 多工作流分区迁移完成，历史数据 workflow_name 标记为 '__legacy__'");
|
||
Ok(())
|
||
}
|
||
|
||
/// 检测 grid_points 表是否已含指定列（基于 PRAGMA table_info，与 sqlite_queue.rs 的迁移惯用法一致）。
|
||
fn has_grid_points_column(conn: &rusqlite::Connection, col: &str) -> Result<bool> {
|
||
let mut stmt = conn.prepare("PRAGMA table_info(grid_points)")?;
|
||
let rows = stmt.query_map([], |r| r.get::<_, String>(1))?;
|
||
for r in rows {
|
||
if r.map(|name| name == col).unwrap_or(false) {
|
||
return Ok(true);
|
||
}
|
||
}
|
||
Ok(false)
|
||
}
|
||
|
||
/// 将 SQLite db 文件及其 WAL/SHM 侧车文件权限收紧为 0600（仅 owner 读写）。
|
||
/// 文件不存在或设置失败时静默忽略（不阻断启动，仅作加固）。
|
||
#[cfg(unix)]
|
||
fn restrict_db_file_perms(db_path: &str) {
|
||
use std::os::unix::fs::PermissionsExt;
|
||
let candidates = [
|
||
std::path::PathBuf::from(db_path),
|
||
std::path::PathBuf::from(format!("{}-wal", db_path)),
|
||
std::path::PathBuf::from(format!("{}-shm", db_path)),
|
||
];
|
||
for p in candidates {
|
||
if let Ok(meta) = std::fs::metadata(&p) {
|
||
let mut perms = meta.permissions();
|
||
perms.set_mode(0o600);
|
||
let _ = std::fs::set_permissions(&p, perms);
|
||
}
|
||
}
|
||
}
|
||
|
||
impl r2d2::CustomizeConnection<rusqlite::Connection, rusqlite::Error> for SqliteCustomizer {
|
||
fn on_acquire(&self, conn: &mut rusqlite::Connection) -> Result<(), rusqlite::Error> {
|
||
// 多节点心跳/claim + 后台调度 + dashboard 查询并发时，5s 易触发 SQLITE_BUSY 直接 bail。
|
||
// 调大到 15s 给重试足够窗口，配合 IMMEDIATE 事务退避。
|
||
conn.pragma_update(None, "busy_timeout", 15000)?;
|
||
conn.pragma_update(None, "wal_autocheckpoint", 1000)?;
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
pub struct SeedCacheItem {
|
||
pub point_name: String,
|
||
pub params: GridPointParams,
|
||
pub file_path: String,
|
||
}
|
||
|
||
/// exact_family 种子索引的桶键。
|
||
///
|
||
/// exact_family 判定（seed_finder.rs）：`d_teff < 5000 && d_logg < 0.01 && d_loghe < 0.01`。
|
||
/// 把这三个轴量化到桶。由于 exact 要求「双侧」严格小于阈值（target 和候选两侧都可能在
|
||
/// 量化边界两侧），对每个轴都做 **floor / floor+1 双桶** 写入与查询，确保跨越量化边界的
|
||
/// 真实 exact 候选必被覆盖：
|
||
/// - teff 按 5000K 量化为整数（floor），查 floor 与 floor+1 两桶覆盖 [floor*5000, (floor+2)*5000)。
|
||
/// - logg/loghe 按 0.01 精度量化（×100 后 floor），查 floor 与 floor+1。
|
||
///
|
||
/// 旧实现仅对 teff 双写，logg/loghe 用 round 单桶，导致 d_logg<0.01 但 *100 round 落在相邻
|
||
/// 整数的两点（如 5.004→500 vs 5.005→501）被分到不同桶、永不相遇——exact 候选被静默丢失，
|
||
/// 且全局回退扫描又显式跳过 exact 候选，无法补救。改为三个轴一致地双写双查后消除该边界 Bug。
|
||
///
|
||
/// 命中桶后仍在桶内做精确 distance 计算取最优，故量化只用于缩小候选集，不影响正确性。
|
||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||
struct SeedBucketKey {
|
||
teff_bucket: i64,
|
||
logg_q: i64,
|
||
loghe_q: i64,
|
||
}
|
||
|
||
impl SeedBucketKey {
|
||
/// 返回该参数应落入的全部桶键（每个轴的 floor 与 floor+1 笛卡尔积，共 8 个）。
|
||
/// 插入时对每个键写入，查询时对每个键查询，确保跨量化边界的 exact 候选必命中。
|
||
fn from_params(params: &GridPointParams) -> Vec<Self> {
|
||
let teff_floor = (params.teff.value() / 5000.0).floor() as i64;
|
||
let logg_floor = (params.logg.value() * 100.0).floor() as i64;
|
||
let loghe_floor = (params.loghe.value() * 100.0).floor() as i64;
|
||
let mut keys = Vec::with_capacity(8);
|
||
for dt in [0, 1] {
|
||
for dg in [0, 1] {
|
||
for dh in [0, 1] {
|
||
keys.push(SeedBucketKey {
|
||
teff_bucket: teff_floor + dt,
|
||
logg_q: logg_floor + dg,
|
||
loghe_q: loghe_floor + dh,
|
||
});
|
||
}
|
||
}
|
||
}
|
||
keys
|
||
}
|
||
}
|
||
|
||
#[derive(Clone)]
|
||
pub struct Database {
|
||
/// crate 内可见：供同 crate 测试以裸 SQL 构造夹具（如回拨 tasks.created_at）。
|
||
pub(crate) pool: Pool<SqliteConnectionManager>,
|
||
seed_cache: std::sync::Arc<tokio::sync::RwLock<Vec<SeedCacheItem>>>,
|
||
/// exact_family 快速索引：桶键 → 该桶全部种子。命中 exact_family 的查询走 O(1)~O(小)，
|
||
/// 未命中才退化到 seed_cache 全量 global 扫描。
|
||
seed_index: std::sync::Arc<
|
||
tokio::sync::RwLock<std::collections::HashMap<SeedBucketKey, Vec<SeedCacheItem>>>,
|
||
>,
|
||
/// node token 反查缓存：token_hash → (node_id, 插入时间, 回填时的缓存 generation)。
|
||
/// 鉴权中间件每个 Node 请求都查 find_node_by_token，此缓存把高频心跳/领用请求
|
||
/// 的 DB 查询降为内存读。TTL 由 `TOKEN_CACHE_TTL` 控制；issue（重发）时整体失效。
|
||
///
|
||
/// cache_generation 是单调递增的"失效代次"：每次 invalidate_token_cache 自增。
|
||
/// find_node_by_token 在 DB 查询前记录当前 generation，回填时若 generation 已变化
|
||
/// （说明期间发生过 reissue 导致的 invalidate），则丢弃本次回填，彻底消除
|
||
/// "旧 token_hash 复活"的 TOCTOU 窗口（旧实现仅缩小窗口、未消除）。
|
||
token_cache: std::sync::Arc<tokio::sync::RwLock<TokenCache>>,
|
||
}
|
||
|
||
/// token 反查缓存的单条存活时长（秒）。issue（重发）会立即整体失效，TTL 仅兜底。
|
||
const TOKEN_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(60);
|
||
|
||
/// token 反查缓存的内部结构：entries 表 + 单调递增的失效代次。
|
||
struct TokenCache {
|
||
entries: std::collections::HashMap<String, (String, std::time::Instant)>,
|
||
/// 每次 invalidate_token_cache 自增；find_node_by_token 回填时据此判断是否发生过失效。
|
||
generation: u64,
|
||
}
|
||
|
||
impl TokenCache {
|
||
fn new() -> Self {
|
||
Self {
|
||
entries: std::collections::HashMap::new(),
|
||
generation: 0,
|
||
}
|
||
}
|
||
}
|
||
|
||
mod grid;
|
||
mod nodes;
|
||
mod seeds;
|
||
mod snapshots;
|
||
mod tasks;
|
||
mod workflows;
|
||
|
||
impl Database {
|
||
pub async fn new(db_path: &str) -> Result<Self> {
|
||
let db_path_owned = db_path.to_string();
|
||
let pool = tokio::task::spawn_blocking(move || -> Result<Pool<SqliteConnectionManager>> {
|
||
if let Some(parent) = std::path::Path::new(&db_path_owned).parent() {
|
||
let _ = std::fs::create_dir_all(parent);
|
||
}
|
||
let manager = SqliteConnectionManager::file(&db_path_owned);
|
||
let pool = Pool::builder()
|
||
.max_size(16)
|
||
.connection_customizer(Box::new(SqliteCustomizer))
|
||
.build(manager)
|
||
.context("Failed to build SQLite main DB connection pool")?;
|
||
|
||
// 收紧 db 文件权限为 0600（仅 owner 读写），防止裸机部署时其他用户读取节点/凭据信息。
|
||
// 容器内以非 root 运行，此设置仅作加固；WAL/SHM 侧车文件一并处理。
|
||
#[cfg(unix)]
|
||
restrict_db_file_perms(&db_path_owned);
|
||
|
||
Ok(pool)
|
||
})
|
||
.await??;
|
||
|
||
let db = Self {
|
||
pool,
|
||
seed_cache: std::sync::Arc::new(tokio::sync::RwLock::new(Vec::new())),
|
||
seed_index: std::sync::Arc::new(tokio::sync::RwLock::new(
|
||
std::collections::HashMap::new(),
|
||
)),
|
||
token_cache: std::sync::Arc::new(tokio::sync::RwLock::new(TokenCache::new())),
|
||
};
|
||
db.init_tables().await?;
|
||
db.apply_migrations().await?;
|
||
db.reload_seed_cache().await?;
|
||
Ok(db)
|
||
}
|
||
|
||
/// Phase 0：应用版本化迁移（docs/database_refactor_design.md §2）。
|
||
/// init_tables bootstrap 后调用：全新库上所有迁移经 detect 守卫跳过（仅推进版本号），
|
||
/// 旧库只补缺的列/索引。每个迁移独立事务，中途失败不推进版本，进程启动时重试。
|
||
async fn apply_migrations(&self) -> Result<()> {
|
||
let pool = self.pool.clone();
|
||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||
let mut conn = pool
|
||
.get()
|
||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||
migrations::apply_migrations(&mut conn)
|
||
})
|
||
.await??;
|
||
Ok(())
|
||
}
|
||
|
||
async fn init_tables(&self) -> Result<()> {
|
||
let pool = self.pool.clone();
|
||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||
let mut conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||
let _: String = conn.pragma_update_and_check(None, "journal_mode", "WAL", |r| r.get(0))?;
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS nodes (
|
||
node_id TEXT PRIMARY KEY,
|
||
max_slots INTEGER NOT NULL,
|
||
active_slots INTEGER NOT NULL DEFAULT 0,
|
||
status TEXT NOT NULL DEFAULT 'online',
|
||
cpu_usage REAL NOT NULL DEFAULT 0.0,
|
||
memory_usage REAL NOT NULL DEFAULT 0.0,
|
||
last_heartbeat DATETIME NOT NULL,
|
||
-- registration_secret：审批前一次性凭据（注册时下发、取走专属 token 前消费）。
|
||
-- 与 node_credentials.token_hash 分居两表是有意为之（token_hash NOT NULL +
|
||
-- 唯一索引决定其只能存 nodes，审批前无 credentials 行）。不迁移，Phase 4 决策
|
||
-- （见 docs/database_refactor_design.md §6）。
|
||
registration_secret TEXT
|
||
);",
|
||
[],
|
||
)?;
|
||
// 旧库迁移：为 nodes 表补 registration_secret 列（H8：check_status 取 token 需此凭据）。
|
||
let has_reg_secret = conn
|
||
.prepare("PRAGMA table_info(nodes)")?
|
||
.query_map([], |r| r.get::<_, String>(1))?
|
||
.any(|r| r.map(|n| n == "registration_secret").unwrap_or(false));
|
||
if !has_reg_secret {
|
||
let _ = conn.execute("ALTER TABLE nodes ADD COLUMN registration_secret TEXT", []);
|
||
}
|
||
// 动态 CPU 槽位配额（见 docs/dynamic_cpu_slots_design.md）：管理员强制并发上限。
|
||
// 幂等追加列，缺列才 ALTER。None/NULL 表示无限制（沿用节点物理 max_slots）。
|
||
let has_admin_slots = conn
|
||
.prepare("PRAGMA table_info(nodes)")?
|
||
.query_map([], |r| r.get::<_, String>(1))?
|
||
.any(|r| r.map(|n| n == "admin_max_slots").unwrap_or(false));
|
||
if !has_admin_slots {
|
||
let _ = conn.execute("ALTER TABLE nodes ADD COLUMN admin_max_slots INTEGER", []);
|
||
}
|
||
|
||
// 复用 GRID_POINTS_SCHEMA（单一真相源，审查修复 #S4）；init_tables 需要
|
||
// IF NOT EXISTS 语义（新库 bootstrap），故把 schema 常量首行替换为带 IF NOT EXISTS。
|
||
let grid_points_create_if_not_exists =
|
||
GRID_POINTS_SCHEMA.replacen("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1);
|
||
conn.execute(&grid_points_create_if_not_exists, [])?;
|
||
// 多工作流分区迁移：旧库的 grid_points 表只有 name UNIQUE（无 workflow_name），
|
||
// 无法支撑「同一物理点属于多个工作流」。SQLite 不能原地删除 CREATE TABLE 内联的 UNIQUE
|
||
// 约束，故用 PRAGMA table_info 检测旧表形态：若缺 workflow_name 列，则重建表为
|
||
// (workflow_name, name) 复合唯一。旧数据 workflow_name 回填为 '__legacy__' 兜底，
|
||
// 避免新工作流查询 WHERE workflow_name=? 误命中历史残留行。
|
||
migrate_grid_points_for_workflow_partition(&mut conn)?;
|
||
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS tasks (
|
||
task_id TEXT PRIMARY KEY,
|
||
point_name TEXT NOT NULL,
|
||
node_id TEXT,
|
||
seed_point_name TEXT,
|
||
status TEXT NOT NULL DEFAULT 'pending',
|
||
max_relc REAL,
|
||
atmosphere_has_nan BOOLEAN NOT NULL DEFAULT 0,
|
||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||
created_at DATETIME NOT NULL,
|
||
started_at DATETIME,
|
||
completed_at DATETIME,
|
||
error_message TEXT,
|
||
workflow_name TEXT,
|
||
tlusty_enabled BOOLEAN NOT NULL DEFAULT 1,
|
||
tlusty_policy TEXT NOT NULL DEFAULT 'skip_converged',
|
||
tlusty_strategies TEXT NOT NULL DEFAULT '[\"cold_run\"]',
|
||
synspec_enabled BOOLEAN NOT NULL DEFAULT 1,
|
||
synspec_policy TEXT NOT NULL DEFAULT 'skip_converged',
|
||
synspec_strategies TEXT NOT NULL DEFAULT '[\"standard\"]',
|
||
atmosphere_ref TEXT,
|
||
-- P1（Phase 1）：阶段结果补全（failed_stage 归因 + summary_json 全量）。
|
||
-- 最新形态列：新库由本 CREATE 建立，旧库经 M1 迁移补齐。
|
||
failed_stage TEXT,
|
||
summary_json TEXT
|
||
);",
|
||
[],
|
||
)?;
|
||
let has_tasks_wf = conn
|
||
.prepare("PRAGMA table_info(tasks)")?
|
||
.query_map([], |r| r.get::<_, String>(1))?
|
||
.any(|r| r.map(|n| n == "workflow_name").unwrap_or(false));
|
||
if !has_tasks_wf {
|
||
let _ = conn.execute("ALTER TABLE tasks ADD COLUMN workflow_name TEXT", []);
|
||
}
|
||
// P3 耗时落库：tasks.elapsed_sec = 单次尝试真实墙钟（Worker 回报携带，
|
||
// 旧版收到即丢弃）；grid_points.last_elapsed_sec = 最近一次尝试耗时（列表展示用）。
|
||
// 幂等追加列，与上方 workflow_name 迁移同模式（PRAGMA 检测 → 缺列才 ALTER）。
|
||
let has_tasks_elapsed = conn
|
||
.prepare("PRAGMA table_info(tasks)")?
|
||
.query_map([], |r| r.get::<_, String>(1))?
|
||
.any(|r| r.map(|n| n == "elapsed_sec").unwrap_or(false));
|
||
if !has_tasks_elapsed {
|
||
let _ = conn.execute("ALTER TABLE tasks ADD COLUMN elapsed_sec REAL", []);
|
||
}
|
||
// grid_points 幂等补列：last_elapsed_sec（列表展示用）与 summary_json（点级诊断快照）。
|
||
// 新库由 GRID_POINTS_SCHEMA 建表时即含此二列；此 ALTER 仅兜底旧库（表已存在但缺列）。
|
||
// 与上方 tasks 列迁移同模式：PRAGMA 检测 → 缺列才 ALTER。
|
||
for (col, sql) in [
|
||
("last_elapsed_sec", "ALTER TABLE grid_points ADD COLUMN last_elapsed_sec REAL"),
|
||
("summary_json", "ALTER TABLE grid_points ADD COLUMN summary_json TEXT"),
|
||
] {
|
||
let has_col = conn
|
||
.prepare("PRAGMA table_info(grid_points)")?
|
||
.query_map([], |r| r.get::<_, String>(1))?
|
||
.any(|r| r.map(|n| n == col).unwrap_or(false));
|
||
if !has_col {
|
||
let _ = conn.execute(sql, []);
|
||
}
|
||
}
|
||
|
||
// 阶段独立配置迁移（见 docs/task_engine_decoupling_design.md §4.1）：
|
||
// 为 tasks 表追加 TLUSTY/SYNSPEC 阶段控制列。幂等：PRAGMA 检测缺列才 ALTER。
|
||
// 旧库的旧行这些列获得 DEFAULT 值（enabled=1, policy=skip_converged,
|
||
// strategies 默认链）。
|
||
// 注（审查 #6）：`tlusty_strategies` DB 默认 `["cold_run"]` 与
|
||
// `PhaseConfig::default_tlusty()`（内存默认 `[cold_run, seed_step]`）不同
|
||
// 是**刻意为之**——DB 默认只兜底旧库遗留行（INSERT 未显式带 strategies 的
|
||
// 历史兼容路径），按设计文档 §4.1 取 `["cold_run"]`；调度器每次派发都会写入
|
||
// 完整解析链，正常路径不受 DB 默认影响。两处各按文档口径保持一致。
|
||
for (col, sql) in [
|
||
("tlusty_enabled", "ALTER TABLE tasks ADD COLUMN tlusty_enabled BOOLEAN NOT NULL DEFAULT 1"),
|
||
("tlusty_policy", "ALTER TABLE tasks ADD COLUMN tlusty_policy TEXT NOT NULL DEFAULT 'skip_converged'"),
|
||
("tlusty_strategies", "ALTER TABLE tasks ADD COLUMN tlusty_strategies TEXT NOT NULL DEFAULT '[\"cold_run\"]'"),
|
||
("synspec_enabled", "ALTER TABLE tasks ADD COLUMN synspec_enabled BOOLEAN NOT NULL DEFAULT 1"),
|
||
("synspec_policy", "ALTER TABLE tasks ADD COLUMN synspec_policy TEXT NOT NULL DEFAULT 'skip_converged'"),
|
||
("synspec_strategies", "ALTER TABLE tasks ADD COLUMN synspec_strategies TEXT NOT NULL DEFAULT '[\"standard\"]'"),
|
||
("atmosphere_ref", "ALTER TABLE tasks ADD COLUMN atmosphere_ref TEXT"),
|
||
] {
|
||
let has_col = conn
|
||
.prepare("PRAGMA table_info(tasks)")?
|
||
.query_map([], |r| r.get::<_, String>(1))?
|
||
.any(|r| r.map(|n| n == col).unwrap_or(false));
|
||
if !has_col {
|
||
let _ = conn.execute(sql, []);
|
||
}
|
||
}
|
||
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS seeds (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
point_name TEXT UNIQUE NOT NULL,
|
||
teff REAL NOT NULL,
|
||
logg REAL NOT NULL,
|
||
loghe REAL NOT NULL,
|
||
logc REAL NOT NULL,
|
||
logn REAL NOT NULL,
|
||
logo REAL NOT NULL,
|
||
file_path TEXT NOT NULL,
|
||
is_clean BOOLEAN NOT NULL DEFAULT 1
|
||
);",
|
||
[],
|
||
)?;
|
||
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS workflows (
|
||
name TEXT PRIMARY KEY,
|
||
description TEXT,
|
||
config_yaml TEXT NOT NULL,
|
||
status TEXT NOT NULL DEFAULT 'idle',
|
||
created_at DATETIME NOT NULL,
|
||
updated_at DATETIME NOT NULL
|
||
);",
|
||
[],
|
||
)?;
|
||
|
||
conn.execute(
|
||
// 多工作流分区后，调度查询恒带 WHERE workflow_name=?，故索引前置 workflow_name。
|
||
"CREATE INDEX IF NOT EXISTS idx_grid_points_wf_status ON grid_points(workflow_name, status, wave, cno_sum, teff);",
|
||
[],
|
||
)?;
|
||
conn.execute(
|
||
// 复合唯一约束：(workflow_name, name) 唯一，支撑 ON CONFLICT(workflow_name, name)。
|
||
"CREATE UNIQUE INDEX IF NOT EXISTS idx_grid_points_wf_name ON grid_points(workflow_name, name);",
|
||
[],
|
||
)?;
|
||
|
||
// P3 进度时间序列：后台循环对运行中工作流定期记录计数快照（仅在计数变化时
|
||
// 写入，见 record_progress_snapshot 去重），供详情页进度曲线与经验速率 ETA。
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS workflow_progress_snapshots (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
workflow_name TEXT NOT NULL,
|
||
ts DATETIME NOT NULL DEFAULT (datetime('now')),
|
||
total INTEGER NOT NULL,
|
||
pending INTEGER NOT NULL,
|
||
queued INTEGER NOT NULL,
|
||
running INTEGER NOT NULL,
|
||
completed INTEGER NOT NULL,
|
||
failed INTEGER NOT NULL
|
||
);",
|
||
[],
|
||
)?;
|
||
conn.execute(
|
||
"CREATE INDEX IF NOT EXISTS idx_progress_snapshots_wf_ts ON workflow_progress_snapshots(workflow_name, ts);",
|
||
[],
|
||
)?;
|
||
|
||
conn.execute(
|
||
"CREATE INDEX IF NOT EXISTS idx_seeds_is_clean ON seeds(is_clean);",
|
||
[],
|
||
)?;
|
||
|
||
// 逐点"最近一次尝试"关联子查询（GET /workflows/:name/points）的支撑索引：
|
||
// 按 (point_name, workflow_name) 定位，completed_at 排序取最新 task 行。
|
||
conn.execute(
|
||
"CREATE INDEX IF NOT EXISTS idx_tasks_point_wf_time ON tasks(point_name, workflow_name, completed_at);",
|
||
[],
|
||
)?;
|
||
|
||
// Node 专属凭据表（L2 鉴权）：存储每个 node 颁发的 token 的 SHA-256 hash（不存明文）。
|
||
// token 失效靠重发覆盖 token_hash 实现（旧 hash 不再存在 → 鉴权失败），无独立吊销标记。
|
||
// 明文 token 仅在注册/重发时返回一次。
|
||
// 注：历史库的 revoked 死列由 M4 迁移清除（Phase 4，P5）；全新库不再建该列。
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS node_credentials (
|
||
node_id TEXT PRIMARY KEY,
|
||
token_hash TEXT NOT NULL,
|
||
issued_at DATETIME NOT NULL,
|
||
raw_token_pending TEXT
|
||
);",
|
||
[],
|
||
)?;
|
||
let has_raw_pending = conn
|
||
.prepare("PRAGMA table_info(node_credentials)")?
|
||
.query_map([], |r| r.get::<_, String>(1))?
|
||
.any(|r| r.map(|n| n == "raw_token_pending").unwrap_or(false));
|
||
if !has_raw_pending {
|
||
let _ = conn.execute("ALTER TABLE node_credentials ADD COLUMN raw_token_pending TEXT", []);
|
||
}
|
||
|
||
conn.execute(
|
||
"CREATE UNIQUE INDEX IF NOT EXISTS idx_node_credentials_token_hash ON node_credentials(token_hash);",
|
||
[],
|
||
)?;
|
||
|
||
// 清理过期或处理过的暂存明文（取走即焚的安全兜底：超 1 天未拉取则作废）
|
||
let _ = conn.execute("UPDATE node_credentials SET raw_token_pending = NULL WHERE datetime('now', '-1 day') >= issued_at", []);
|
||
|
||
// Phase 0：bootstrap 完成。全新库的 user_version 天然为 0（SQLite 默认），
|
||
// 无需显式锚定；后续结构变更一律走 migrations::apply_migrations，
|
||
// 不再在 init_tables 内新增手写 ALTER 块。
|
||
info!("成功初始化 dcts.db 数据库结构表及索引");
|
||
Ok(())
|
||
})
|
||
.await??;
|
||
|
||
Ok(())
|
||
}
|
||
|
||
pub async fn backup_database(&self, backup_dir: &str) -> Result<()> {
|
||
let pool = self.pool.clone();
|
||
let dir = backup_dir.to_string();
|
||
|
||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||
let path = std::path::Path::new(&dir);
|
||
if !path.exists() {
|
||
std::fs::create_dir_all(path)?;
|
||
}
|
||
|
||
let now = std::time::SystemTime::now();
|
||
let seven_days = std::time::Duration::from_secs(7 * 24 * 3600);
|
||
// 保留期清理：只删除本程序写出的 `dcts_backup_*.db` 文件中超过 7 天的。
|
||
// 旧实现会删除 backup_dir 内任何超期 *.db（含运维放置的无关 .db 文件）。
|
||
if let Ok(entries) = std::fs::read_dir(path) {
|
||
for entry in entries.flatten() {
|
||
let file_path = entry.path();
|
||
if !file_path.is_file() {
|
||
continue;
|
||
}
|
||
// 仅匹配自身产物命名 dcts_backup_<时间戳>.db，避免误删无关 .db
|
||
let matches_name = file_path
|
||
.file_name()
|
||
.and_then(|n| n.to_str())
|
||
.map(|n| n.starts_with("dcts_backup_") && n.ends_with(".db"))
|
||
.unwrap_or(false);
|
||
if !matches_name {
|
||
continue;
|
||
}
|
||
if let Ok(meta) = entry.metadata() {
|
||
if let Ok(modified) = meta.modified() {
|
||
if now
|
||
.duration_since(modified)
|
||
.unwrap_or(std::time::Duration::from_secs(0))
|
||
> seven_days
|
||
{
|
||
let _ = std::fs::remove_file(file_path);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
let timestamp = chrono::Local::now().format("%Y%m%d_%H%M%S");
|
||
let backup_file = path.join(format!("dcts_backup_{}.db", timestamp));
|
||
|
||
let conn = pool
|
||
.get()
|
||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||
conn.execute(
|
||
"VACUUM INTO ?1",
|
||
params![backup_file.to_string_lossy().to_string()],
|
||
)?;
|
||
|
||
// 收紧备份文件权限为 0600（仅 owner 读写），与主库口径一致。
|
||
// 备份通过 VACUUM INTO 直接由 SQLite 写出，默认沿用 umask（可能 0644），
|
||
// 可被同机其他用户读取；备份含 node 凭据 hash 与（瞬时）明文待发 token。
|
||
#[cfg(unix)]
|
||
{
|
||
use std::os::unix::fs::PermissionsExt;
|
||
if let Ok(meta) = std::fs::metadata(&backup_file) {
|
||
let mut perms = meta.permissions();
|
||
perms.set_mode(0o600);
|
||
let _ = std::fs::set_permissions(&backup_file, perms);
|
||
}
|
||
}
|
||
|
||
Ok(())
|
||
})
|
||
.await??;
|
||
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
/// 管理视图：节点信息 + 凭据状态（用于 admin 列表 / 凭据管理）。
|
||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||
pub struct NodeCredentialView {
|
||
pub node_id: String,
|
||
pub max_slots: i32,
|
||
pub active_slots: i32,
|
||
pub status: String,
|
||
pub cpu_usage: f32,
|
||
pub memory_usage: f32,
|
||
pub last_heartbeat: chrono::DateTime<chrono::Utc>,
|
||
/// 凭据状态：none(无凭据记录) / active(有效)。token 失效靠重发覆盖 hash 实现，
|
||
/// 不存在「已吊销」中间态——旧 token 失效后该节点若无新 token 即为 none。
|
||
pub token_status: String,
|
||
/// 凭据颁发时间（ISO 字符串，无凭据时为 None）
|
||
pub token_issued_at: Option<String>,
|
||
/// 管理员强制并发槽位上限（动态调整 CPU 核数）。None = 无限制，使用 max_slots。
|
||
pub admin_max_slots: Option<i32>,
|
||
}
|
||
|
||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||
pub struct WorkflowSummary {
|
||
pub name: String,
|
||
pub description: Option<String>,
|
||
pub status: String,
|
||
pub created_at: String,
|
||
pub updated_at: String,
|
||
/// 内联的网格点聚合计数（首页卡片进度条数据源）。
|
||
/// 工作流尚无网格点（未启动）时为 None → 序列化为 null，前端据此不渲染进度条。
|
||
#[serde(default)]
|
||
pub stats: Option<WorkflowListStats>,
|
||
}
|
||
|
||
/// 工作流列表内联的网格统计（单条 GROUP BY 聚合回填，无 N+1）。
|
||
///
|
||
/// 阶段分项计数（TLUSTY/SYNSPEC 双视图）：前端按选中阶段切换展示，每阶段独立统计，
|
||
/// 不再混用整体 status。`tlusty_*` 基于 `tlusty_status` 列，`synspec_*` 基于 `synspec_status` 列。
|
||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||
pub struct WorkflowListStats {
|
||
pub total: i64,
|
||
pub completed: i64,
|
||
pub failed: i64,
|
||
pub running: i64,
|
||
pub cold_run_converged: i64,
|
||
pub seed_step_converged: i64,
|
||
/// TLUSTY 阶段收敛总数（tlusty_status='converged'）——权威口径，语义同
|
||
/// WorkflowStats.tlusty_converged（2026-08-25 补，见彼处注释）。
|
||
#[serde(default)]
|
||
pub tlusty_converged: i64,
|
||
#[serde(default)]
|
||
pub seed_step_stab_converged: i64,
|
||
/// TLUSTY 阶段发散点数（tlusty_status='failed'）。
|
||
pub tlusty_failed: i64,
|
||
/// SYNSPEC 阶段收敛点数（synspec_status='converged'）。
|
||
pub synspec_converged: i64,
|
||
/// SYNSPEC 阶段失败点数（synspec_status='failed'）。
|
||
pub synspec_failed: i64,
|
||
/// SYNSPEC 阶段未运行（synspec_status='pending'，多为 tlusty 未收敛而阻塞）。
|
||
pub synspec_pending: i64,
|
||
}
|
||
|
||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||
pub struct WorkflowItem {
|
||
pub name: String,
|
||
pub description: Option<String>,
|
||
pub config_yaml: String,
|
||
pub status: String,
|
||
pub created_at: String,
|
||
pub updated_at: String,
|
||
}
|
||
|
||
/// 难度波次（wave）统计条目：同 cno_sum 分组的网格点批次进度。
|
||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||
pub struct WaveStats {
|
||
pub wave: i32,
|
||
pub total: i64,
|
||
pub completed: i64,
|
||
pub failed: i64,
|
||
}
|
||
|
||
/// 单工作流执行统计（详情页数据源）。
|
||
///
|
||
/// 阶段分项计数（TLUSTY/SYNSPEC 双视图）：前端按选中阶段切换展示。
|
||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||
pub struct WorkflowStats {
|
||
pub name: String,
|
||
pub status: String,
|
||
pub total: i64,
|
||
pub pending: i64,
|
||
pub queued: i64,
|
||
pub running: i64,
|
||
pub completed: i64,
|
||
pub failed: i64,
|
||
pub cold_run_converged: i64,
|
||
pub seed_step_converged: i64,
|
||
/// TLUSTY 阶段收敛总数（tlusty_status='converged'，不按 method 拆分）——权威口径。
|
||
/// 前端「N 大气收敛」应消费此字段；cold/seed/stab 为策略细分。2026-08-25 补：
|
||
/// 此前前端用 cold+seed 之和，seed_step_stab（2026-08-18 引入）收敛点被漏计
|
||
/// （生产 9137/9216 差额主因）。
|
||
#[serde(default)]
|
||
pub tlusty_converged: i64,
|
||
/// TLUSTY 阶段以 seed_step_stab 策略收敛的点数（稳定化/waypoint 种子链）。
|
||
#[serde(default)]
|
||
pub seed_step_stab_converged: i64,
|
||
/// TLUSTY 阶段发散点数（tlusty_status='failed'）。
|
||
pub tlusty_failed: i64,
|
||
/// SYNSPEC 阶段收敛点数（synspec_status='converged'）。
|
||
pub synspec_converged: i64,
|
||
/// SYNSPEC 阶段失败点数（synspec_status='failed'）。
|
||
pub synspec_failed: i64,
|
||
/// SYNSPEC 阶段未运行（synspec_status='pending'）。
|
||
pub synspec_pending: i64,
|
||
pub waves: Vec<WaveStats>,
|
||
/// 近似单点平均墙钟耗时（秒，含排队等待，仅参考）；无历史数据为 None。
|
||
pub avg_point_sec: Option<f64>,
|
||
/// 预估剩余时间（秒）≈ avg_point_sec × 剩余点数；无法估算为 None。
|
||
pub eta_sec: Option<f64>,
|
||
}
|
||
|
||
/// 网格点列表行：点参数 + 状态 + 最近一次尝试的附加信息。
|
||
///
|
||
/// `last_*` 字段来自 tasks 表最新一行（按 completed_at/created_at 取最新）；
|
||
/// 从未派发过的点这些字段为 None。
|
||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||
pub struct PointRow {
|
||
pub name: String,
|
||
pub teff: f64,
|
||
pub logg: f64,
|
||
pub loghe: f64,
|
||
pub logc: f64,
|
||
pub logn: f64,
|
||
pub logo: f64,
|
||
/// TLUSTY 大气金属丰度排序量（cno_sum = logc+logn+logo；仅对大气阶段有意义，光谱阶段无关）。
|
||
pub cno_sum: f64,
|
||
/// 按 cno_sum 分组的批次波次（TLUSTY 大气难度的调度优先级；SYNSPEC-only 任务继承大气波次）。
|
||
pub wave: i32,
|
||
pub status: String,
|
||
/// TLUSTY 阶段收敛策略（cold_run/seed_step/策略名；TLUSTY 禁用为 NULL）。
|
||
pub tlusty_success_method: Option<String>,
|
||
/// 光谱阶段收敛策略（synspec 以何策略收敛；TLUSTY-only 为 NULL）。
|
||
pub synspec_success_method: Option<String>,
|
||
/// TLUSTY 阶段状态（'converged'/'failed'/'pending'/NULL）。
|
||
/// 独立于整体 status——半失败点（大气收敛+光谱失败）此处为 'converged' 而 status='failed'。
|
||
#[serde(default)]
|
||
pub tlusty_status: Option<String>,
|
||
/// SYNSPEC 阶段状态（'converged'/'failed'/'pending'/NULL）。
|
||
#[serde(default)]
|
||
pub synspec_status: Option<String>,
|
||
pub attempt_count: i32,
|
||
/// 最近尝试的收敛指标（最大相对修正）。
|
||
pub last_max_relc: Option<f64>,
|
||
/// 最近尝试的种子来源点名（seed_step 时有值）。
|
||
pub seed_point_name: Option<String>,
|
||
/// 最近尝试的执行节点。
|
||
pub node_id: Option<String>,
|
||
pub last_completed_at: Option<String>,
|
||
pub last_error: Option<String>,
|
||
/// 最近一次尝试的真实墙钟耗时（秒，Worker 回报值；旧数据为 None）。
|
||
pub last_elapsed_sec: Option<f64>,
|
||
}
|
||
|
||
/// PointRow 行映射器：list_workflow_points 与 get_workflow_point_row 共用同一 SELECT 列序。
|
||
fn point_row_from_query(r: &rusqlite::Row<'_>) -> rusqlite::Result<PointRow> {
|
||
Ok(PointRow {
|
||
name: r.get(0)?,
|
||
teff: r.get(1)?,
|
||
logg: r.get(2)?,
|
||
loghe: r.get(3)?,
|
||
logc: r.get(4)?,
|
||
logn: r.get(5)?,
|
||
logo: r.get(6)?,
|
||
cno_sum: r.get(7)?,
|
||
wave: r.get(8)?,
|
||
status: r.get(9)?,
|
||
tlusty_success_method: r.get(10)?,
|
||
attempt_count: r.get(11)?,
|
||
last_max_relc: r.get(12)?,
|
||
seed_point_name: r.get(13)?,
|
||
node_id: r.get(14)?,
|
||
last_completed_at: r.get(15)?,
|
||
last_error: r.get(16)?,
|
||
last_elapsed_sec: r.get(17)?,
|
||
synspec_success_method: r.get(19)?,
|
||
tlusty_status: r.get(20)?,
|
||
synspec_status: r.get(21)?,
|
||
})
|
||
}
|
||
|
||
/// 逐点列表查询条件。所有枚举值与 ORDER BY 片段必须由 handler 白名单预校验，
|
||
/// 值一律参数化绑定——本结构不接触任何未经校验的用户输入。
|
||
#[derive(Debug, Clone, Default)]
|
||
pub struct PointFilter {
|
||
pub status: Option<String>,
|
||
pub method: Option<String>,
|
||
pub wave: Option<i32>,
|
||
/// 点名子串（原始值；本层负责 LIKE 通配符转义）。
|
||
pub q: Option<String>,
|
||
/// 编译期列名 + ASC/DESC 拼成的 ORDER BY 片段（不含用户文本）。
|
||
pub order_by: String,
|
||
/// None = 不限制（联合分析拉全量，避免截断导致分析失真）。
|
||
pub limit: Option<i64>,
|
||
pub offset: i64,
|
||
}
|
||
|
||
/// 进度快照行（详情页进度曲线/经验速率/停滞检测的时序数据源）。
|
||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||
pub struct ProgressPoint {
|
||
pub ts: String,
|
||
pub total: i64,
|
||
pub pending: i64,
|
||
pub queued: i64,
|
||
pub running: i64,
|
||
pub completed: i64,
|
||
pub failed: i64,
|
||
}
|
||
|
||
/// 单次任务尝试记录（点详情的"尝试历史"数据源）。一个网格点可有多行
|
||
/// （冷启动失败 → 种子步进救回），按 created_at 升序还原完整剧情。
|
||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||
pub struct AttemptRow {
|
||
pub task_id: String,
|
||
pub seed_point_name: Option<String>,
|
||
pub status: String,
|
||
pub max_relc: Option<f64>,
|
||
pub atmosphere_has_nan: bool,
|
||
pub node_id: Option<String>,
|
||
pub error_message: Option<String>,
|
||
pub created_at: String,
|
||
pub completed_at: Option<String>,
|
||
/// 本次尝试真实墙钟耗时（秒；旧数据为 None）。
|
||
pub elapsed_sec: Option<f64>,
|
||
/// 失败阶段归因（"tlusty"/"synspec"；旧节点/旧行 NULL → 前端兜底显示 TLUSTY）。
|
||
#[serde(default)]
|
||
pub failed_stage: Option<String>,
|
||
/// 完整 ModelSummary JSON（含 synspec_rc/error/sec 与各子步骤摘要）；错误路径为 `{"error": ...}`，
|
||
/// 前端解析必须容错（解析失败即显示错误文本）。
|
||
#[serde(default)]
|
||
pub summary_json: Option<String>,
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use common::models::PhaseConfig;
|
||
use uuid::Uuid;
|
||
|
||
/// H1 修复：空/缺省 workflow_name 归一到 '__legacy__'，与主库迁移回填口径一致。
|
||
/// 这是旧版在途任务（payload 无 workflow_name）能命中 legacy 网格点的关键映射。
|
||
#[test]
|
||
fn test_normalize_workflow_name_legacy_mapping() {
|
||
assert_eq!(normalize_workflow_name(None), "__legacy__");
|
||
assert_eq!(normalize_workflow_name(Some("")), "__legacy__");
|
||
assert_eq!(normalize_workflow_name(Some("sdB_cno")), "sdB_cno");
|
||
// 留空字符串的旧值同样归一，避免 0 行命中。
|
||
assert_eq!(normalize_workflow_name(Some("__legacy__")), "__legacy__");
|
||
}
|
||
|
||
/// wave 修复：`compute_wave_for_cno_sum` 按"工作流内 cno_sum 严格小于本点的去重值个数"
|
||
/// 计算波次，与 initialize_grid 的分组口径一致。导入点据此归入正确波次，不再硬编码 0。
|
||
#[tokio::test]
|
||
async fn test_compute_wave_for_cno_sum() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db_path = temp_dir.path().join("wave_calc.db");
|
||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||
|
||
let mk = |logc: f64, logn: f64, logo: f64| GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: logc.into(),
|
||
logn: logn.into(),
|
||
logo: logo.into(),
|
||
};
|
||
|
||
// 三个不同 cno_sum 的点：-9, -6, -3（升序 rank 应为 0/1/2）。
|
||
db.upsert_grid_point(&mk(-3.0, -3.0, -3.0), 0, "wf_w")
|
||
.await
|
||
.unwrap();
|
||
db.upsert_grid_point(&mk(-2.0, -2.0, -2.0), 0, "wf_w")
|
||
.await
|
||
.unwrap();
|
||
db.upsert_grid_point(&mk(-1.0, -1.0, -1.0), 0, "wf_w")
|
||
.await
|
||
.unwrap();
|
||
|
||
assert_eq!(db.compute_wave_for_cno_sum("wf_w", -9.0).await.unwrap(), 0);
|
||
assert_eq!(db.compute_wave_for_cno_sum("wf_w", -6.0).await.unwrap(), 1);
|
||
assert_eq!(db.compute_wave_for_cno_sum("wf_w", -3.0).await.unwrap(), 2);
|
||
// 不存在的 cno 值：rank 仍按现有库的去重值判定（-8 夹在 -9/-6 之间 → 1）。
|
||
assert_eq!(db.compute_wave_for_cno_sum("wf_w", -8.0).await.unwrap(), 1);
|
||
assert_eq!(db.compute_wave_for_cno_sum("wf_w", -12.0).await.unwrap(), 0);
|
||
assert_eq!(db.compute_wave_for_cno_sum("wf_w", 0.0).await.unwrap(), 3);
|
||
// 空工作流：任何 cno 的 rank 都是 0。
|
||
assert_eq!(
|
||
db.compute_wave_for_cno_sum("wf_empty", -3.0).await.unwrap(),
|
||
0
|
||
);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_db_node_and_grid_operations() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db_path = temp_dir.path().join("test_db.db");
|
||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||
|
||
// 1. Node registration & heartbeat
|
||
let reg_req = NodeRegisterRequest {
|
||
node_id: "node-test-1".to_string(),
|
||
max_slots: 4,
|
||
};
|
||
db.register_node(®_req).await.unwrap();
|
||
db.approve_node("node-test-1").await.unwrap();
|
||
|
||
let active_nodes = db.get_active_nodes().await.unwrap();
|
||
assert_eq!(active_nodes.len(), 1);
|
||
assert_eq!(active_nodes[0].node_id, "node-test-1");
|
||
|
||
let hb_req = NodeHeartbeatRequest {
|
||
node_id: "node-test-1".to_string(),
|
||
active_slots: 2,
|
||
cpu_usage: 45.0,
|
||
memory_usage: 60.0,
|
||
};
|
||
db.heartbeat_node(&hb_req).await.unwrap();
|
||
|
||
// 2. Grid points & task reports
|
||
let params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
db.upsert_grid_point(¶ms, 0, "test_wf").await.unwrap();
|
||
|
||
let pending = db.get_pending_grid_points("test_wf").await.unwrap();
|
||
assert_eq!(pending.len(), 1);
|
||
assert_eq!(pending[0].0, params.model_name());
|
||
|
||
// Record successful report
|
||
let report = TaskReport {
|
||
task_id: Uuid::new_v4(),
|
||
point_name: params.model_name(),
|
||
params: Some(params.clone()),
|
||
node_id: "node-test-1".to_string(),
|
||
status: TaskStatus::Completed,
|
||
result_valid: true,
|
||
max_relc: Some(0.0001),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 15.0,
|
||
error_message: None,
|
||
summary_json: "{}".to_string(),
|
||
failed_stage: None,
|
||
};
|
||
db.record_task_report(&report, "test_wf").await.unwrap();
|
||
|
||
// Check grid point is marked converged
|
||
let pending_after = db.get_pending_grid_points("test_wf").await.unwrap();
|
||
assert_eq!(pending_after.len(), 0);
|
||
|
||
// 3. Workflow CRUD
|
||
db.upsert_workflow(
|
||
"test_wf",
|
||
Some("Test Workflow"),
|
||
"grid:\n teff: [35000]",
|
||
"idle",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
let wf = db.get_workflow("test_wf").await.unwrap();
|
||
assert!(wf.is_some());
|
||
assert_eq!(wf.unwrap().name, "test_wf");
|
||
|
||
db.delete_workflow("test_wf").await.unwrap();
|
||
assert!(db.get_workflow("test_wf").await.unwrap().is_none());
|
||
}
|
||
|
||
/// 多工作流分区隔离测试（#3 修复核心验证）：
|
||
/// 1. 同一物理点写入两个工作流，互不覆盖（复合唯一约束）。
|
||
/// 2. reset_queued_grid_points_to_pending 按 workflow 隔离：重置 wf_a 不影响 wf_b。
|
||
/// 3. update_grid_status 按 workflow 隔离：改 wf_a 的点不影响 wf_b 同名点。
|
||
/// 4. reset_specific_grid_points_to_pending 按 workflow 隔离。
|
||
#[tokio::test]
|
||
async fn test_multi_workflow_grid_isolation() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db_path = temp_dir.path().join("iso_db.db");
|
||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
|
||
// 1. 两个工作流写入同一物理点 —— 应各自独立存在（复合唯一 (wf, name)）。
|
||
db.upsert_grid_point(¶ms, 0, "wf_a").await.unwrap();
|
||
db.upsert_grid_point(¶ms, 0, "wf_b").await.unwrap();
|
||
assert_eq!(db.get_pending_grid_points("wf_a").await.unwrap().len(), 1);
|
||
assert_eq!(db.get_pending_grid_points("wf_b").await.unwrap().len(), 1);
|
||
|
||
// 2. 把 wf_a 的点置 queued，wf_b 保持 pending；reset wf_a 的 queued 不应波及 wf_b。
|
||
db.update_grid_status(&name, GridPointStatus::Queued, "wf_a")
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
db.get_grid_point_status(&name, "wf_a")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"queued"
|
||
);
|
||
assert_eq!(
|
||
db.get_grid_point_status(&name, "wf_b")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"pending"
|
||
);
|
||
let reset_cnt = db
|
||
.reset_queued_grid_points_to_pending("wf_a")
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(reset_cnt, 1);
|
||
assert_eq!(
|
||
db.get_grid_point_status(&name, "wf_a")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"pending"
|
||
);
|
||
// wf_b 仍是 pending（未被误改）
|
||
assert_eq!(
|
||
db.get_grid_point_status(&name, "wf_b")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"pending"
|
||
);
|
||
|
||
// 3. update_grid_status 按 workflow 隔离：把 wf_a 标 failed，wf_b 不受影响。
|
||
db.update_grid_status(&name, GridPointStatus::Failed, "wf_a")
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
db.get_grid_point_status(&name, "wf_a")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"failed"
|
||
);
|
||
assert_eq!(
|
||
db.get_grid_point_status(&name, "wf_b")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"pending"
|
||
);
|
||
|
||
// 4. reset_specific 按 workflow 隔离：wf_a 的 queued→running 点被重置，wf_b 同名点不动。
|
||
db.update_grid_status(&name, GridPointStatus::Queued, "wf_a")
|
||
.await
|
||
.unwrap();
|
||
db.update_grid_status(&name, GridPointStatus::Queued, "wf_b")
|
||
.await
|
||
.unwrap();
|
||
let cnt = db
|
||
.reset_specific_grid_points_to_pending(std::slice::from_ref(&name), "wf_a")
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(cnt, 1, "仅 wf_a 的 queued 点被重置");
|
||
assert_eq!(
|
||
db.get_grid_point_status(&name, "wf_a")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"pending"
|
||
);
|
||
assert_eq!(
|
||
db.get_grid_point_status(&name, "wf_b")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"queued"
|
||
);
|
||
}
|
||
|
||
/// grid_summary_stats 按 workflow 聚合 + 全局聚合测试。
|
||
/// 覆盖 pending/queued 拆分计数与导入点按途径（cold_run/seed_step）归类的口径。
|
||
#[tokio::test]
|
||
async fn test_grid_summary_stats_workflow_scoping() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("stats_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
let p = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let p2 = GridPointParams {
|
||
teff: 40000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
db.upsert_grid_point(&p, 0, "wf_a").await.unwrap();
|
||
db.upsert_grid_point(&p, 0, "wf_b").await.unwrap();
|
||
db.upsert_grid_point(&p2, 1, "wf_a").await.unwrap();
|
||
|
||
// wf_b 的点入队（queued），wf_a 的 p2 走历史导入收敛（标记为 seed_step 途径）
|
||
db.update_grid_status(&p.model_name(), GridPointStatus::Queued, "wf_b")
|
||
.await
|
||
.unwrap();
|
||
{
|
||
let summary = common::models::ModelSummary {
|
||
name: p2.model_name(),
|
||
params: p2.clone(),
|
||
stages: Vec::new(),
|
||
result_valid: true,
|
||
final_max_relc: Some(0.001),
|
||
final_chmax: Some(0.001),
|
||
seed: None,
|
||
atmosphere_has_nan: false,
|
||
synspec_rc: None,
|
||
synspec_error: None,
|
||
synspec_sec: None,
|
||
elapsed_sec: 0.0,
|
||
energy_check: None,
|
||
temp_check: None,
|
||
emflux_check: None,
|
||
bfac_check: None,
|
||
ladder_seeds: Vec::new(),
|
||
note: None,
|
||
};
|
||
db.upsert_point_summary(&summary.name, "wf_a", &summary, "seed_step")
|
||
.await
|
||
.unwrap();
|
||
}
|
||
|
||
// 全局（None）：3 个点，pending/queued/converged 分开计数；
|
||
// 导入点按 seed_step 途径计入 seed_step_converged（不再有独立 imported 分类）
|
||
let all = db.get_grid_summary_stats(None).await.unwrap();
|
||
assert_eq!(all["total"], 3);
|
||
assert_eq!(all["pending"], 1);
|
||
assert_eq!(all["queued"], 1);
|
||
assert_eq!(all["completed"], 1);
|
||
assert_eq!(all["cold_run_converged"], 0);
|
||
assert_eq!(all["seed_step_converged"], 1);
|
||
// tlusty_converged（2026-08-25）：不按 method 拆分的权威总数。
|
||
assert_eq!(all["tlusty_converged"], 1);
|
||
assert_eq!(all["seed_step_stab_converged"], 0);
|
||
// 单工作流 wf_a：1 pending + 1 seed_step 收敛
|
||
let a = db.get_grid_summary_stats(Some("wf_a")).await.unwrap();
|
||
assert_eq!(a["total"], 2);
|
||
assert_eq!(a["pending"], 1);
|
||
assert_eq!(a["queued"], 0);
|
||
assert_eq!(a["seed_step_converged"], 1);
|
||
assert_eq!(a["tlusty_converged"], 1);
|
||
// 不存在的工作流：0
|
||
let none = db
|
||
.get_grid_summary_stats(Some("nonexistent"))
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(none["total"], 0);
|
||
assert_eq!(none["queued"], 0);
|
||
}
|
||
|
||
/// seed_finder exact_family 索引与全量扫描结果一致性测试（#7 优化正确性）。
|
||
/// 构造一个 exact_family 候选 + 一个 global 候选，验证索引路径仍命中正确结果。
|
||
#[tokio::test]
|
||
async fn test_seed_index_exact_and_global_consistency() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("seed_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
// exact_family 种子：与 target 同 teff/logg/loghe，仅 CNO 略有差异。
|
||
let exact = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let target = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.1).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
db.insert_seed(&exact, "/tmp/exact.7").await.unwrap();
|
||
|
||
// 应命中 exact_family（走索引路径）
|
||
let m = db.find_best_seed_from_db(&target).await.unwrap();
|
||
assert!(m.is_some(), "exact_family 索引路径应命中");
|
||
assert_eq!(m.unwrap().name, exact.model_name());
|
||
}
|
||
|
||
/// find_exact_family_seed_from_db 严格同物理族判定（2026-08-19 生产修复回归）：
|
||
/// ladder 中间种子（如 teff=22500）即便 CNO 距离为 0，也因 Teff 不同被排除；
|
||
/// 同 Teff/logg/logHe 的 CNO 邻居正常命中；排除目标自身名。
|
||
#[tokio::test]
|
||
async fn test_find_exact_family_seed_strict_same_family() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("seed_stab_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let target = GridPointParams {
|
||
teff: 20000.0.into(),
|
||
logg: 6.5.into(),
|
||
loghe: 2.0.into(),
|
||
logc: (-4.0).into(),
|
||
logn: (-3.0).into(),
|
||
logo: (-4.0).into(),
|
||
};
|
||
// ladder 中间种子：同 logg/logHe、CNO 完全一致，但 teff=22500 —— 必须排除。
|
||
let ladder = GridPointParams {
|
||
teff: 22500.0.into(),
|
||
logg: 6.5.into(),
|
||
loghe: 2.0.into(),
|
||
logc: (-4.0).into(),
|
||
logn: (-3.0).into(),
|
||
logo: (-4.0).into(),
|
||
};
|
||
// 合法 CNO 邻居：同 Teff/logg/logHe，Δo=1。
|
||
let cno_neighbor = GridPointParams {
|
||
teff: 20000.0.into(),
|
||
logg: 6.5.into(),
|
||
loghe: 2.0.into(),
|
||
logc: (-4.0).into(),
|
||
logn: (-3.0).into(),
|
||
logo: (-3.0).into(),
|
||
};
|
||
db.insert_seed(&ladder, "/tmp/ladder.7").await.unwrap();
|
||
db.insert_seed(&cno_neighbor, "/tmp/neighbor.7").await.unwrap();
|
||
|
||
let m = db
|
||
.find_exact_family_seed_from_db(&target, &[target.model_name()])
|
||
.await
|
||
.unwrap();
|
||
assert!(m.is_some(), "应命中 CNO 邻居");
|
||
assert_eq!(
|
||
m.unwrap().name,
|
||
cno_neighbor.model_name(),
|
||
"不得选 teff 不同的 ladder 中间种子（即便其 CNO 距离为 0）"
|
||
);
|
||
|
||
// 无严格同族候选时返回 None（不退化到 global）。
|
||
let lonely = GridPointParams {
|
||
teff: 30000.0.into(),
|
||
logg: 5.0.into(),
|
||
loghe: (-2.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let none = db
|
||
.find_exact_family_seed_from_db(&lonely, &[lonely.model_name()])
|
||
.await
|
||
.unwrap();
|
||
assert!(none.is_none());
|
||
}
|
||
|
||
|
||
/// 种子轮换回归（2026-08-20 修复）：find_exact_family_seed_from_db 排除列表
|
||
/// 应使调用方在重试间轮换到下一个未试过的同族 CNO 邻居；全部排除后返回 None。
|
||
#[tokio::test]
|
||
async fn test_find_exact_family_seed_rotation() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("seed_rot_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
let base = |logc: f64, logn: f64, logo: f64| GridPointParams {
|
||
teff: 20000.0.into(),
|
||
logg: 6.0.into(),
|
||
loghe: 2.0.into(),
|
||
logc: logc.into(),
|
||
logn: logn.into(),
|
||
logo: logo.into(),
|
||
};
|
||
let target = base(-3.0, -3.0, -1.0);
|
||
let n1 = base(-3.0, -2.0, -1.0);
|
||
let n2 = base(-3.0, -3.0, -2.0);
|
||
db.insert_seed(&n1, "/tmp/rot_n1.7").await.unwrap();
|
||
db.insert_seed(&n2, "/tmp/rot_n2.7").await.unwrap();
|
||
|
||
let self_name = target.model_name();
|
||
// 首次：选距离最近的未排除邻居。
|
||
let first = db
|
||
.find_exact_family_seed_from_db(&target, &[self_name.clone()])
|
||
.await
|
||
.unwrap()
|
||
.expect("应命中 CNO 邻居");
|
||
// 排除首个后：轮换到另一个邻居。
|
||
let second = db
|
||
.find_exact_family_seed_from_db(&target, &[self_name, first.name.clone()])
|
||
.await
|
||
.unwrap()
|
||
.expect("排除首个邻居后应轮换到第二个邻居");
|
||
assert_ne!(first.name, second.name);
|
||
// 全部排除后：None（终态 failed，不死循环）。
|
||
let none = db
|
||
.find_exact_family_seed_from_db(
|
||
&target,
|
||
&[target.model_name(), first.name, second.name],
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert!(none.is_none(), "邻居全部试过后应返回 None");
|
||
}
|
||
|
||
/// "种子回退仅一次"守卫语义（2026-08-02 涡旋事故定稿）：has_seed_step_attempt
|
||
/// 统计**一切** seed_step 行（含 pending）。pending 行在新架构下只有两种来源：
|
||
/// (a) 真在途（排队/已领用）——计数它正是对在途回退的去重，挡住救援途中迟到失败
|
||
/// 报告触发的第二份 seed_step；(b) 僵尸行（insert 后 push 前崩溃等）——由派发
|
||
/// 去重/孤儿回收/回退内种子僵尸卫生结构性清除，不再可能永久存在，故 2026-08-01
|
||
/// 的"僵尸堵死救援"前提已消失，无需再排除 pending（旧版排除法会放过重复回退）。
|
||
#[tokio::test]
|
||
async fn test_has_seed_step_attempt_counts_pending_live_task() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("seed_attempt_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let gp_params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = gp_params.model_name();
|
||
|
||
// 尚无 seed_step 行 → 守卫放行（false）。
|
||
assert!(
|
||
!db.has_seed_step_attempt(&name, "wf_z").await.unwrap(),
|
||
"无 seed_step 行时应放行回退"
|
||
);
|
||
|
||
// 派发 seed_step（pending 在途）→ 守卫立即生效，挡住救援途中的重复回退。
|
||
// Phase 6 起判定派生自 strategies[0]，故显式设策略链首项为 seed_step。
|
||
let seed_task_id = uuid::Uuid::new_v4();
|
||
db.insert_task(&common::models::TaskSpec {
|
||
task_id: seed_task_id,
|
||
point_name: name.clone(),
|
||
params: gp_params.clone(),
|
||
seed_point_name: Some("some_seed".to_string()),
|
||
timeout_sec: 7200,
|
||
workflow_name: Some("wf_z".to_string()),
|
||
wave: 0,
|
||
tlusty_config: PhaseConfig {
|
||
strategies: vec!["seed_step".to_string()],
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
..Default::default()
|
||
})
|
||
.await
|
||
.unwrap();
|
||
assert!(
|
||
db.has_seed_step_attempt(&name, "wf_z").await.unwrap(),
|
||
"pending 在途 seed_step 应计为已派发（在途去重）"
|
||
);
|
||
|
||
// 该 seed_step 产出执行结果（failed）后守卫当然继续生效 → "仅回退一次"。
|
||
{
|
||
let pool = db.pool.clone();
|
||
let tid = seed_task_id.to_string();
|
||
tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'failed' WHERE task_id = ?1",
|
||
params![tid],
|
||
)
|
||
.unwrap();
|
||
})
|
||
.await
|
||
.unwrap();
|
||
}
|
||
assert!(
|
||
db.has_seed_step_attempt(&name, "wf_z").await.unwrap(),
|
||
"真实执行过（failed）的 seed_step 应计为已尝试"
|
||
);
|
||
|
||
// 僵尸行被结构性清除（delete_tasks_by_ids）后，守卫重新放行正当救援。
|
||
let removed = db
|
||
.delete_tasks_by_ids(&[seed_task_id.to_string()])
|
||
.await
|
||
.unwrap();
|
||
// 该行已是 failed（非 pending），delete_tasks_by_ids 只删 pending → 不删。
|
||
assert_eq!(removed, 0, "delete_tasks_by_ids 只清除 pending 行");
|
||
}
|
||
|
||
/// 构造 TaskReport 的测试夹具。
|
||
fn mk_report(
|
||
point_name: &str,
|
||
params: &GridPointParams,
|
||
status: TaskStatus,
|
||
converged: bool,
|
||
) -> TaskReport {
|
||
TaskReport {
|
||
task_id: Uuid::new_v4(),
|
||
point_name: point_name.to_string(),
|
||
params: Some(params.clone()),
|
||
node_id: "node-test".to_string(),
|
||
status,
|
||
result_valid: converged,
|
||
max_relc: Some(0.0001),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 15.0,
|
||
error_message: None,
|
||
summary_json: "{}".to_string(),
|
||
failed_stage: None,
|
||
}
|
||
}
|
||
|
||
/// 阶段独立配置持久化与策略链弹栈（见 docs/task_engine_decoupling_design.md §4.1-§4.2）：
|
||
/// 1. insert_task 把 PhaseConfig 打平写入 tasks 表新列；
|
||
/// 2. get_latest_tlusty_strategies 读回完整链；
|
||
/// 3. pop_tlusty_strategy_for_fallback 原子弹出首项、返回剩余链；
|
||
/// 4. 再次 pop 反映已写回的剩余链（链耗尽返回 None）。
|
||
#[tokio::test]
|
||
async fn test_stage_config_persistence_and_strategy_pop() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("stage_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_grid_point(¶ms, 0, "wf_st").await.unwrap();
|
||
|
||
// 插入一个携带多策略链 + SYNSPEC 自定义配置的任务。
|
||
let spec = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
seed_point_name: None,
|
||
timeout_sec: 7200,
|
||
workflow_name: Some("wf_st".to_string()),
|
||
wave: 0,
|
||
tlusty_config: common::models::PhaseConfig {
|
||
enabled: true,
|
||
policy: common::models::ResumePolicy::ForceRecompute,
|
||
strategies: vec!["cold_run".to_string(), "seed_step".to_string()],
|
||
},
|
||
synspec_config: common::models::PhaseConfig {
|
||
enabled: false,
|
||
policy: common::models::ResumePolicy::SkipFailed,
|
||
strategies: vec!["standard".to_string()],
|
||
},
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&spec).await.unwrap();
|
||
// pop 过滤 status != 'pending'：须把该行标记为已上报（failed）才能被弹。
|
||
{
|
||
let pool = db.pool.clone();
|
||
let tid = spec.task_id.to_string();
|
||
tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'failed', completed_at = datetime('now') WHERE task_id = ?1",
|
||
rusqlite::params![tid],
|
||
)
|
||
.unwrap();
|
||
})
|
||
.await
|
||
.unwrap();
|
||
}
|
||
|
||
// 读回完整链（get_latest 不过滤 status，取最新行）。
|
||
let chain = db
|
||
.get_latest_tlusty_strategies(&name, "wf_st")
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(chain, vec!["cold_run".to_string(), "seed_step".to_string()]);
|
||
|
||
// 弹出首项 cold_run → 剩 [seed_step]（只读：旧行 strategies 不变），
|
||
// policy 同步带回该行落库值（派发时快照）。
|
||
let popped = db
|
||
.pop_tlusty_strategy_for_fallback(&name, "wf_st")
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
popped,
|
||
Some(FallbackSnapshot {
|
||
rest_strategies: vec!["seed_step".to_string()],
|
||
popped: "cold_run".to_string(),
|
||
policy: common::models::ResumePolicy::ForceRecompute,
|
||
}),
|
||
"首项弹出后剩余 seed_step（policy 取派发时快照）"
|
||
);
|
||
|
||
// pop 是只读的：旧行 strategies 仍为完整链（审计正确，不复制/不陈旧）。
|
||
let chain_after = db
|
||
.get_latest_tlusty_strategies(&name, "wf_st")
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
chain_after,
|
||
vec!["cold_run".to_string(), "seed_step".to_string()],
|
||
"只读 pop 不改写旧行"
|
||
);
|
||
|
||
// 模拟调度器派发回退任务：用剩余链 [seed_step] 插入新行（新 task_id，更晚 created_at）。
|
||
let fallback_spec = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
seed_point_name: None,
|
||
timeout_sec: 7200,
|
||
workflow_name: Some("wf_st".to_string()),
|
||
wave: 0,
|
||
tlusty_config: common::models::PhaseConfig {
|
||
enabled: true,
|
||
policy: common::models::ResumePolicy::ForceRecompute,
|
||
strategies: vec!["seed_step".to_string()],
|
||
},
|
||
synspec_config: common::models::PhaseConfig::default_synspec(),
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&fallback_spec).await.unwrap();
|
||
// 新行也是 pending；标记 failed + 回拨 created_at 晚 1 秒（datetime('now') 秒精度，
|
||
// 同秒插入的两行 ORDER BY 不稳定，须显式拉开 created_at 差距）才能被下一轮弹。
|
||
{
|
||
let pool = db.pool.clone();
|
||
let tid = fallback_spec.task_id.to_string();
|
||
tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'failed', created_at = datetime('now', '+1 second'), completed_at = datetime('now', '+1 second') WHERE task_id = ?1",
|
||
rusqlite::params![tid],
|
||
)
|
||
.unwrap();
|
||
})
|
||
.await
|
||
.unwrap();
|
||
}
|
||
|
||
// 再弹 → 取最新行（fallback_spec），弹出 seed_step → 剩空链。
|
||
let popped2 = db
|
||
.pop_tlusty_strategy_for_fallback(&name, "wf_st")
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
popped2,
|
||
Some(FallbackSnapshot {
|
||
rest_strategies: vec![],
|
||
popped: "seed_step".to_string(),
|
||
policy: common::models::ResumePolicy::ForceRecompute,
|
||
}),
|
||
"最新行弹出后链空（policy 仍为派发时快照）"
|
||
);
|
||
|
||
// pending 行不被弹（status 过滤）：插入一个 pending 行，pop 应跳过它取已上报行。
|
||
let pending_spec = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&pending_spec).await.unwrap();
|
||
// pending_spec 是最新行但 status=pending → pop 跳过它，取 fallback_spec（已上报）。
|
||
// 只读 pop 语义：fallback_spec 的链仍为 [seed_step]（未被写改），再弹仍返回 ([], seed_step)。
|
||
let popped3 = db
|
||
.pop_tlusty_strategy_for_fallback(&name, "wf_st")
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
popped3,
|
||
Some(FallbackSnapshot {
|
||
rest_strategies: vec![],
|
||
popped: "seed_step".to_string(),
|
||
policy: common::models::ResumePolicy::ForceRecompute,
|
||
}),
|
||
"pending 行被跳过，pop 回落到已上报的 fallback_spec 行"
|
||
);
|
||
// 确认 pending_spec 行的链从未被读取（其 strategies 是 default_tlusty=[cold_run,seed_step]，
|
||
// 若 pop 错误地选了它，会返回 ([seed_step], cold_run) 而非 ([], seed_step)）。
|
||
}
|
||
|
||
/// 终态守卫（2026-08-02 涡旋事故修复）：
|
||
/// 1. converged 吸收迟到失败报告（bool=false，状态不变）——事故直接症状的回归测试；
|
||
/// 2. failed 吸收重复失败报告（bool=false）——防重复触发种子回退；
|
||
/// 3. queued → failed 生效（bool=true）——真实领用计算过的任务打在非终态点上应迁移；
|
||
/// 4. 迟到真收敛获胜（failed → converged，bool=true）。
|
||
#[tokio::test]
|
||
async fn test_record_task_report_terminal_guards() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("guards_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_grid_point(¶ms, 0, "wf_g").await.unwrap();
|
||
|
||
// 点置 running 后正常失败 → 迁移生效。
|
||
assert!(db.mark_grid_point_running(&name, "wf_g").await.unwrap());
|
||
let changed = db
|
||
.record_task_report(
|
||
&mk_report(&name, ¶ms, TaskStatus::Failed, false),
|
||
"wf_g",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert!(changed, "running → failed 应迁移");
|
||
assert_eq!(
|
||
db.get_grid_point_status(&name, "wf_g")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"failed"
|
||
);
|
||
|
||
// 重复失败报告（不同 task_id）打在 failed 点 → 吸收，bool=false。
|
||
let changed2 = db
|
||
.record_task_report(
|
||
&mk_report(&name, ¶ms, TaskStatus::Failed, false),
|
||
"wf_g",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert!(!changed2, "failed 点应吸收重复失败报告");
|
||
|
||
// 迟到真收敛获胜：failed → converged，bool=true（种子救援路径语义）。
|
||
let changed3 = db
|
||
.record_task_report(
|
||
&mk_report(&name, ¶ms, TaskStatus::Completed, true),
|
||
"wf_g",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert!(changed3, "failed → converged 应迁移");
|
||
assert_eq!(
|
||
db.get_grid_point_status(&name, "wf_g")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"completed"
|
||
);
|
||
|
||
// 迟到失败报告打在 converged 点 → 吸收，bool=false，状态不变（核心回归）。
|
||
let changed4 = db
|
||
.record_task_report(
|
||
&mk_report(&name, ¶ms, TaskStatus::Failed, false),
|
||
"wf_g",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert!(!changed4, "converged 点不可被迟到失败报告翻黑");
|
||
assert_eq!(
|
||
db.get_grid_point_status(&name, "wf_g")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"completed"
|
||
);
|
||
|
||
// 重复成功报告 → 吸收，bool=false（阶段归因不被覆盖）。
|
||
let changed5 = db
|
||
.record_task_report(
|
||
&mk_report(&name, ¶ms, TaskStatus::Completed, true),
|
||
"wf_g",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert!(!changed5, "converged 点应吸收重复成功报告");
|
||
|
||
// queued → failed 生效（真实领用计算过的任务，如回收器重置后的迟到上报）。
|
||
let params2 = GridPointParams {
|
||
teff: 40000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name2 = params2.model_name();
|
||
db.upsert_grid_point(¶ms2, 0, "wf_g").await.unwrap();
|
||
db.update_grid_status(&name2, GridPointStatus::Queued, "wf_g")
|
||
.await
|
||
.unwrap();
|
||
let changed6 = db
|
||
.record_task_report(
|
||
&mk_report(&name2, ¶ms2, TaskStatus::Failed, false),
|
||
"wf_g",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert!(changed6, "queued → failed 应迁移");
|
||
assert_eq!(
|
||
db.get_grid_point_status(&name2, "wf_g")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"failed"
|
||
);
|
||
}
|
||
|
||
/// H2 回归：并发重复上报同一 task 的结算去重守卫。
|
||
///
|
||
/// 场景：report1 首次结算把点置 failed 并触发回退（回退把点 requeue 回 queued）；
|
||
/// 同一 task 的 report2 若在 report1 回退后到达 DB 层，旧代码的失败分支
|
||
/// `WHERE status NOT IN ('completed','failed')` 会命中 queued 行 → 把点再翻回 failed、
|
||
/// 返回 changed=true → 重复触发 seed_step 回退。修复后 report2 因 tasks 行已终态
|
||
/// 被 H2 守卫吸收（changed=false），点保持 queued 不被翻黑。
|
||
#[tokio::test]
|
||
async fn test_record_task_report_duplicate_task_settlement_absorbed() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("h2_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
|
||
// 派发一个任务（tasks 行 status='pending'）。
|
||
let task_id = Uuid::new_v4();
|
||
db.insert_task(&common::models::TaskSpec {
|
||
task_id,
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
timeout_sec: 7200,
|
||
workflow_name: Some("wf_h2".to_string()),
|
||
wave: 0,
|
||
..Default::default()
|
||
})
|
||
.await
|
||
.unwrap();
|
||
db.upsert_grid_point(¶ms, 0, "wf_h2").await.unwrap();
|
||
assert!(db.mark_grid_point_running(&name, "wf_h2").await.unwrap());
|
||
|
||
let mk_report_for = |task_id: Uuid, status: TaskStatus, converged: bool| -> TaskReport {
|
||
TaskReport {
|
||
task_id,
|
||
point_name: name.clone(),
|
||
params: Some(params.clone()),
|
||
node_id: "node-h2".to_string(),
|
||
status,
|
||
result_valid: converged,
|
||
max_relc: Some(0.0001),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 15.0,
|
||
error_message: None,
|
||
summary_json: "{}".to_string(),
|
||
failed_stage: None,
|
||
}
|
||
};
|
||
|
||
// report1：首次结算 → changed=true，点置 failed。
|
||
let changed1 = db
|
||
.record_task_report(&mk_report_for(task_id, TaskStatus::Failed, false), "wf_h2")
|
||
.await
|
||
.unwrap();
|
||
assert!(changed1, "running → failed 应迁移");
|
||
assert_eq!(
|
||
db.get_grid_point_status(&name, "wf_h2")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"failed"
|
||
);
|
||
|
||
// 模拟 report1 触发回退后把点 requeue 回 queued（trigger_strategy_fallback 的正常动作）。
|
||
db.update_grid_status(&name, GridPointStatus::Queued, "wf_h2")
|
||
.await
|
||
.unwrap();
|
||
|
||
// report2：同一 task 的重复失败报告 → H2 守卫吸收（tasks 行已终态），changed=false，
|
||
// 点保持 queued（不被翻回 failed，从而不重复触发回退）。
|
||
let changed2 = db
|
||
.record_task_report(&mk_report_for(task_id, TaskStatus::Failed, false), "wf_h2")
|
||
.await
|
||
.unwrap();
|
||
assert!(!changed2, "同一 task 的重复结算应被吸收");
|
||
assert_eq!(
|
||
db.get_grid_point_status(&name, "wf_h2")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"queued",
|
||
"重复报告不得把回退后的 queued 点再翻回 failed"
|
||
);
|
||
}
|
||
|
||
/// H1 活锁修复：pending_strategies 标记的读写清（set/get/clear）供调度路径识别
|
||
/// 「该点已失败过 cold_run、正在等种子」→ 重派用剩余链而非完整 YAML 链。
|
||
#[tokio::test]
|
||
async fn test_pending_strategies_marker_roundtrip() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("ps_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
let params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_grid_point(¶ms, 0, "wf_ps").await.unwrap();
|
||
|
||
// 初始无标记。
|
||
assert_eq!(
|
||
db.get_pending_strategies(&name, "wf_ps").await.unwrap(),
|
||
None
|
||
);
|
||
// set → 读回
|
||
db.set_pending_strategies(&name, "wf_ps", r#"["seed_step"]"#)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
db.get_pending_strategies(&name, "wf_ps").await.unwrap(),
|
||
Some(r#"["seed_step"]"#.to_string())
|
||
);
|
||
// clear → 回到 None
|
||
db.clear_pending_strategies(&name, "wf_ps").await.unwrap();
|
||
assert_eq!(
|
||
db.get_pending_strategies(&name, "wf_ps").await.unwrap(),
|
||
None
|
||
);
|
||
}
|
||
|
||
/// P1（Phase 1）：结算补落阶段信息——failed_stage 精确归因 + summary_json 全量透传。
|
||
/// 覆盖：半失败（大气成+光谱败 → failed_stage='synspec'）、旧节点（failed_stage=None
|
||
/// → COALESCE 兜底 'tlusty'）、错误路径 summary_json={"error":...} 透传、
|
||
/// list_point_attempts 返回新字段。
|
||
#[tokio::test]
|
||
async fn test_task_report_persists_stage_info() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("p1_stage.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_grid_point(¶ms, 0, "wf_p1").await.unwrap();
|
||
|
||
// 预置两条 tasks 行（report 结算 UPDATE 需命中行才写入阶段信息）。
|
||
let mk_spec = |task_id: Uuid| common::models::TaskSpec {
|
||
task_id,
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
seed_point_name: None,
|
||
timeout_sec: 7200,
|
||
workflow_name: Some("wf_p1".to_string()),
|
||
wave: 0,
|
||
tlusty_config: common::models::PhaseConfig::default_tlusty(),
|
||
synspec_config: common::models::PhaseConfig::default_synspec(),
|
||
synspec_params: None,
|
||
tlusty_chain_params: None,
|
||
seed_chain_params: None,
|
||
tlusty_input_params: None,
|
||
atmosphere_ref: None,
|
||
energy_tolerance: None,
|
||
temp_max_factor: None,
|
||
temp_floor: None,
|
||
temp_ceiling: None,
|
||
emflux_tolerance: None,
|
||
convergence_min_ratio: None,
|
||
bfac_max: None,
|
||
bfac_min: None,
|
||
linelist: None,
|
||
};
|
||
let syn_task = Uuid::new_v4();
|
||
let old_task = Uuid::new_v4();
|
||
db.insert_task(&mk_spec(syn_task)).await.unwrap();
|
||
db.insert_task(&mk_spec(old_task)).await.unwrap();
|
||
|
||
// ① 半失败：大气收敛 + synspec 失败 → failed_stage='synspec'，summary_json 全量透传。
|
||
db.record_task_report(
|
||
&TaskReport {
|
||
task_id: syn_task,
|
||
point_name: name.clone(),
|
||
params: Some(params.clone()),
|
||
node_id: "node-p1".to_string(),
|
||
status: TaskStatus::Failed,
|
||
result_valid: true,
|
||
max_relc: Some(0.0001),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 20.0,
|
||
error_message: Some("synspec: SEGMENTATION_FAULT".to_string()),
|
||
summary_json: r#"{"name":"t35000_g5.5_he-1_c-2_n-2_o-2","converged":true,"synspec_rc":139,"synspec_error":"SEGMENTATION_FAULT","synspec_sec":3.2}"#.to_string(),
|
||
failed_stage: Some("synspec".to_string()),
|
||
},
|
||
"wf_p1",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
// ② 旧节点：failed_stage=None → COALESCE 兜底 'tlusty'；错误路径 {"error":...} 透传。
|
||
db.record_task_report(
|
||
&TaskReport {
|
||
task_id: old_task,
|
||
point_name: name.clone(),
|
||
params: Some(params.clone()),
|
||
node_id: "node-old".to_string(),
|
||
status: TaskStatus::Failed,
|
||
result_valid: false,
|
||
max_relc: Some(0.5),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 10.0,
|
||
error_message: Some("tlusty diverged".to_string()),
|
||
summary_json: r#"{"error":"tlusty: not converged after NITER"}"#.to_string(),
|
||
failed_stage: None,
|
||
},
|
||
"wf_p1",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
// ③ list_point_attempts 返回新字段，且语义正确。
|
||
let attempts = db.list_point_attempts("wf_p1", &name).await.unwrap();
|
||
assert_eq!(attempts.len(), 2);
|
||
let syn = attempts
|
||
.iter()
|
||
.find(|a| a.task_id == syn_task.to_string())
|
||
.unwrap();
|
||
let old = attempts
|
||
.iter()
|
||
.find(|a| a.task_id == old_task.to_string())
|
||
.unwrap();
|
||
assert_eq!(syn.failed_stage.as_deref(), Some("synspec"));
|
||
assert_eq!(
|
||
old.failed_stage.as_deref(),
|
||
Some("tlusty"),
|
||
"旧节点缺省归因应兜底 tlusty"
|
||
);
|
||
assert!(syn
|
||
.summary_json
|
||
.as_deref()
|
||
.unwrap_or("")
|
||
.contains("synspec_rc"));
|
||
assert!(old
|
||
.summary_json
|
||
.as_deref()
|
||
.unwrap_or("")
|
||
.contains("\"error\""));
|
||
}
|
||
|
||
/// P1（Phase 1）：旧库升级——手工构造缺 failed_stage/summary_json 的旧 tasks schema 库，
|
||
/// 经 Database::new（bootstrap + M1 迁移）后：新列补齐、历史数据保留、版本推进到最新。
|
||
/// 对应设计 §2.4「旧库升级」测试用例。
|
||
#[tokio::test]
|
||
async fn test_old_schema_upgrade_preserves_data_and_adds_columns() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db_path = temp_dir
|
||
.path()
|
||
.join("old_schema.db")
|
||
.to_string_lossy()
|
||
.to_string();
|
||
// 手工构造旧 schema 库：tasks 无 failed_stage/summary_json，含一条历史数据。
|
||
{
|
||
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||
conn.execute_batch(
|
||
"CREATE TABLE tasks (
|
||
task_id TEXT PRIMARY KEY,
|
||
point_name TEXT NOT NULL,
|
||
node_id TEXT,
|
||
task_type TEXT NOT NULL,
|
||
seed_point_name TEXT,
|
||
status TEXT NOT NULL DEFAULT 'pending',
|
||
max_relc REAL,
|
||
atmosphere_has_nan BOOLEAN NOT NULL DEFAULT 0,
|
||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||
created_at DATETIME NOT NULL,
|
||
started_at DATETIME,
|
||
completed_at DATETIME,
|
||
error_message TEXT,
|
||
workflow_name TEXT
|
||
);
|
||
INSERT INTO tasks (task_id, point_name, node_id, task_type, status, retry_count, created_at, workflow_name)
|
||
VALUES ('t-old-1', 'p1', 'node-1', 'cold_run', 'failed', 0, datetime('now'), '__legacy__');",
|
||
)
|
||
.unwrap();
|
||
}
|
||
// bootstrap + M1 迁移。
|
||
let db = Database::new(&db_path).await.unwrap();
|
||
|
||
// 数据保留 + 新列补齐（旧行新列值 NULL，不破坏既有行）。
|
||
let attempts = db.list_point_attempts("__legacy__", "p1").await.unwrap();
|
||
assert_eq!(attempts.len(), 1);
|
||
assert_eq!(attempts[0].task_id, "t-old-1");
|
||
assert!(attempts[0].failed_stage.is_none(), "旧行新列应留 NULL");
|
||
assert!(attempts[0].summary_json.is_none(), "旧行新列应留 NULL");
|
||
|
||
// M6（Phase 6）：旧 schema 的 task_type 死列被删除。
|
||
let conn = db.pool.get().unwrap();
|
||
let mut stmt = conn.prepare("PRAGMA table_info(tasks)").unwrap();
|
||
let cols: Vec<String> = stmt
|
||
.query_map([], |r| r.get::<_, String>(1))
|
||
.unwrap()
|
||
.filter_map(Result::ok)
|
||
.collect();
|
||
assert!(
|
||
!cols.iter().any(|c| c == "task_type"),
|
||
"旧 schema 的 task_type 列应被 M6 删除"
|
||
);
|
||
|
||
// 版本推进到最新（M1 detect 命中 → 跳过 up，仅推进 user_version）。
|
||
let ver: u32 = conn
|
||
.pragma_query_value(None, "user_version", |r| r.get(0))
|
||
.unwrap();
|
||
assert_eq!(
|
||
ver,
|
||
crate::migrations::MIGRATIONS
|
||
.last()
|
||
.map(|m| m.version)
|
||
.unwrap_or(0)
|
||
);
|
||
}
|
||
|
||
/// P2（Phase 2）：`idx_tasks_wf_status_created` 覆盖索引——`COUNT(*) WHERE workflow_name=?`
|
||
/// 应走 COVERING INDEX 而非全表扫（消除详情页统计的 N+1 全表扫）。M2 迁移负责建索引。
|
||
#[tokio::test]
|
||
async fn test_tasks_wf_count_uses_covering_index() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("idx_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
let conn = db.pool.get().unwrap();
|
||
let plan: String = conn
|
||
.query_row(
|
||
"EXPLAIN QUERY PLAN SELECT COUNT(*) FROM tasks WHERE workflow_name = ?1",
|
||
params!["wf_idx"],
|
||
|r| r.get(3),
|
||
)
|
||
.unwrap();
|
||
assert!(
|
||
plan.contains("idx_tasks_wf_status_created"),
|
||
"COUNT 应按 workflow_name 走覆盖索引，实际计划：{plan}"
|
||
);
|
||
}
|
||
|
||
/// P3（Phase 3）：ROW_NUMBER() 窗口替代逐行相关子查询后的结果集等价性验证。
|
||
/// 1. 多点多尝试 → 每点取最新任务（completed_at DESC 优先）；
|
||
/// 2. 边界：点只有未完成任务（completed_at 全 NULL）→ 取最新 created_at（与旧子查询同语义）；
|
||
/// 3. 从未派发点 → last_* 全 NULL；
|
||
/// 4. 分页 total 计数不受 JOIN 影响（= 网格点数）。
|
||
#[tokio::test]
|
||
async fn test_list_workflow_points_window_equivalence() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("p3_win.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let mk = |teff: f64| GridPointParams {
|
||
teff: teff.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let pa = mk(35000.0);
|
||
let pb = mk(36000.0);
|
||
let pc = mk(37000.0);
|
||
db.upsert_grid_point(&pa, 0, "wf_p3").await.unwrap();
|
||
db.upsert_grid_point(&pb, 0, "wf_p3").await.unwrap();
|
||
db.upsert_grid_point(&pc, 0, "wf_p3").await.unwrap();
|
||
|
||
// 裸 SQL 插 tasks 行以精确控制 created_at / completed_at / elapsed_sec / node_id
|
||
//（NOT NULL 列补齐）。PointRow.status 是 grid_points.status（网格点状态），不是任务状态，
|
||
// 故用 node_id 区分"窗口选了哪条任务行"。
|
||
let insert_task_raw = |conn: &rusqlite::Connection,
|
||
task_id: &str,
|
||
point: &str,
|
||
node: &str,
|
||
created: &str,
|
||
completed: Option<&str>,
|
||
elapsed: Option<f64>,
|
||
max_relc: Option<f64>| {
|
||
conn.execute(
|
||
"INSERT INTO tasks (task_id, point_name, status, retry_count, created_at, completed_at, workflow_name, node_id, elapsed_sec, tlusty_strategies, synspec_strategies, max_relc)
|
||
VALUES (?1, ?2, 'completed', 0, ?4, ?5, 'wf_p3', ?3, ?6, '[\"cold_run\"]', '[\"standard\"]', ?7)",
|
||
rusqlite::params![task_id, point, node, created, completed, elapsed, max_relc],
|
||
)
|
||
.unwrap();
|
||
};
|
||
let conn = db.pool.get().unwrap();
|
||
// 点 A：旧失败（早）+ 新成功（晚）→ 取新（completed_at 晚）。
|
||
insert_task_raw(
|
||
&conn,
|
||
"a1",
|
||
&pa.model_name(),
|
||
"node-a1",
|
||
"2024-01-01 00:00:00",
|
||
Some("2024-01-02 00:00:00"),
|
||
Some(5.0),
|
||
Some(0.001),
|
||
);
|
||
insert_task_raw(
|
||
&conn,
|
||
"a2",
|
||
&pa.model_name(),
|
||
"node-a2",
|
||
"2024-01-03 00:00:00",
|
||
Some("2024-01-04 00:00:00"),
|
||
Some(3.0),
|
||
Some(0.0001),
|
||
);
|
||
// 点 B：两个未完成任务（completed_at 均 NULL）→ 取 created_at 新（b2）。
|
||
insert_task_raw(
|
||
&conn,
|
||
"b1",
|
||
&pb.model_name(),
|
||
"node-b1",
|
||
"2024-01-01 00:00:00",
|
||
None,
|
||
None,
|
||
None,
|
||
);
|
||
insert_task_raw(
|
||
&conn,
|
||
"b2",
|
||
&pb.model_name(),
|
||
"node-b2",
|
||
"2024-01-02 00:00:00",
|
||
None,
|
||
None,
|
||
None,
|
||
);
|
||
// 点 C：无任务。
|
||
|
||
let filter = crate::db::PointFilter::default();
|
||
let (total, points) = db.list_workflow_points("wf_p3", &filter).await.unwrap();
|
||
assert_eq!(total, 3, "total 计数 = 网格点数，不受 JOIN 影响");
|
||
|
||
let a = points.iter().find(|p| p.name == pa.model_name()).unwrap();
|
||
assert_eq!(a.last_max_relc, Some(0.0001), "应取 completed_at 更晚的 a2");
|
||
assert_eq!(a.last_elapsed_sec, Some(3.0));
|
||
assert_eq!(a.node_id.as_deref(), Some("node-a2"));
|
||
|
||
let b = points.iter().find(|p| p.name == pb.model_name()).unwrap();
|
||
assert_eq!(
|
||
b.node_id.as_deref(),
|
||
Some("node-b2"),
|
||
"completed_at 全 NULL 边界：应取 created_at 更新的 b2"
|
||
);
|
||
assert_eq!(b.last_max_relc, None);
|
||
|
||
let c = points.iter().find(|p| p.name == pc.model_name()).unwrap();
|
||
assert!(c.last_max_relc.is_none(), "从未派发点 last_* 全 NULL");
|
||
assert!(c.last_completed_at.is_none());
|
||
|
||
// 单点端点等价。
|
||
let row = db
|
||
.get_workflow_point_row("wf_p3", &pa.model_name())
|
||
.await
|
||
.unwrap();
|
||
let row = row.expect("点 A 应存在");
|
||
assert_eq!(row.last_max_relc, Some(0.0001));
|
||
let none_row = db
|
||
.get_workflow_point_row("wf_p3", "t40400_g5.5_he-1_c-2_n-2_o-2")
|
||
.await
|
||
.unwrap();
|
||
assert!(none_row.is_none(), "不存在的点返回 None");
|
||
}
|
||
|
||
/// 阶段归因（修复审查 #4 + Phase 6 派生口径 + P9 拆分）：synspec-only 任务
|
||
/// （tlusty_enabled=0）光谱归因取 `synspec_strategies[0]`（如 "standard"）；
|
||
/// tlusty 启用任务大气归因取 `tlusty_strategies[0]`（task_type 列已删，全派生）。同时验证
|
||
/// json_extract 依赖（rusqlite bundled SQLite 内建 JSON1）在生产 SQL 中可用。
|
||
#[tokio::test]
|
||
async fn test_record_task_report_success_method_synspec_only() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("sm_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_grid_point(¶ms, 0, "wf_sm").await.unwrap();
|
||
|
||
// 场景 B 任务：tlusty 关闭、synspec 启用（策略链 [standard]）。
|
||
let task = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
tlusty_config: PhaseConfig {
|
||
enabled: false,
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
synspec_config: PhaseConfig {
|
||
strategies: vec!["standard".to_string()],
|
||
..PhaseConfig::default_synspec()
|
||
},
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&task).await.unwrap();
|
||
|
||
let report = TaskReport {
|
||
task_id: task.task_id,
|
||
point_name: name.clone(),
|
||
params: Some(params.clone()),
|
||
node_id: "node-test".to_string(),
|
||
status: TaskStatus::Completed,
|
||
result_valid: true,
|
||
max_relc: Some(0.0001),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 15.0,
|
||
error_message: None,
|
||
summary_json: "{}".to_string(),
|
||
failed_stage: None,
|
||
};
|
||
db.record_task_report(&report, "wf_sm").await.unwrap();
|
||
|
||
let sm: String = {
|
||
let pool = db.pool.clone();
|
||
let n = name.clone();
|
||
let wf = "wf_sm".to_string();
|
||
tokio::task::spawn_blocking(move || -> String {
|
||
let conn = pool.get().unwrap();
|
||
conn.query_row(
|
||
"SELECT synspec_success_method FROM grid_points WHERE name = ?1 AND workflow_name = ?2",
|
||
rusqlite::params![n, wf],
|
||
|r| r.get(0),
|
||
)
|
||
.unwrap()
|
||
})
|
||
.await
|
||
.unwrap()
|
||
};
|
||
assert_eq!(
|
||
sm, "standard",
|
||
"synspec-only 任务光谱归因应取 synspec 链首项（tlusty_success_method 为 NULL）"
|
||
);
|
||
|
||
// tlusty 启用任务对照：仍归因 task_type（cold_run）。
|
||
let params2 = GridPointParams {
|
||
teff: 40000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name2 = params2.model_name();
|
||
db.upsert_grid_point(¶ms2, 0, "wf_sm").await.unwrap();
|
||
let task2 = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: name2.clone(),
|
||
params: params2.clone(),
|
||
// tlusty 启用：策略链首项 seed_step（派生归因应取它，等价旧 task_type=SeedStep）。
|
||
tlusty_config: PhaseConfig {
|
||
strategies: vec!["seed_step".to_string()],
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&task2).await.unwrap();
|
||
let report2 = TaskReport {
|
||
task_id: task2.task_id,
|
||
point_name: name2.clone(),
|
||
params: Some(params2.clone()),
|
||
node_id: "node-test".to_string(),
|
||
status: TaskStatus::Completed,
|
||
result_valid: true,
|
||
max_relc: Some(0.0001),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 15.0,
|
||
error_message: None,
|
||
summary_json: "{}".to_string(),
|
||
failed_stage: None,
|
||
};
|
||
db.record_task_report(&report2, "wf_sm").await.unwrap();
|
||
let sm2: String = {
|
||
let pool = db.pool.clone();
|
||
let n = name2.clone();
|
||
let wf = "wf_sm".to_string();
|
||
tokio::task::spawn_blocking(move || -> String {
|
||
let conn = pool.get().unwrap();
|
||
conn.query_row(
|
||
"SELECT tlusty_success_method FROM grid_points WHERE name = ?1 AND workflow_name = ?2",
|
||
rusqlite::params![n, wf],
|
||
|r| r.get(0),
|
||
)
|
||
.unwrap()
|
||
})
|
||
.await
|
||
.unwrap()
|
||
};
|
||
assert_eq!(
|
||
sm2, "seed_step",
|
||
"tlusty 启用任务按 strategies[0] 归因（派生）"
|
||
);
|
||
}
|
||
|
||
/// P6（Phase 5a）：`synspec_success_method` 归因列——synspec 收敛点记录光谱策略，
|
||
/// TLUSTY-only 成功保持 NULL；`get_grid_summary_stats` 的 `synspec_converged` 桶正确计数。
|
||
#[tokio::test]
|
||
async fn test_synspec_success_method_and_stats_bucket() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("synspec5a.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let mk = |teff: f64| GridPointParams {
|
||
teff: teff.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
|
||
// ① synspec-only：tlusty 关、synspec 开（[standard]）→ 归因 standard。
|
||
let p_syn_only = mk(35000.0);
|
||
db.upsert_grid_point(&p_syn_only, 0, "wf_5a").await.unwrap();
|
||
let spec1 = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: p_syn_only.model_name(),
|
||
params: p_syn_only.clone(),
|
||
tlusty_config: PhaseConfig {
|
||
enabled: false,
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
synspec_config: PhaseConfig {
|
||
strategies: vec!["standard".to_string()],
|
||
..PhaseConfig::default_synspec()
|
||
},
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&spec1).await.unwrap();
|
||
db.record_task_report(
|
||
&TaskReport {
|
||
task_id: spec1.task_id,
|
||
point_name: p_syn_only.model_name(),
|
||
params: Some(p_syn_only.clone()),
|
||
node_id: "node".to_string(),
|
||
status: TaskStatus::Completed,
|
||
result_valid: true,
|
||
max_relc: Some(0.0001),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 10.0,
|
||
error_message: None,
|
||
summary_json: r#"{"synspec_rc":0,"synspec_error":null}"#.to_string(),
|
||
failed_stage: None,
|
||
},
|
||
"wf_5a",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
// ② 双阶段：tlusty 开 + synspec 开（[standard]）→ synspec_success_method=standard。
|
||
let p_both = mk(36000.0);
|
||
db.upsert_grid_point(&p_both, 0, "wf_5a").await.unwrap();
|
||
let spec2 = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: p_both.model_name(),
|
||
params: p_both.clone(),
|
||
synspec_config: PhaseConfig {
|
||
strategies: vec!["standard".to_string()],
|
||
..PhaseConfig::default_synspec()
|
||
},
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&spec2).await.unwrap();
|
||
db.record_task_report(
|
||
&TaskReport {
|
||
task_id: spec2.task_id,
|
||
point_name: p_both.model_name(),
|
||
params: Some(p_both.clone()),
|
||
node_id: "node".to_string(),
|
||
status: TaskStatus::Completed,
|
||
result_valid: true,
|
||
max_relc: Some(0.0001),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 10.0,
|
||
error_message: None,
|
||
summary_json: r#"{"synspec_rc":0,"synspec_error":null}"#.to_string(),
|
||
failed_stage: None,
|
||
},
|
||
"wf_5a",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
// ③ TLUSTY-only：synspec 关 → synspec_success_method 保持 NULL。
|
||
let p_tlusty_only = mk(37000.0);
|
||
db.upsert_grid_point(&p_tlusty_only, 0, "wf_5a")
|
||
.await
|
||
.unwrap();
|
||
let spec3 = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: p_tlusty_only.model_name(),
|
||
params: p_tlusty_only.clone(),
|
||
synspec_config: PhaseConfig {
|
||
enabled: false,
|
||
..PhaseConfig::default_synspec()
|
||
},
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&spec3).await.unwrap();
|
||
db.record_task_report(
|
||
&TaskReport {
|
||
task_id: spec3.task_id,
|
||
point_name: p_tlusty_only.model_name(),
|
||
params: Some(p_tlusty_only.clone()),
|
||
node_id: "node".to_string(),
|
||
status: TaskStatus::Completed,
|
||
result_valid: true,
|
||
max_relc: Some(0.0001),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 10.0,
|
||
error_message: None,
|
||
summary_json: "{}".to_string(),
|
||
failed_stage: None,
|
||
},
|
||
"wf_5a",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
// 统计桶：①②计入 synspec_converged（2），③不计入。
|
||
let stats = db.get_grid_summary_stats(Some("wf_5a")).await.unwrap();
|
||
assert_eq!(stats["completed"], 3);
|
||
assert_eq!(
|
||
stats["synspec_converged"], 2,
|
||
"synspec-only + 双阶段共 2 个光谱收敛点"
|
||
);
|
||
|
||
// 归因列值抽查（返回 future，调用处 await）。
|
||
let read_col = |name: String, col: String| {
|
||
let pool = db.pool.clone();
|
||
async move {
|
||
tokio::task::spawn_blocking(move || -> Option<String> {
|
||
let conn = pool.get().unwrap();
|
||
// `col` 为测试内写死的列名常量（非用户输入）。
|
||
conn.query_row(
|
||
&format!("SELECT {col} FROM grid_points WHERE name = ?1 AND workflow_name = 'wf_5a'"),
|
||
rusqlite::params![name],
|
||
|r| r.get(0),
|
||
)
|
||
.ok()
|
||
})
|
||
.await
|
||
.unwrap()
|
||
}
|
||
};
|
||
assert_eq!(
|
||
read_col(
|
||
p_syn_only.model_name(),
|
||
"synspec_success_method".to_string()
|
||
)
|
||
.await,
|
||
Some("standard".to_string())
|
||
);
|
||
assert_eq!(
|
||
read_col(p_both.model_name(), "synspec_success_method".to_string()).await,
|
||
Some("standard".to_string())
|
||
);
|
||
assert_eq!(
|
||
read_col(
|
||
p_tlusty_only.model_name(),
|
||
"synspec_success_method".to_string()
|
||
)
|
||
.await,
|
||
None,
|
||
"TLUSTY-only 成功不落 synspec 归因"
|
||
);
|
||
|
||
// P9 拆分：tlusty_success_method = TLUSTY 阶段策略（tlusty 禁用为 NULL）。
|
||
assert_eq!(
|
||
read_col(p_syn_only.model_name(), "tlusty_success_method".to_string()).await,
|
||
None,
|
||
"synspec-only 任务 tlusty 禁用 → tlusty_success_method 保持 NULL"
|
||
);
|
||
assert_eq!(
|
||
read_col(p_both.model_name(), "tlusty_success_method".to_string()).await,
|
||
Some("cold_run".to_string()),
|
||
"双阶段任务 tlusty_success_method 取 tlusty 链首项"
|
||
);
|
||
assert_eq!(
|
||
read_col(
|
||
p_tlusty_only.model_name(),
|
||
"tlusty_success_method".to_string()
|
||
)
|
||
.await,
|
||
Some("cold_run".to_string()),
|
||
"TLUSTY-only 任务 tlusty_success_method 取 tlusty 链首项"
|
||
);
|
||
}
|
||
|
||
/// P6（Phase 5b）：阶段状态列——半失败点（大气收敛+光谱失败）可查 tlusty_status/synspec_status。
|
||
/// 覆盖：半失败（converged/failed）、全成功（converged/converged）、大气失败（failed/pending）、
|
||
/// 半失败重试的 synspec-only 成功（tlusty_status 守卫保留 converged，仅 synspec 侧流转）。
|
||
#[tokio::test]
|
||
async fn test_grid_point_stage_status_half_failure() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("p5b_stage.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let mk = |teff: f64| GridPointParams {
|
||
teff: teff.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let read_cols = |name: String, wf: String| {
|
||
let pool = db.pool.clone();
|
||
async move {
|
||
tokio::task::spawn_blocking(move || -> (Option<String>, Option<String>) {
|
||
let conn = pool.get().unwrap();
|
||
let mut stmt = conn
|
||
.prepare("SELECT tlusty_status, synspec_status FROM grid_points WHERE name = ?1 AND workflow_name = ?2")
|
||
.unwrap();
|
||
stmt.query_row(rusqlite::params![name, wf], |r| {
|
||
Ok((r.get::<_, Option<String>>(0)?, r.get::<_, Option<String>>(1)?))
|
||
})
|
||
.unwrap_or((None, None))
|
||
})
|
||
.await
|
||
.unwrap()
|
||
}
|
||
};
|
||
let report = |task_id: uuid::Uuid,
|
||
name: &str,
|
||
params: &GridPointParams,
|
||
status: TaskStatus,
|
||
converged: bool| TaskReport {
|
||
task_id,
|
||
point_name: name.to_string(),
|
||
params: Some(params.clone()),
|
||
node_id: "node".to_string(),
|
||
status,
|
||
result_valid: converged,
|
||
max_relc: Some(0.0001),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 10.0,
|
||
error_message: None,
|
||
summary_json: "{}".to_string(),
|
||
failed_stage: None,
|
||
};
|
||
|
||
// ① 半失败：tlusty 收敛 + synspec 失败。
|
||
let p_half = mk(35000.0);
|
||
db.upsert_grid_point(&p_half, 0, "wf_5b").await.unwrap();
|
||
let spec_half = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: p_half.model_name(),
|
||
params: p_half.clone(),
|
||
synspec_config: PhaseConfig {
|
||
strategies: vec!["standard".to_string()],
|
||
..PhaseConfig::default_synspec()
|
||
},
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&spec_half).await.unwrap();
|
||
db.record_task_report(
|
||
&report(
|
||
spec_half.task_id,
|
||
&p_half.model_name(),
|
||
&p_half,
|
||
TaskStatus::Failed,
|
||
true,
|
||
),
|
||
"wf_5b",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
read_cols(p_half.model_name(), "wf_5b".to_string()).await,
|
||
(Some("converged".to_string()), Some("failed".to_string())),
|
||
"半失败：tlusty=converged, synspec=failed"
|
||
);
|
||
assert_eq!(
|
||
db.get_grid_point_status(&p_half.model_name(), "wf_5b")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"failed"
|
||
);
|
||
|
||
// ② 生命周期守卫（审查补测）：半失败点被重新 claim/running（synspec 重试在途）时，
|
||
// tlusty_status='converged' 必须保留（设计打开项 #2），synspec_status 自由流转。
|
||
db.update_grid_status(&p_half.model_name(), GridPointStatus::Pending, "wf_5b")
|
||
.await
|
||
.unwrap();
|
||
let claimed = db.claim_pending_grid_points(100, "wf_5b").await.unwrap();
|
||
assert!(claimed.iter().any(|(n, _, _)| n == &p_half.model_name()));
|
||
assert_eq!(
|
||
read_cols(p_half.model_name(), "wf_5b".to_string()).await,
|
||
(Some("converged".to_string()), Some("queued".to_string())),
|
||
"claim 守卫：tlusty_status 保留 converged，synspec 流转为 queued"
|
||
);
|
||
assert!(db
|
||
.mark_grid_point_running(&p_half.model_name(), "wf_5b")
|
||
.await
|
||
.unwrap());
|
||
assert_eq!(
|
||
read_cols(p_half.model_name(), "wf_5b".to_string()).await,
|
||
(Some("converged".to_string()), Some("running".to_string())),
|
||
"running 守卫：tlusty_status 保留 converged，synspec 流转为 running"
|
||
);
|
||
|
||
// ③ 半失败重试（synspec-only 成功）：tlusty_status 守卫保留 converged，synspec→converged。
|
||
let retry = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: p_half.model_name(),
|
||
params: p_half.clone(),
|
||
tlusty_config: PhaseConfig {
|
||
enabled: false,
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
synspec_config: PhaseConfig {
|
||
strategies: vec!["standard".to_string()],
|
||
..PhaseConfig::default_synspec()
|
||
},
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&retry).await.unwrap();
|
||
db.record_task_report(
|
||
&report(
|
||
retry.task_id,
|
||
&p_half.model_name(),
|
||
&p_half,
|
||
TaskStatus::Completed,
|
||
true,
|
||
),
|
||
"wf_5b",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
read_cols(p_half.model_name(), "wf_5b".to_string()).await,
|
||
(Some("converged".to_string()), Some("converged".to_string())),
|
||
"重试成功：tlusty 守卫保留 converged，synspec 流转为 converged"
|
||
);
|
||
|
||
// ④ 全成功：两阶段均 converged。
|
||
let p_ok = mk(36000.0);
|
||
db.upsert_grid_point(&p_ok, 0, "wf_5b").await.unwrap();
|
||
let spec_ok = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: p_ok.model_name(),
|
||
params: p_ok.clone(),
|
||
synspec_config: PhaseConfig {
|
||
strategies: vec!["standard".to_string()],
|
||
..PhaseConfig::default_synspec()
|
||
},
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&spec_ok).await.unwrap();
|
||
db.record_task_report(
|
||
&report(
|
||
spec_ok.task_id,
|
||
&p_ok.model_name(),
|
||
&p_ok,
|
||
TaskStatus::Completed,
|
||
true,
|
||
),
|
||
"wf_5b",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
read_cols(p_ok.model_name(), "wf_5b".to_string()).await,
|
||
(Some("converged".to_string()), Some("converged".to_string()))
|
||
);
|
||
|
||
// ⑤ 大气失败：tlusty=failed, synspec=pending（未运行）。
|
||
let p_atmo = mk(37000.0);
|
||
db.upsert_grid_point(&p_atmo, 0, "wf_5b").await.unwrap();
|
||
let spec_atmo = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: p_atmo.model_name(),
|
||
params: p_atmo.clone(),
|
||
synspec_config: PhaseConfig {
|
||
strategies: vec!["standard".to_string()],
|
||
..PhaseConfig::default_synspec()
|
||
},
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&spec_atmo).await.unwrap();
|
||
db.record_task_report(
|
||
&report(
|
||
spec_atmo.task_id,
|
||
&p_atmo.model_name(),
|
||
&p_atmo,
|
||
TaskStatus::Failed,
|
||
false,
|
||
),
|
||
"wf_5b",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
read_cols(p_atmo.model_name(), "wf_5b".to_string()).await,
|
||
(Some("failed".to_string()), Some("pending".to_string())),
|
||
"大气失败：tlusty=failed, synspec=pending"
|
||
);
|
||
}
|
||
|
||
/// P9（Phase 7a + E 拆分）：逐点列表 `method=synspec_only` 过滤器——仅返回光谱专用收敛点
|
||
/// （tlusty 归因 NULL + synspec_success_method 落库）；冷启动/种子步进及双阶段点被排除。
|
||
/// 与徽章/parSets 的 SYNSPEC 档同口径。
|
||
#[tokio::test]
|
||
async fn test_list_points_synspec_method_filter() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("p7a_filter.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let mk = |teff: f64| GridPointParams {
|
||
teff: teff.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let p_syn = mk(35000.0);
|
||
let p_cold = mk(36000.0);
|
||
db.upsert_grid_point(&p_syn, 0, "wf_7a").await.unwrap();
|
||
db.upsert_grid_point(&p_cold, 0, "wf_7a").await.unwrap();
|
||
|
||
// 裸 SQL 直接设置归因列（模拟结算落库），避免完整 report 链路噪音。
|
||
// p_syn：synspec-only 收敛点（synspec_success_method = 光谱策略 standard，tlusty 禁用为 NULL）；
|
||
// p_cold：纯 TLUSTY 点（tlusty_success_method = seed_step，无光谱归因）。
|
||
{
|
||
let conn = db.pool.get().unwrap();
|
||
conn.execute(
|
||
"UPDATE grid_points SET status='completed', synspec_success_method='standard' WHERE name=?1 AND workflow_name='wf_7a'",
|
||
rusqlite::params![p_syn.model_name()],
|
||
)
|
||
.unwrap();
|
||
conn.execute(
|
||
"UPDATE grid_points SET status='completed', tlusty_success_method='seed_step' WHERE name=?1 AND workflow_name='wf_7a'",
|
||
rusqlite::params![p_cold.model_name()],
|
||
)
|
||
.unwrap();
|
||
}
|
||
|
||
// 无过滤：2 点。
|
||
let (total, all) = db
|
||
.list_workflow_points("wf_7a", &crate::db::PointFilter::default())
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(total, 2);
|
||
assert_eq!(all.len(), 2);
|
||
|
||
// method=synspec_only：仅光谱专用收敛点（tlusty 归因 NULL + synspec 落库）。
|
||
let (total_syn, syn_points) = db
|
||
.list_workflow_points(
|
||
"wf_7a",
|
||
&crate::db::PointFilter {
|
||
method: Some("synspec_only".to_string()),
|
||
..Default::default()
|
||
},
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(total_syn, 1, "synspec_only 过滤器只命中光谱专用点");
|
||
assert_eq!(syn_points[0].name, p_syn.model_name());
|
||
|
||
// method=seed_step：按 tlusty_success_method 过滤（TLUSTY 点不受 synspec_only 过滤器影响）。
|
||
let (_, cold_points) = db
|
||
.list_workflow_points(
|
||
"wf_7a",
|
||
&crate::db::PointFilter {
|
||
method: Some("seed_step".to_string()),
|
||
..Default::default()
|
||
},
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(cold_points.len(), 1);
|
||
assert_eq!(cold_points[0].name, p_cold.model_name());
|
||
}
|
||
|
||
/// 幂等上报吸收（api/task.rs 修复）：find_settled_task_claim 判定某任务是否已由本节点
|
||
/// 结算（node_id 归属匹配 + 终态）。响应丢失后节点重试上报时据此放行而非 403 误诊
|
||
/// 「token 失效」；未结算 / 非归属节点返回 None。
|
||
#[tokio::test]
|
||
async fn test_find_settled_task_claim_idempotent() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("stl_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_grid_point(¶ms, 0, "wf_stl").await.unwrap();
|
||
|
||
let task = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
// 生产调度器始终写入 workflow_name（多工作流分区）；测试对齐。
|
||
workflow_name: Some("wf_stl".to_string()),
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&task).await.unwrap();
|
||
|
||
// 未结算：node_id 未写 → None。
|
||
assert_eq!(
|
||
db.find_settled_task_claim(&task.task_id.to_string(), "node-x")
|
||
.await
|
||
.unwrap(),
|
||
None,
|
||
"未上报任务不应命中已结算判定"
|
||
);
|
||
|
||
// node-x 上报（converged，settled → status='completed', node_id='node-x'）。
|
||
let report = TaskReport {
|
||
task_id: task.task_id,
|
||
point_name: name.clone(),
|
||
params: Some(params.clone()),
|
||
node_id: "node-x".to_string(),
|
||
status: TaskStatus::Completed,
|
||
result_valid: true,
|
||
max_relc: Some(0.0001),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 15.0,
|
||
error_message: None,
|
||
summary_json: "{}".to_string(),
|
||
failed_stage: None,
|
||
};
|
||
db.record_task_report(&report, "wf_stl").await.unwrap();
|
||
|
||
// 已结算 + 归属匹配 → Some((point, workflow))。
|
||
let settled = db
|
||
.find_settled_task_claim(&task.task_id.to_string(), "node-x")
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
settled,
|
||
Some((name.clone(), Some("wf_stl".to_string()))),
|
||
"已结算任务应由归属节点命中（幂等重放依据）"
|
||
);
|
||
// 其它节点仍 None。
|
||
assert_eq!(
|
||
db.find_settled_task_claim(&task.task_id.to_string(), "node-y")
|
||
.await
|
||
.unwrap(),
|
||
None,
|
||
"非归属节点不得命中"
|
||
);
|
||
}
|
||
|
||
/// mark_grid_point_running 终态守卫：仅 pending/queued 可转 running，
|
||
/// converged/failed 不被迟到领用复活（2026-08-02 涡旋事故修复）。
|
||
#[tokio::test]
|
||
async fn test_mark_grid_point_running_guarded() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("mark_run_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let mk = |teff: f64| GridPointParams {
|
||
teff: teff.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
|
||
// pending → running ✓
|
||
let p1 = mk(30000.0);
|
||
db.upsert_grid_point(&p1, 0, "wf_m").await.unwrap();
|
||
assert!(db
|
||
.mark_grid_point_running(&p1.model_name(), "wf_m")
|
||
.await
|
||
.unwrap());
|
||
assert_eq!(
|
||
db.get_grid_point_status(&p1.model_name(), "wf_m")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"running"
|
||
);
|
||
|
||
// queued → running ✓
|
||
let p2 = mk(32000.0);
|
||
db.upsert_grid_point(&p2, 0, "wf_m").await.unwrap();
|
||
db.update_grid_status(&p2.model_name(), GridPointStatus::Queued, "wf_m")
|
||
.await
|
||
.unwrap();
|
||
assert!(db
|
||
.mark_grid_point_running(&p2.model_name(), "wf_m")
|
||
.await
|
||
.unwrap());
|
||
|
||
// converged 不被覆盖 ✗
|
||
let p3 = mk(34000.0);
|
||
db.upsert_grid_point(&p3, 0, "wf_m").await.unwrap();
|
||
db.update_grid_status(&p3.model_name(), GridPointStatus::Completed, "wf_m")
|
||
.await
|
||
.unwrap();
|
||
assert!(!db
|
||
.mark_grid_point_running(&p3.model_name(), "wf_m")
|
||
.await
|
||
.unwrap());
|
||
assert_eq!(
|
||
db.get_grid_point_status(&p3.model_name(), "wf_m")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"completed"
|
||
);
|
||
|
||
// failed 不被覆盖 ✗
|
||
let p4 = mk(36000.0);
|
||
db.upsert_grid_point(&p4, 0, "wf_m").await.unwrap();
|
||
db.update_grid_status(&p4.model_name(), GridPointStatus::Failed, "wf_m")
|
||
.await
|
||
.unwrap();
|
||
assert!(!db
|
||
.mark_grid_point_running(&p4.model_name(), "wf_m")
|
||
.await
|
||
.unwrap());
|
||
assert_eq!(
|
||
db.get_grid_point_status(&p4.model_name(), "wf_m")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"failed"
|
||
);
|
||
}
|
||
|
||
/// has_pending_tasks_for_point：列出点的 pending tasks 行，支持 task_type 过滤、
|
||
/// 工作流隔离；终态行不计（2026-08-02 涡旋事故修复的活性校验基础）。
|
||
#[tokio::test]
|
||
async fn test_has_pending_tasks_for_point() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("pend_tasks_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_grid_point(¶ms, 0, "wf_p").await.unwrap();
|
||
|
||
let mk_spec = |first_strategy: &str, wf: &str| common::models::TaskSpec {
|
||
task_id: Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
seed_point_name: None,
|
||
timeout_sec: 7200,
|
||
workflow_name: Some(wf.to_string()),
|
||
wave: 0,
|
||
// Phase 6 起策略链首项即"当前策略"（派生过滤依据），故直接指定首项。
|
||
tlusty_config: PhaseConfig {
|
||
strategies: vec![first_strategy.to_string()],
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
..Default::default()
|
||
};
|
||
|
||
// 两条 pending：cold_run + seed_step（均属 wf_p）。
|
||
let cold = mk_spec("cold_run", "wf_p");
|
||
let seed = mk_spec("seed_step", "wf_p");
|
||
db.insert_task(&cold).await.unwrap();
|
||
db.insert_task(&seed).await.unwrap();
|
||
|
||
let all = db
|
||
.has_pending_tasks_for_point(&name, "wf_p", None)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(all.len(), 2);
|
||
let only_seed = db
|
||
.has_pending_tasks_for_point(&name, "wf_p", Some("seed_step"))
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(only_seed, vec![seed.task_id.to_string()]);
|
||
|
||
// 跨工作流不命中。
|
||
let other_wf = db
|
||
.has_pending_tasks_for_point(&name, "wf_other", None)
|
||
.await
|
||
.unwrap();
|
||
assert!(other_wf.is_empty());
|
||
|
||
// cold_run 上报转终态后不再计入（裸 SQL 模拟按 task_id 的上报落库）。
|
||
{
|
||
let pool = db.pool.clone();
|
||
let tid = cold.task_id.to_string();
|
||
tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'failed' WHERE task_id = ?1",
|
||
params![tid],
|
||
)
|
||
.unwrap();
|
||
})
|
||
.await
|
||
.unwrap();
|
||
}
|
||
let after = db
|
||
.has_pending_tasks_for_point(&name, "wf_p", None)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(after, vec![seed.task_id.to_string()]);
|
||
}
|
||
|
||
/// delete_tasks_by_ids：只删 pending 行（TOCTOU 防护：刚完成的行不被误删），
|
||
/// 空切片短路，返回删除数。
|
||
#[tokio::test]
|
||
async fn test_delete_tasks_by_ids_only_pending() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("del_ids_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let mk_spec = || common::models::TaskSpec {
|
||
task_id: Uuid::new_v4(),
|
||
point_name: params.model_name(),
|
||
params: params.clone(),
|
||
seed_point_name: None,
|
||
timeout_sec: 7200,
|
||
workflow_name: Some("wf_d".to_string()),
|
||
wave: 0,
|
||
..Default::default()
|
||
};
|
||
let (t1, t2, t3) = (mk_spec(), mk_spec(), mk_spec());
|
||
for t in [&t1, &t2, &t3] {
|
||
db.insert_task(t).await.unwrap();
|
||
}
|
||
// t2 转 completed（模拟刚完成）。
|
||
{
|
||
let pool = db.pool.clone();
|
||
let tid = t2.task_id.to_string();
|
||
tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'completed' WHERE task_id = ?1",
|
||
params![tid],
|
||
)
|
||
.unwrap();
|
||
})
|
||
.await
|
||
.unwrap();
|
||
}
|
||
|
||
// 空切片短路。
|
||
assert_eq!(db.delete_tasks_by_ids(&[]).await.unwrap(), 0);
|
||
|
||
// 删三条：t2 因 completed 幸存，仅 t1/t3 被删。
|
||
let ids: Vec<String> = [&t1, &t2, &t3].map(|t| t.task_id.to_string()).to_vec();
|
||
assert_eq!(db.delete_tasks_by_ids(&ids).await.unwrap(), 2);
|
||
let remaining = db
|
||
.has_pending_tasks_for_point(¶ms.model_name(), "wf_d", None)
|
||
.await
|
||
.unwrap();
|
||
assert!(remaining.is_empty(), "pending 行应被清空");
|
||
// t2（completed）仍在表中。
|
||
let count: i64 = {
|
||
let pool = db.pool.clone();
|
||
let tid = t2.task_id.to_string();
|
||
tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
conn.query_row(
|
||
"SELECT COUNT(*) FROM tasks WHERE task_id = ?1",
|
||
params![tid],
|
||
|r| r.get(0),
|
||
)
|
||
.unwrap()
|
||
})
|
||
.await
|
||
.unwrap()
|
||
};
|
||
assert_eq!(count, 1, "completed 审计行必须幸存");
|
||
}
|
||
|
||
/// find_stale_pending_points + rescue_orphaned_point（2026-08-02 涡旋事故重构）：
|
||
/// 候选 = running/queued 点 + 老于 stale_sec 的 pending 行；终态点与新鲜行不命中；
|
||
/// rescue 仅对 running/queued 生效（并发终态防护）。
|
||
#[tokio::test]
|
||
async fn test_find_stale_pending_points_and_rescue() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("stale_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let mk = |teff: f64| GridPointParams {
|
||
teff: teff.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let mk_spec = |p: &GridPointParams, wf: &str| common::models::TaskSpec {
|
||
task_id: Uuid::new_v4(),
|
||
point_name: p.model_name(),
|
||
params: p.clone(),
|
||
seed_point_name: None,
|
||
timeout_sec: 7200,
|
||
workflow_name: Some(wf.to_string()),
|
||
wave: 0,
|
||
..Default::default()
|
||
};
|
||
|
||
// 点 A：running + 老 pending 行 → 命中。
|
||
let pa = mk(30000.0);
|
||
db.upsert_grid_point(&pa, 0, "wf_s").await.unwrap();
|
||
db.update_grid_status(&pa.model_name(), GridPointStatus::Running, "wf_s")
|
||
.await
|
||
.unwrap();
|
||
let ta = mk_spec(&pa, "wf_s");
|
||
db.insert_task(&ta).await.unwrap();
|
||
|
||
// 点 B：queued + 老 pending 行 → 命中。
|
||
let pb = mk(32000.0);
|
||
db.upsert_grid_point(&pb, 0, "wf_s").await.unwrap();
|
||
db.update_grid_status(&pb.model_name(), GridPointStatus::Queued, "wf_s")
|
||
.await
|
||
.unwrap();
|
||
let tb = mk_spec(&pb, "wf_s");
|
||
db.insert_task(&tb).await.unwrap();
|
||
|
||
// 点 C：running + 新鲜 pending 行 → 不命中。
|
||
let pc = mk(34000.0);
|
||
db.upsert_grid_point(&pc, 0, "wf_s").await.unwrap();
|
||
db.update_grid_status(&pc.model_name(), GridPointStatus::Running, "wf_s")
|
||
.await
|
||
.unwrap();
|
||
let tc = mk_spec(&pc, "wf_s");
|
||
db.insert_task(&tc).await.unwrap();
|
||
|
||
// 点 D：converged + 老 pending 行（僵尸）→ 点态终态，不命中。
|
||
let pd = mk(36000.0);
|
||
db.upsert_grid_point(&pd, 0, "wf_s").await.unwrap();
|
||
db.update_grid_status(&pd.model_name(), GridPointStatus::Completed, "wf_s")
|
||
.await
|
||
.unwrap();
|
||
let td = mk_spec(&pd, "wf_s");
|
||
db.insert_task(&td).await.unwrap();
|
||
|
||
// 把 A/B/D 的任务行 created_at 回拨 7 小时（C 保持新鲜）。
|
||
{
|
||
let pool = db.pool.clone();
|
||
let ids: Vec<String> = [&ta, &tb, &td].map(|t| t.task_id.to_string()).to_vec();
|
||
tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
for id in ids {
|
||
conn.execute(
|
||
"UPDATE tasks SET created_at = datetime('now', '-7 hours') WHERE task_id = ?1",
|
||
params![id],
|
||
)
|
||
.unwrap();
|
||
}
|
||
})
|
||
.await
|
||
.unwrap();
|
||
}
|
||
|
||
let stale = db.find_stale_pending_points(21600).await.unwrap();
|
||
let mut points: Vec<(String, String)> = stale
|
||
.iter()
|
||
.map(|(n, w, _)| (n.clone(), w.clone()))
|
||
.collect();
|
||
points.sort();
|
||
points.dedup();
|
||
assert_eq!(points.len(), 2, "仅 A（running）与 B（queued）命中");
|
||
assert!(points
|
||
.iter()
|
||
.all(|(n, _)| n == &pa.model_name() || n == &pb.model_name()));
|
||
|
||
// rescue：A 成功置 pending；终态点 rescue 返回 false（并发防护）。
|
||
assert!(db
|
||
.rescue_orphaned_point(&pa.model_name(), "wf_s")
|
||
.await
|
||
.unwrap());
|
||
assert_eq!(
|
||
db.get_grid_point_status(&pa.model_name(), "wf_s")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"pending"
|
||
);
|
||
assert!(!db
|
||
.rescue_orphaned_point(&pd.model_name(), "wf_s")
|
||
.await
|
||
.unwrap());
|
||
assert_eq!(
|
||
db.get_grid_point_status(&pd.model_name(), "wf_s")
|
||
.await
|
||
.unwrap()
|
||
.unwrap()
|
||
.0,
|
||
"completed"
|
||
);
|
||
}
|
||
|
||
/// 工作流计算完成自动状态迁移测试（running -> completed）：
|
||
/// 当工作流处于 running，且所有网格点全部到达终态（converged/failed）时，
|
||
/// list_workflows/sync_all_running_workflows_completion 应自动将状态翻转为 completed。
|
||
#[tokio::test]
|
||
async fn test_workflow_auto_completion_status_transition() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("auto_comp_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
|
||
// 1. 注册并启动工作流 auto_wf
|
||
db.upsert_workflow("auto_wf", Some("Auto Comp Test"), "config", "idle")
|
||
.await
|
||
.unwrap();
|
||
db.update_workflow_status("auto_wf", "running")
|
||
.await
|
||
.unwrap();
|
||
db.upsert_grid_point(¶ms, 0, "auto_wf").await.unwrap();
|
||
|
||
// 此时网格点为 pending，工作流应保持 running
|
||
let list = db.list_workflows().await.unwrap();
|
||
assert_eq!(list[0].status, "running");
|
||
|
||
// 2. 网格点完成计算（converged）
|
||
db.update_grid_status(&name, GridPointStatus::Completed, "auto_wf")
|
||
.await
|
||
.unwrap();
|
||
|
||
// 3. 执行同步巡检，应当触发自动翻转为 completed
|
||
db.sync_all_running_workflows_completion().await.unwrap();
|
||
let list_after = db.list_workflows().await.unwrap();
|
||
assert_eq!(
|
||
list_after[0].status, "completed",
|
||
"所有点完成计算后，工作流状态应自动转换为 completed"
|
||
);
|
||
|
||
// 4. get_workflow 也返回 completed
|
||
let item = db.get_workflow("auto_wf").await.unwrap().unwrap();
|
||
assert_eq!(item.status, "completed");
|
||
}
|
||
|
||
/// 完成 flip 的「未消费回退链」阻塞回归（2026-08-22 修复）：
|
||
/// 最后一个活跃点失败时，其最新 failed 行策略链弹掉失败首项后仍有顺位
|
||
/// （长度 >1）→ workflow 不得置 completed（否则 trigger_strategy_fallback
|
||
/// 的 still_running 守卫拦截，链上 seed_step/seed_step_stab 永不派发；
|
||
/// 生产实证 sdB_cno/t60000_g5.0_he-4_c-4_n-4_o-1，8-22 04:00:50）。
|
||
/// 链耗尽（长度 1）后放行翻转。
|
||
#[tokio::test]
|
||
async fn test_workflow_flip_blocked_by_unconsumed_fallback_chain() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(
|
||
&temp_dir
|
||
.path()
|
||
.join("flip_block_db.db")
|
||
.to_string_lossy(),
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 60000.0.into(),
|
||
logg: 5.0.into(),
|
||
loghe: (-4.0).into(),
|
||
logc: (-4.0).into(),
|
||
logn: (-4.0).into(),
|
||
logo: (-1.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_workflow("wf_fb2", None, "config", "idle")
|
||
.await
|
||
.unwrap();
|
||
db.update_workflow_status("wf_fb2", "running").await.unwrap();
|
||
db.upsert_grid_point(¶ms, 0, "wf_fb2").await.unwrap();
|
||
// 点终态 failed、无 pending/queued/running——旧条件已满足 flip。
|
||
db.update_grid_status(&name, GridPointStatus::Failed, "wf_fb2")
|
||
.await
|
||
.unwrap();
|
||
|
||
// 最新 failed 行携带 3 顺位链（cold_run 刚失败，尚余 seed_step/seed_step_stab）。
|
||
let t1 = uuid::Uuid::new_v4();
|
||
db.insert_task(&common::models::TaskSpec {
|
||
task_id: t1,
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
timeout_sec: 7200,
|
||
workflow_name: Some("wf_fb2".to_string()),
|
||
wave: 0,
|
||
tlusty_config: PhaseConfig {
|
||
strategies: vec![
|
||
"cold_run".to_string(),
|
||
"seed_step".to_string(),
|
||
"seed_step_stab".to_string(),
|
||
],
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
..Default::default()
|
||
})
|
||
.await
|
||
.unwrap();
|
||
{
|
||
let pool = db.pool.clone();
|
||
let tid = t1.to_string();
|
||
tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'failed', completed_at = datetime('now') \
|
||
WHERE task_id = ?1",
|
||
params![tid],
|
||
)
|
||
.unwrap();
|
||
})
|
||
.await
|
||
.unwrap();
|
||
}
|
||
|
||
db.sync_all_running_workflows_completion().await.unwrap();
|
||
let wf = db.get_workflow("wf_fb2").await.unwrap().unwrap();
|
||
assert_eq!(
|
||
wf.status, "running",
|
||
"存在未消费回退链的 failed 点时，workflow 不应翻转 completed"
|
||
);
|
||
|
||
// 回退推进到链尾（更新的 failed 行仅剩单顺位）→ 阻塞解除，翻转放行。
|
||
let t2 = uuid::Uuid::new_v4();
|
||
db.insert_task(&common::models::TaskSpec {
|
||
task_id: t2,
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
timeout_sec: 7200,
|
||
workflow_name: Some("wf_fb2".to_string()),
|
||
wave: 0,
|
||
tlusty_config: PhaseConfig {
|
||
strategies: vec!["seed_step_stab".to_string()],
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
..Default::default()
|
||
})
|
||
.await
|
||
.unwrap();
|
||
{
|
||
let pool = db.pool.clone();
|
||
let tid = t2.to_string();
|
||
tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'failed', completed_at = datetime('now'), \
|
||
created_at = datetime('now', '+1 hour') WHERE task_id = ?1",
|
||
params![tid],
|
||
)
|
||
.unwrap();
|
||
})
|
||
.await
|
||
.unwrap();
|
||
}
|
||
|
||
db.sync_all_running_workflows_completion().await.unwrap();
|
||
let wf2 = db.get_workflow("wf_fb2").await.unwrap().unwrap();
|
||
assert_eq!(
|
||
wf2.status, "completed",
|
||
"链耗尽（长度 1）后应恢复完成翻转"
|
||
);
|
||
}
|
||
|
||
/// 完成 flip 的「种子轮换待执行」阻塞回归（2026-08-22 审查补充）：
|
||
/// 链尾 [seed_step_stab] 失败 + 存在未试过的同物理族干净种子 → 不翻转
|
||
/// （否则 rotation 臂被 still_running 守卫吞掉，waypoint 只有一次机会）；
|
||
/// 同族种子全部用过后放行翻转。
|
||
#[tokio::test]
|
||
async fn test_workflow_flip_blocked_by_seed_rotation_pending() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(
|
||
&temp_dir
|
||
.path()
|
||
.join("flip_rot_db.db")
|
||
.to_string_lossy(),
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 60000.0.into(),
|
||
logg: 5.0.into(),
|
||
loghe: (-4.0).into(),
|
||
logc: (-4.0).into(),
|
||
logn: (-4.0).into(),
|
||
logo: (-1.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_workflow("wf_rot", None, "config", "idle")
|
||
.await
|
||
.unwrap();
|
||
db.update_workflow_status("wf_rot", "running").await.unwrap();
|
||
db.upsert_grid_point(¶ms, 0, "wf_rot").await.unwrap();
|
||
db.update_grid_status(&name, GridPointStatus::Failed, "wf_rot")
|
||
.await
|
||
.unwrap();
|
||
|
||
// 链尾 [seed_step_stab] 失败行。
|
||
let t1 = uuid::Uuid::new_v4();
|
||
db.insert_task(&common::models::TaskSpec {
|
||
task_id: t1,
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
timeout_sec: 7200,
|
||
workflow_name: Some("wf_rot".to_string()),
|
||
wave: 0,
|
||
tlusty_config: PhaseConfig {
|
||
strategies: vec!["seed_step_stab".to_string()],
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
..Default::default()
|
||
})
|
||
.await
|
||
.unwrap();
|
||
{
|
||
let pool = db.pool.clone();
|
||
let tid = t1.to_string();
|
||
tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'failed', completed_at = datetime('now') \
|
||
WHERE task_id = ?1",
|
||
params![tid],
|
||
)
|
||
.unwrap();
|
||
})
|
||
.await
|
||
.unwrap();
|
||
}
|
||
|
||
// 无同族种子 → 无可轮换 → 放行翻转。
|
||
db.sync_all_running_workflows_completion().await.unwrap();
|
||
assert_eq!(
|
||
db.get_workflow("wf_rot").await.unwrap().unwrap().status,
|
||
"completed",
|
||
"无同族候选种子时链尾失败应放行翻转"
|
||
);
|
||
|
||
// 重启场景：workflow 回 running，注入一个未用过的同族干净种子。
|
||
db.update_workflow_status("wf_rot", "running").await.unwrap();
|
||
let neighbor = GridPointParams {
|
||
logn: (-3.489798).into(),
|
||
..params.clone()
|
||
};
|
||
db.insert_seed_named(
|
||
"t60000_g5_he-4_c-4_n-3.489798_o-1_ladder",
|
||
&neighbor,
|
||
"/tmp/seed.7",
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
db.sync_all_running_workflows_completion().await.unwrap();
|
||
assert_eq!(
|
||
db.get_workflow("wf_rot").await.unwrap().unwrap().status,
|
||
"running",
|
||
"存在未试过的同族种子时，链尾 seed_step_stab 失败应阻塞翻转（rotation 待执行）"
|
||
);
|
||
|
||
// 轮换消费该种子（任务行携带 seed_point_name）→ 候选耗尽 → 放行。
|
||
let t2 = uuid::Uuid::new_v4();
|
||
db.insert_task(&common::models::TaskSpec {
|
||
task_id: t2,
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
seed_point_name: Some("t60000_g5_he-4_c-4_n-3.489798_o-1_ladder".to_string()),
|
||
timeout_sec: 7200,
|
||
workflow_name: Some("wf_rot".to_string()),
|
||
wave: 0,
|
||
tlusty_config: PhaseConfig {
|
||
strategies: vec!["seed_step_stab".to_string()],
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
..Default::default()
|
||
})
|
||
.await
|
||
.unwrap();
|
||
{
|
||
let pool = db.pool.clone();
|
||
let tid = t2.to_string();
|
||
tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'failed', completed_at = datetime('now'), \
|
||
created_at = datetime('now', '+1 hour') WHERE task_id = ?1",
|
||
params![tid],
|
||
)
|
||
.unwrap();
|
||
})
|
||
.await
|
||
.unwrap();
|
||
}
|
||
|
||
db.sync_all_running_workflows_completion().await.unwrap();
|
||
assert_eq!(
|
||
db.get_workflow("wf_rot").await.unwrap().unwrap().status,
|
||
"completed",
|
||
"同族种子全部用过（无新候选）后应放行翻转"
|
||
);
|
||
}
|
||
|
||
/// 完成 flip 的阶段归因回归（2026-08-22 审查补充）：synspec 失败行的
|
||
/// tlusty_strategies 是完整审计副本（scheduler.rs synspec 回退刻意保留），
|
||
/// 阻塞判定必须按 failed_stage='synspec' 只看 synspec 链——否则 synspec
|
||
/// 工作流会因 stale 审计副本永久卡 running。
|
||
#[tokio::test]
|
||
async fn test_workflow_flip_stage_attribution_ignores_tlusty_audit_copy() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(
|
||
&temp_dir
|
||
.path()
|
||
.join("flip_syn_db.db")
|
||
.to_string_lossy(),
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 30000.0.into(),
|
||
logg: 5.0.into(),
|
||
loghe: (-2.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_workflow("wf_syn2", None, "config", "idle")
|
||
.await
|
||
.unwrap();
|
||
db.update_workflow_status("wf_syn2", "running").await.unwrap();
|
||
db.upsert_grid_point(¶ms, 0, "wf_syn2").await.unwrap();
|
||
db.update_grid_status(&name, GridPointStatus::Failed, "wf_syn2")
|
||
.await
|
||
.unwrap();
|
||
|
||
// synspec 失败行：synspec 链已耗尽（长度 1），但 tlusty 链是完整审计
|
||
// 副本（长度 3）——阶段归因下不应阻塞。
|
||
let t1 = uuid::Uuid::new_v4();
|
||
db.insert_task(&common::models::TaskSpec {
|
||
task_id: t1,
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
timeout_sec: 7200,
|
||
workflow_name: Some("wf_syn2".to_string()),
|
||
wave: 0,
|
||
tlusty_config: PhaseConfig {
|
||
strategies: vec![
|
||
"cold_run".to_string(),
|
||
"seed_step".to_string(),
|
||
"seed_step_stab".to_string(),
|
||
],
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
..Default::default()
|
||
})
|
||
.await
|
||
.unwrap();
|
||
{
|
||
let pool = db.pool.clone();
|
||
let tid = t1.to_string();
|
||
tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'failed', failed_stage = 'synspec', \
|
||
synspec_strategies = '[\"standard\"]', completed_at = datetime('now') \
|
||
WHERE task_id = ?1",
|
||
params![tid],
|
||
)
|
||
.unwrap();
|
||
})
|
||
.await
|
||
.unwrap();
|
||
}
|
||
|
||
db.sync_all_running_workflows_completion().await.unwrap();
|
||
assert_eq!(
|
||
db.get_workflow("wf_syn2").await.unwrap().unwrap().status,
|
||
"completed",
|
||
"synspec 链已耗尽的失败行，其 tlusty 审计副本不应阻塞完成翻转"
|
||
);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_take_pending_node_token_atomic() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db_path = temp_dir.path().join("token_test.db");
|
||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||
|
||
let reg = NodeRegisterRequest {
|
||
node_id: "node-atomic-test".to_string(),
|
||
max_slots: 2,
|
||
};
|
||
let (_, _, secret) = db.register_node(®).await.unwrap();
|
||
let token = db.approve_node("node-atomic-test").await.unwrap();
|
||
assert!(!token.is_empty());
|
||
|
||
// 第一次调用：提供正确 registration_secret，返回 token
|
||
let pending1 = db
|
||
.take_pending_node_token("node-atomic-test", secret.as_deref())
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(pending1, Some(token));
|
||
|
||
// 第二次调用：已被置为 NULL，返回 None
|
||
let pending2 = db
|
||
.take_pending_node_token("node-atomic-test", secret.as_deref())
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(pending2, None);
|
||
|
||
// 错误的 registration_secret：不应返回 token（H8 防护）
|
||
let pending3 = db
|
||
.take_pending_node_token("node-atomic-test", Some("wrong-secret"))
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(pending3, None);
|
||
}
|
||
|
||
/// P5（Phase 4）：清除 node_credentials.revoked 死列 + registration_secret 链路回归。
|
||
/// 1. 全新库 bootstrap 后 node_credentials 无 revoked 列（M4 detect 跳过，不执行 DROP）；
|
||
/// 2. 注册 → 审批 → 取 token 全链路行为不变（registration_secret 保留在 nodes）。
|
||
#[tokio::test]
|
||
async fn test_node_credentials_no_revoked_and_secret_chain() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("p4_creds.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
// ① 新库无 revoked 列（CREATE TABLE 已不含该列，M4 detect=true 跳过）。
|
||
{
|
||
let conn = db.pool.get().unwrap();
|
||
let mut stmt = conn.prepare("PRAGMA table_info(node_credentials)").unwrap();
|
||
let cols: Vec<String> = stmt
|
||
.query_map([], |r| r.get::<_, String>(1))
|
||
.unwrap()
|
||
.filter_map(Result::ok)
|
||
.collect();
|
||
assert!(
|
||
!cols.iter().any(|c| c == "revoked"),
|
||
"新库不应含 revoked 列，实际列：{cols:?}"
|
||
);
|
||
assert!(cols.iter().any(|c| c == "raw_token_pending"));
|
||
}
|
||
|
||
// ② 注册 → 审批 → 取 token 链路回归（registration_secret 行为不变）。
|
||
let reg = NodeRegisterRequest {
|
||
node_id: "node-p4".to_string(),
|
||
max_slots: 2,
|
||
};
|
||
let (_, _, secret) = db.register_node(®).await.unwrap();
|
||
let token = db.approve_node("node-p4").await.unwrap();
|
||
assert!(!token.is_empty());
|
||
let pending = db
|
||
.take_pending_node_token("node-p4", secret.as_deref())
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(pending, Some(token));
|
||
// 错误 secret 不放行（H8 防护回归）。
|
||
let denied = db
|
||
.take_pending_node_token("node-p4", Some("wrong-secret"))
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(denied, None);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_heartbeat_node_status_check() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db_path = temp_dir.path().join("hb_check.db");
|
||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||
|
||
let reg = NodeRegisterRequest {
|
||
node_id: "node-hb-test".to_string(),
|
||
max_slots: 2,
|
||
};
|
||
db.register_node(®).await.unwrap();
|
||
db.approve_node("node-hb-test").await.unwrap();
|
||
|
||
// 此时 node 状态为 online，心跳正常更新
|
||
let hb_req = NodeHeartbeatRequest {
|
||
node_id: "node-hb-test".to_string(),
|
||
active_slots: 1,
|
||
cpu_usage: 10.0,
|
||
memory_usage: 20.0,
|
||
};
|
||
db.heartbeat_node(&hb_req).await.unwrap();
|
||
let nodes = db.get_active_nodes().await.unwrap();
|
||
assert_eq!(nodes.len(), 1);
|
||
|
||
// 人为修改节点状态为 rejected
|
||
let pool = db.pool.clone();
|
||
tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
conn.execute(
|
||
"UPDATE nodes SET status = 'rejected' WHERE node_id = 'node-hb-test'",
|
||
[],
|
||
)
|
||
.unwrap();
|
||
})
|
||
.await
|
||
.unwrap();
|
||
|
||
// 再次发心跳：不应重置 status 为 online
|
||
db.heartbeat_node(&hb_req).await.unwrap();
|
||
let nodes2 = db.get_active_nodes().await.unwrap();
|
||
assert_eq!(
|
||
nodes2.len(),
|
||
0,
|
||
"status 为 rejected 的节点心跳时不应更新为 online"
|
||
);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_delete_workflow_cascade_cleanup() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db_path = temp_dir.path().join("del_wf.db");
|
||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||
|
||
let wf_name = "wf_to_delete";
|
||
db.upsert_workflow(wf_name, Some("Test"), "config", "idle")
|
||
.await
|
||
.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
db.upsert_grid_point(¶ms, 0, wf_name).await.unwrap();
|
||
|
||
let spec = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: params.model_name(),
|
||
params: params.clone(),
|
||
seed_point_name: None,
|
||
timeout_sec: 600,
|
||
workflow_name: Some(wf_name.to_string()),
|
||
wave: 0,
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&spec).await.unwrap();
|
||
|
||
// 确认插入成功
|
||
assert!(db.get_workflow(wf_name).await.unwrap().is_some());
|
||
assert_eq!(db.get_pending_grid_points(wf_name).await.unwrap().len(), 1);
|
||
|
||
// 删除工作流
|
||
db.delete_workflow(wf_name).await.unwrap();
|
||
|
||
// 验证 workflows, grid_points, tasks 被级联清理
|
||
assert!(db.get_workflow(wf_name).await.unwrap().is_none());
|
||
assert_eq!(db.get_pending_grid_points(wf_name).await.unwrap().len(), 0);
|
||
|
||
let pool = db.pool.clone();
|
||
let wf_owned = wf_name.to_string();
|
||
let task_cnt: i64 = tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
conn.query_row(
|
||
"SELECT COUNT(*) FROM tasks WHERE workflow_name = ?1",
|
||
params![wf_owned],
|
||
|r| r.get(0),
|
||
)
|
||
.unwrap()
|
||
})
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(task_cnt, 0, "关联 tasks 记录应被清理");
|
||
}
|
||
|
||
/// 原子选点测试（#5 修复验证）：
|
||
/// 1. claim_pending_grid_points 返回 pending 点并原子标记为 queued。
|
||
/// 2. 第二次 claim 返回空（点已非 pending）。
|
||
/// 3. 排序正确：wave ASC, cno_sum ASC, teff ASC。
|
||
#[tokio::test]
|
||
async fn test_claim_pending_grid_points_atomic() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("claim_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let wf = "claim_wf";
|
||
|
||
// 插入 3 个不同 wave 的点
|
||
let p1 = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let p2 = GridPointParams {
|
||
teff: 40000.0.into(),
|
||
logg: 5.0.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-3.0).into(),
|
||
logn: (-3.0).into(),
|
||
logo: (-3.0).into(),
|
||
};
|
||
let p3 = GridPointParams {
|
||
teff: 30000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-1.0).into(),
|
||
logn: (-1.0).into(),
|
||
logo: (-1.0).into(),
|
||
};
|
||
db.upsert_grid_point(&p1, 1, wf).await.unwrap();
|
||
db.upsert_grid_point(&p2, 0, wf).await.unwrap();
|
||
db.upsert_grid_point(&p3, 2, wf).await.unwrap();
|
||
|
||
// 第一次 claim：应返回全部 3 个，按 wave ASC 排序（p2 wave=0, p1 wave=1, p3 wave=2）
|
||
let claimed = db.claim_pending_grid_points(100, wf).await.unwrap();
|
||
assert_eq!(claimed.len(), 3);
|
||
assert_eq!(claimed[0].0, p2.model_name(), "wave=0 应排第一");
|
||
assert_eq!(claimed[1].0, p1.model_name(), "wave=1 应排第二");
|
||
assert_eq!(claimed[2].0, p3.model_name(), "wave=2 应排第三");
|
||
|
||
// 验证点已变为 queued
|
||
let pending_after = db.get_pending_grid_points(wf).await.unwrap();
|
||
assert_eq!(pending_after.len(), 0, "claim 后不应有 pending 点");
|
||
|
||
// 第二次 claim：应返回空
|
||
let claimed_again = db.claim_pending_grid_points(100, wf).await.unwrap();
|
||
assert_eq!(claimed_again.len(), 0, "已 queued 的点不应被再次 claim");
|
||
|
||
// LIMIT 测试：重置回 pending 后只 claim 2 个
|
||
db.reset_queued_grid_points_to_pending(wf).await.unwrap();
|
||
let partial = db.claim_pending_grid_points(2, wf).await.unwrap();
|
||
assert_eq!(partial.len(), 2, "LIMIT 2 应只返回 2 个点");
|
||
let remaining = db.claim_pending_grid_points(100, wf).await.unwrap();
|
||
assert_eq!(remaining.len(), 1, "剩余 1 个点");
|
||
}
|
||
|
||
/// 阶段参数化弹栈（docs/task_engine_decoupling_design.md §4.2 注）：`synspec` 弹
|
||
/// `synspec_strategies` 列，与 TLUSTY 链互不干扰（修复审查 #2 的 DB 层）。
|
||
#[tokio::test]
|
||
async fn test_pop_stage_strategy_synspec_independent() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db = Database::new(&temp_dir.path().join("pop_syn_db.db").to_string_lossy())
|
||
.await
|
||
.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 35000.0.into(),
|
||
logg: 5.5.into(),
|
||
loghe: (-1.0).into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_grid_point(¶ms, 0, "wf_ps").await.unwrap();
|
||
|
||
// 双链各含两项：tlusty [cold_run, seed_step]、synspec [standard, standard]。
|
||
let spec = common::models::TaskSpec {
|
||
task_id: uuid::Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
seed_point_name: None,
|
||
timeout_sec: 7200,
|
||
workflow_name: Some("wf_ps".to_string()),
|
||
wave: 0,
|
||
tlusty_config: common::models::PhaseConfig {
|
||
strategies: vec!["cold_run".to_string(), "seed_step".to_string()],
|
||
..common::models::PhaseConfig::default_tlusty()
|
||
},
|
||
synspec_config: common::models::PhaseConfig {
|
||
strategies: vec!["standard".to_string(), "standard".to_string()],
|
||
..common::models::PhaseConfig::default_synspec()
|
||
},
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&spec).await.unwrap();
|
||
{
|
||
let pool = db.pool.clone();
|
||
let tid = spec.task_id.to_string();
|
||
tokio::task::spawn_blocking(move || {
|
||
let conn = pool.get().unwrap();
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'failed', completed_at = datetime('now') WHERE task_id = ?1",
|
||
rusqlite::params![tid],
|
||
)
|
||
.unwrap();
|
||
})
|
||
.await
|
||
.unwrap();
|
||
}
|
||
|
||
// 弹 synspec：剩 [standard]（只读，不改写旧行），policy 取该行 synspec 快照。
|
||
let popped_syn = db
|
||
.pop_stage_strategy_for_fallback(&name, "wf_ps", "synspec")
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
popped_syn,
|
||
Some(FallbackSnapshot {
|
||
rest_strategies: vec!["standard".to_string()],
|
||
popped: "standard".to_string(),
|
||
policy: common::models::ResumePolicy::SkipConverged,
|
||
})
|
||
);
|
||
// 弹 tlusty：剩 [seed_step]，不受 synspec 弹栈影响（同行的两列独立）。
|
||
let popped_tl = db
|
||
.pop_tlusty_strategy_for_fallback(&name, "wf_ps")
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
popped_tl,
|
||
Some(FallbackSnapshot {
|
||
rest_strategies: vec!["seed_step".to_string()],
|
||
popped: "cold_run".to_string(),
|
||
policy: common::models::ResumePolicy::SkipConverged,
|
||
})
|
||
);
|
||
// 只读：两列原始内容不变。
|
||
assert_eq!(
|
||
db.get_latest_tlusty_strategies(&name, "wf_ps")
|
||
.await
|
||
.unwrap(),
|
||
vec!["cold_run".to_string(), "seed_step".to_string()]
|
||
);
|
||
// 非法 stage 兜底 tlusty 列（防注入）。
|
||
let popped_bad = db
|
||
.pop_stage_strategy_for_fallback(&name, "wf_ps", "'; DROP TABLE tasks;--")
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
popped_bad,
|
||
Some(FallbackSnapshot {
|
||
rest_strategies: vec!["seed_step".to_string()],
|
||
popped: "cold_run".to_string(),
|
||
policy: common::models::ResumePolicy::SkipConverged,
|
||
})
|
||
);
|
||
}
|
||
|
||
/// synspec-only 重跑后 tlusty_success_method / tlusty_status 须保留 prior 值，
|
||
/// 不能被 NULL 覆写（CASE 守卫修复验证）。
|
||
///
|
||
/// 场景：先以 TLUSTY 启用（cold_run）跑成功 → tlusty_success_method = "cold_run"。
|
||
/// 再以 TLUSTY 关闭（仅 SYNSPEC，场景 B）重跑成功 → tlusty_success_method 仍须为
|
||
/// "cold_run"，不能被覆写为 NULL。synspec_success_method 应更新为 "standard"。
|
||
#[tokio::test]
|
||
async fn test_synspec_only_rerun_preserves_tlusty_attribution() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db_path = temp_dir.path().join("synrerun.db");
|
||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||
let wf = "wf_synrerun";
|
||
|
||
let params = GridPointParams {
|
||
teff: 25000.0.into(),
|
||
logg: 5.0.into(),
|
||
loghe: 2.0.into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_grid_point(¶ms, 0, wf).await.unwrap();
|
||
|
||
// ── 第一轮：TLUSTY 启用 + cold_run，成功 ──
|
||
let task1 = common::models::TaskSpec {
|
||
task_id: Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
workflow_name: Some(wf.to_string()),
|
||
wave: 0,
|
||
timeout_sec: 7200,
|
||
tlusty_config: PhaseConfig {
|
||
enabled: true,
|
||
strategies: vec!["cold_run".to_string()],
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
synspec_config: PhaseConfig::default_synspec(),
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&task1).await.unwrap();
|
||
|
||
let summary1 = common::models::ModelSummary {
|
||
name: name.clone(),
|
||
params: params.clone(),
|
||
stages: vec![common::models::StepSummary {
|
||
label: "nl".into(),
|
||
chmax: Some(0.001),
|
||
lte: "F".into(),
|
||
converged: true,
|
||
best_max_relc: Some(0.0005),
|
||
elapsed_sec: 300.0,
|
||
note: None,
|
||
last_iter: Some(17),
|
||
worst_depth: Some(1),
|
||
n_depths: Some(50),
|
||
itek_history: vec![],
|
||
conv_trace_check: None,
|
||
}],
|
||
result_valid: true,
|
||
final_max_relc: Some(0.0005),
|
||
final_chmax: Some(0.001),
|
||
seed: None,
|
||
atmosphere_has_nan: false,
|
||
synspec_rc: Some(0),
|
||
synspec_error: None,
|
||
synspec_sec: Some(0.3),
|
||
elapsed_sec: 300.3,
|
||
energy_check: None,
|
||
temp_check: None,
|
||
emflux_check: None,
|
||
bfac_check: None,
|
||
ladder_seeds: Vec::new(),
|
||
note: None,
|
||
};
|
||
let report1 = TaskReport {
|
||
task_id: task1.task_id,
|
||
point_name: name.clone(),
|
||
params: Some(params.clone()),
|
||
node_id: "test-node".to_string(),
|
||
status: TaskStatus::Completed,
|
||
result_valid: true,
|
||
max_relc: Some(0.0005),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 300.3,
|
||
error_message: None,
|
||
summary_json: serde_json::to_string(&summary1).unwrap(),
|
||
failed_stage: None,
|
||
};
|
||
db.record_task_report(&report1, wf).await.unwrap();
|
||
|
||
// 验证第一轮：tlusty_success_method = cold_run, tlusty_status = converged
|
||
let row1 = read_grid_attrs(&db, &name, wf).await;
|
||
assert_eq!(row1.status, "completed");
|
||
assert_eq!(row1.tlusty_success_method.as_deref(), Some("cold_run"));
|
||
assert_eq!(row1.tlusty_status.as_deref(), Some("converged"));
|
||
assert_eq!(row1.synspec_success_method.as_deref(), Some("standard"));
|
||
assert_eq!(row1.synspec_status.as_deref(), Some("converged"));
|
||
|
||
// ── 模拟场景 B：reset_terminal_points_for_recompute 翻回 pending ──
|
||
db.reset_terminal_points_for_recompute(wf).await.unwrap();
|
||
|
||
// ── 第二轮：TLUSTY 关闭 + SYNSPEC 启用（synspec-only），成功 ──
|
||
let task2 = common::models::TaskSpec {
|
||
task_id: Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
workflow_name: Some(wf.to_string()),
|
||
wave: 0,
|
||
timeout_sec: 7200,
|
||
tlusty_config: PhaseConfig {
|
||
enabled: false,
|
||
strategies: vec!["cold_run".to_string()],
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
synspec_config: PhaseConfig::default_synspec(),
|
||
atmosphere_ref: Some(name.clone()),
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&task2).await.unwrap();
|
||
|
||
// synspec-only 的 summary：stages 为空 → merge_point_summary 走字段级合并
|
||
let summary2 = common::models::ModelSummary {
|
||
name: name.clone(),
|
||
params: params.clone(),
|
||
stages: Vec::new(),
|
||
result_valid: true,
|
||
final_max_relc: None,
|
||
final_chmax: None,
|
||
seed: None,
|
||
atmosphere_has_nan: false,
|
||
synspec_rc: Some(0),
|
||
synspec_error: None,
|
||
synspec_sec: Some(0.25),
|
||
elapsed_sec: 0.25,
|
||
energy_check: None,
|
||
temp_check: None,
|
||
emflux_check: None,
|
||
bfac_check: None,
|
||
ladder_seeds: Vec::new(),
|
||
note: None,
|
||
};
|
||
let report2 = TaskReport {
|
||
task_id: task2.task_id,
|
||
point_name: name.clone(),
|
||
params: Some(params.clone()),
|
||
node_id: "test-node".to_string(),
|
||
status: TaskStatus::Completed,
|
||
result_valid: true,
|
||
max_relc: None,
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 0.25,
|
||
error_message: None,
|
||
summary_json: serde_json::to_string(&summary2).unwrap(),
|
||
failed_stage: None,
|
||
};
|
||
db.record_task_report(&report2, wf).await.unwrap();
|
||
|
||
// ── 核心断言：synspec-only 重跑后 tlusty 侧归因须保留 ──
|
||
let row2 = read_grid_attrs(&db, &name, wf).await;
|
||
assert_eq!(row2.status, "completed", "重跑成功后应为 completed");
|
||
// ★ 修复前：tsm2 = None（裸赋值覆写）。修复后：保留 "cold_run"。
|
||
assert_eq!(
|
||
row2.tlusty_success_method.as_deref(),
|
||
Some("cold_run"),
|
||
"synspec-only 重跑后 tlusty_success_method 须保留，不能覆写为 NULL"
|
||
);
|
||
assert_eq!(
|
||
row2.tlusty_status.as_deref(),
|
||
Some("converged"),
|
||
"synspec-only 重跑后 tlusty_status 须保留"
|
||
);
|
||
// synspec 侧应更新为新值
|
||
assert_eq!(
|
||
row2.synspec_success_method.as_deref(),
|
||
Some("standard"),
|
||
"synspec_success_method 应更新为 standard"
|
||
);
|
||
assert_eq!(
|
||
row2.synspec_status.as_deref(),
|
||
Some("converged"),
|
||
"synspec_status 应为 converged"
|
||
);
|
||
|
||
// summary_json 的 TLUSTY 诊断也须保留（merge_point_summary 字段级合并）
|
||
let merged = db.get_point_summary_json(wf, &name).await.unwrap().unwrap();
|
||
let ms: common::models::ModelSummary = serde_json::from_str(&merged).unwrap();
|
||
assert_eq!(ms.stages.len(), 1, "stages 须保留 prior 的 TLUSTY 链");
|
||
assert_eq!(ms.stages[0].label, "nl");
|
||
assert_eq!(ms.final_max_relc, Some(0.0005), "final_max_relc 须保留");
|
||
assert_eq!(ms.synspec_rc, Some(0), "synspec_rc 应为新值");
|
||
assert_eq!(ms.elapsed_sec, 0.25, "elapsed_sec 应为新值");
|
||
|
||
// last_elapsed_sec 语义：最近一次尝试耗时。synspec-only 重跑后为 0.25s（synspec 耗时），
|
||
// 原 TLUSTY 耗时保留在 stages[].elapsed_sec。ETA 不依赖此列（用 AVG(tasks.elapsed_sec)）。
|
||
let last_elapsed = read_grid_last_elapsed(&db, &name, wf).await;
|
||
assert_eq!(
|
||
last_elapsed, Some(0.25),
|
||
"last_elapsed_sec 应为 synspec-only 耗时（最近一次尝试），非原 TLUSTY 总耗时"
|
||
);
|
||
}
|
||
|
||
/// 辅助：读取 grid_points 的阶段归因列。
|
||
async fn read_grid_attrs(db: &Database, name: &str, wf: &str) -> GridAttrs {
|
||
let pool = db.pool.clone();
|
||
let name = name.to_string();
|
||
let wf = wf.to_string();
|
||
tokio::task::spawn_blocking(move || -> GridAttrs {
|
||
let conn = pool.get().unwrap();
|
||
conn.query_row(
|
||
"SELECT status, tlusty_success_method, synspec_success_method, tlusty_status, synspec_status \
|
||
FROM grid_points WHERE name = ?1 AND workflow_name = ?2",
|
||
rusqlite::params![name, wf],
|
||
|r| {
|
||
Ok(GridAttrs {
|
||
status: r.get(0)?,
|
||
tlusty_success_method: r.get(1)?,
|
||
synspec_success_method: r.get(2)?,
|
||
tlusty_status: r.get(3)?,
|
||
synspec_status: r.get(4)?,
|
||
})
|
||
},
|
||
)
|
||
.unwrap()
|
||
})
|
||
.await
|
||
.unwrap()
|
||
}
|
||
|
||
struct GridAttrs {
|
||
status: String,
|
||
tlusty_success_method: Option<String>,
|
||
synspec_success_method: Option<String>,
|
||
tlusty_status: Option<String>,
|
||
synspec_status: Option<String>,
|
||
}
|
||
|
||
/// synspec-only 重跑**失败**后 tlusty_success_method / tlusty_status 仍须保留。
|
||
///
|
||
/// 失败分支的 UPDATE 不写 success_method 列，但 tlusty_status / synspec_status
|
||
/// 有 CASE 守卫。验证失败报告不会清空 prior 的 TLUSTY 归因。
|
||
#[tokio::test]
|
||
async fn test_synspec_only_rerun_failure_preserves_tlusty_attribution() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db_path = temp_dir.path().join("synfail.db");
|
||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||
let wf = "wf_synfail";
|
||
|
||
let params = GridPointParams {
|
||
teff: 25000.0.into(),
|
||
logg: 5.0.into(),
|
||
loghe: 2.0.into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_grid_point(¶ms, 0, wf).await.unwrap();
|
||
|
||
// 第一轮：TLUSTY 启用 + cold_run，成功。
|
||
let task1 = common::models::TaskSpec {
|
||
task_id: Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
workflow_name: Some(wf.to_string()),
|
||
wave: 0,
|
||
timeout_sec: 7200,
|
||
tlusty_config: PhaseConfig {
|
||
enabled: true,
|
||
strategies: vec!["cold_run".to_string()],
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
synspec_config: PhaseConfig::default_synspec(),
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&task1).await.unwrap();
|
||
let summary1 = common::models::ModelSummary {
|
||
name: name.clone(),
|
||
params: params.clone(),
|
||
stages: vec![common::models::StepSummary {
|
||
label: "nl".into(),
|
||
chmax: Some(0.001),
|
||
lte: "F".into(),
|
||
converged: true,
|
||
best_max_relc: Some(0.0005),
|
||
elapsed_sec: 300.0,
|
||
note: None,
|
||
last_iter: Some(17),
|
||
worst_depth: Some(1),
|
||
n_depths: Some(50),
|
||
itek_history: vec![],
|
||
conv_trace_check: None,
|
||
}],
|
||
result_valid: true,
|
||
final_max_relc: Some(0.0005),
|
||
final_chmax: Some(0.001),
|
||
seed: None,
|
||
atmosphere_has_nan: false,
|
||
synspec_rc: Some(0),
|
||
synspec_error: None,
|
||
synspec_sec: Some(0.3),
|
||
elapsed_sec: 300.3,
|
||
energy_check: None,
|
||
temp_check: None,
|
||
emflux_check: None,
|
||
bfac_check: None,
|
||
ladder_seeds: Vec::new(),
|
||
note: None,
|
||
};
|
||
let report1 = TaskReport {
|
||
task_id: task1.task_id,
|
||
point_name: name.clone(),
|
||
params: Some(params.clone()),
|
||
node_id: "test-node".to_string(),
|
||
status: TaskStatus::Completed,
|
||
result_valid: true,
|
||
max_relc: Some(0.0005),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 300.3,
|
||
error_message: None,
|
||
summary_json: serde_json::to_string(&summary1).unwrap(),
|
||
failed_stage: None,
|
||
};
|
||
db.record_task_report(&report1, wf).await.unwrap();
|
||
assert_eq!(read_grid_attrs(&db, &name, wf).await.tlusty_success_method.as_deref(), Some("cold_run"));
|
||
|
||
// 翻回 pending 模拟场景 B 重跑。
|
||
db.reset_terminal_points_for_recompute(wf).await.unwrap();
|
||
|
||
// 第二轮：synspec-only，失败（synspec 产出脏谱）。
|
||
let task2 = common::models::TaskSpec {
|
||
task_id: Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
workflow_name: Some(wf.to_string()),
|
||
wave: 0,
|
||
timeout_sec: 7200,
|
||
tlusty_config: PhaseConfig {
|
||
enabled: false,
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
synspec_config: PhaseConfig::default_synspec(),
|
||
atmosphere_ref: Some(name.clone()),
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&task2).await.unwrap();
|
||
// synspec 失败：result_valid=false, stages 为空（synspec-only），synspec_rc=1。
|
||
let summary2 = common::models::ModelSummary {
|
||
name: name.clone(),
|
||
params: params.clone(),
|
||
stages: Vec::new(),
|
||
result_valid: false,
|
||
final_max_relc: None,
|
||
final_chmax: None,
|
||
seed: None,
|
||
atmosphere_has_nan: false,
|
||
synspec_rc: Some(1),
|
||
synspec_error: Some("spec 含 NaN".into()),
|
||
synspec_sec: Some(0.2),
|
||
elapsed_sec: 0.2,
|
||
energy_check: None,
|
||
temp_check: None,
|
||
emflux_check: None,
|
||
bfac_check: None,
|
||
ladder_seeds: Vec::new(),
|
||
note: Some("synspec 失败".into()),
|
||
};
|
||
let report2 = TaskReport {
|
||
task_id: task2.task_id,
|
||
point_name: name.clone(),
|
||
params: Some(params.clone()),
|
||
node_id: "test-node".to_string(),
|
||
status: TaskStatus::Failed,
|
||
result_valid: false,
|
||
max_relc: None,
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 0.2,
|
||
error_message: Some("synspec 失败".to_string()),
|
||
summary_json: serde_json::to_string(&summary2).unwrap(),
|
||
failed_stage: Some("synspec".to_string()),
|
||
};
|
||
db.record_task_report(&report2, wf).await.unwrap();
|
||
|
||
// 失败后 tlusty 侧归因仍须保留。
|
||
let row = read_grid_attrs(&db, &name, wf).await;
|
||
assert_eq!(row.status, "failed", "失败后状态应为 failed");
|
||
assert_eq!(
|
||
row.tlusty_success_method.as_deref(),
|
||
Some("cold_run"),
|
||
"synspec-only 失败后 tlusty_success_method 须保留"
|
||
);
|
||
assert_eq!(
|
||
row.tlusty_status.as_deref(),
|
||
Some("converged"),
|
||
"synspec-only 失败后 tlusty_status 须保留（CASE 守卫）"
|
||
);
|
||
// synspec 侧应反映失败。
|
||
assert_eq!(row.synspec_status.as_deref(), Some("failed"));
|
||
}
|
||
|
||
/// 辅助：读取 grid_points.last_elapsed_sec。
|
||
async fn read_grid_last_elapsed(db: &Database, name: &str, wf: &str) -> Option<f64> {
|
||
let pool = db.pool.clone();
|
||
let name = name.to_string();
|
||
let wf = wf.to_string();
|
||
tokio::task::spawn_blocking(move || -> Option<f64> {
|
||
let conn = pool.get().unwrap();
|
||
conn.query_row(
|
||
"SELECT last_elapsed_sec FROM grid_points WHERE name = ?1 AND workflow_name = ?2",
|
||
rusqlite::params![name, wf],
|
||
|r| r.get(0),
|
||
)
|
||
.ok()
|
||
})
|
||
.await
|
||
.unwrap()
|
||
}
|
||
|
||
/// TLUSTY-only 重跑成功后 synspec 归因列应被显式清空（clear_synspec=true），
|
||
/// summary_json 保留 prior synspec 字段（merge_point_summary TLUSTY-only 路径）。
|
||
///
|
||
/// 场景：先正常管线（TLUSTY+SYNSPEC）成功 → 再 TLUSTY-only（synspec 关闭）重跑成功
|
||
/// → 新大气使旧光谱失效 → synspec_success_method/synspec_status 清 NULL，
|
||
/// summary_json 中 synspec_rc/synspec_sec 保留自 prior。
|
||
#[tokio::test]
|
||
async fn test_tlusty_only_rerun_clears_synspec_attribution() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db_path = temp_dir.path().join("tlonly.db");
|
||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||
let wf = "wf_tlonly";
|
||
|
||
let params = GridPointParams {
|
||
teff: 25000.0.into(),
|
||
logg: 5.0.into(),
|
||
loghe: 2.0.into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-2.0).into(),
|
||
logo: (-2.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_grid_point(¶ms, 0, wf).await.unwrap();
|
||
|
||
// ── 第一轮：正常管线（TLUSTY + SYNSPEC 双开），成功 ──
|
||
let task1 = common::models::TaskSpec {
|
||
task_id: Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
workflow_name: Some(wf.to_string()),
|
||
wave: 0,
|
||
timeout_sec: 7200,
|
||
tlusty_config: PhaseConfig {
|
||
enabled: true,
|
||
strategies: vec!["cold_run".to_string()],
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
synspec_config: PhaseConfig::default_synspec(),
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&task1).await.unwrap();
|
||
let summary1 = common::models::ModelSummary {
|
||
name: name.clone(),
|
||
params: params.clone(),
|
||
stages: vec![common::models::StepSummary {
|
||
label: "nl".into(),
|
||
chmax: Some(0.001),
|
||
lte: "F".into(),
|
||
converged: true,
|
||
best_max_relc: Some(0.0005),
|
||
elapsed_sec: 300.0,
|
||
note: None,
|
||
last_iter: Some(17),
|
||
worst_depth: Some(1),
|
||
n_depths: Some(50),
|
||
itek_history: vec![],
|
||
conv_trace_check: None,
|
||
}],
|
||
result_valid: true,
|
||
final_max_relc: Some(0.0005),
|
||
final_chmax: Some(0.001),
|
||
seed: None,
|
||
atmosphere_has_nan: false,
|
||
synspec_rc: Some(0),
|
||
synspec_error: None,
|
||
synspec_sec: Some(0.3),
|
||
elapsed_sec: 300.3,
|
||
energy_check: None,
|
||
temp_check: None,
|
||
emflux_check: None,
|
||
bfac_check: None,
|
||
ladder_seeds: Vec::new(),
|
||
note: None,
|
||
};
|
||
let report1 = TaskReport {
|
||
task_id: task1.task_id,
|
||
point_name: name.clone(),
|
||
params: Some(params.clone()),
|
||
node_id: "test-node".to_string(),
|
||
status: TaskStatus::Completed,
|
||
result_valid: true,
|
||
max_relc: Some(0.0005),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 300.3,
|
||
error_message: None,
|
||
summary_json: serde_json::to_string(&summary1).unwrap(),
|
||
failed_stage: None,
|
||
};
|
||
db.record_task_report(&report1, wf).await.unwrap();
|
||
let row1 = read_grid_attrs(&db, &name, wf).await;
|
||
assert_eq!(row1.synspec_success_method.as_deref(), Some("standard"));
|
||
assert_eq!(row1.synspec_status.as_deref(), Some("converged"));
|
||
|
||
// ── 翻回 pending 模拟 TLUSTY-only 重跑 ──
|
||
db.reset_terminal_points_for_recompute(wf).await.unwrap();
|
||
|
||
// ── 第二轮：TLUSTY-only（synspec 关闭），成功 ──
|
||
let task2 = common::models::TaskSpec {
|
||
task_id: Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
workflow_name: Some(wf.to_string()),
|
||
wave: 0,
|
||
timeout_sec: 7200,
|
||
tlusty_config: PhaseConfig {
|
||
enabled: true,
|
||
strategies: vec!["cold_run".to_string()],
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
synspec_config: PhaseConfig {
|
||
enabled: false,
|
||
..PhaseConfig::default_synspec()
|
||
},
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&task2).await.unwrap();
|
||
// TLUSTY-only summary：stages 非空（TLUSTY 跑了），synspec_rc=None（synspec 没跑）
|
||
let summary2 = common::models::ModelSummary {
|
||
name: name.clone(),
|
||
params: params.clone(),
|
||
stages: vec![common::models::StepSummary {
|
||
label: "nl".into(),
|
||
chmax: Some(0.0008),
|
||
lte: "F".into(),
|
||
converged: true,
|
||
best_max_relc: Some(0.0003),
|
||
elapsed_sec: 280.0,
|
||
note: None,
|
||
last_iter: Some(15),
|
||
worst_depth: Some(1),
|
||
n_depths: Some(50),
|
||
itek_history: vec![],
|
||
conv_trace_check: None,
|
||
}],
|
||
result_valid: true,
|
||
final_max_relc: Some(0.0003),
|
||
final_chmax: Some(0.0008),
|
||
seed: None,
|
||
atmosphere_has_nan: false,
|
||
synspec_rc: None, // synspec 未运行
|
||
synspec_error: None,
|
||
synspec_sec: None,
|
||
elapsed_sec: 280.0,
|
||
energy_check: None,
|
||
temp_check: None,
|
||
emflux_check: None,
|
||
bfac_check: None,
|
||
ladder_seeds: Vec::new(),
|
||
note: None,
|
||
};
|
||
let report2 = TaskReport {
|
||
task_id: task2.task_id,
|
||
point_name: name.clone(),
|
||
params: Some(params.clone()),
|
||
node_id: "test-node".to_string(),
|
||
status: TaskStatus::Completed,
|
||
result_valid: true,
|
||
max_relc: Some(0.0003),
|
||
atmosphere_has_nan: false,
|
||
elapsed_sec: 280.0,
|
||
error_message: None,
|
||
summary_json: serde_json::to_string(&summary2).unwrap(),
|
||
failed_stage: None,
|
||
};
|
||
db.record_task_report(&report2, wf).await.unwrap();
|
||
|
||
// ── 核心断言：synspec 列应被显式清空（clear_synspec=true）──
|
||
let row2 = read_grid_attrs(&db, &name, wf).await;
|
||
assert_eq!(row2.status, "completed");
|
||
assert_eq!(
|
||
row2.tlusty_success_method.as_deref(),
|
||
Some("cold_run"),
|
||
"tlusty_success_method 应更新为 cold_run"
|
||
);
|
||
assert_eq!(
|
||
row2.tlusty_status.as_deref(),
|
||
Some("converged"),
|
||
"tlusty_status 应为 converged"
|
||
);
|
||
// ★ synspec 列被 clear_synspec 显式置 NULL（新大气使旧光谱失效）
|
||
assert_eq!(
|
||
row2.synspec_success_method,
|
||
None,
|
||
"TLUSTY-only 重跑后 synspec_success_method 须清 NULL（clear_synspec）"
|
||
);
|
||
assert_eq!(
|
||
row2.synspec_status,
|
||
None,
|
||
"TLUSTY-only 重跑后 synspec_status 须清 NULL（clear_synspec）"
|
||
);
|
||
|
||
// summary_json：TLUSTY 诊断来自 incoming，synspec 字段保留自 prior
|
||
let merged = db.get_point_summary_json(wf, &name).await.unwrap().unwrap();
|
||
let ms: common::models::ModelSummary = serde_json::from_str(&merged).unwrap();
|
||
assert_eq!(ms.stages.len(), 1, "stages 来自 incoming");
|
||
assert_eq!(ms.final_max_relc, Some(0.0003), "final_max_relc 来自 incoming");
|
||
assert_eq!(
|
||
ms.synspec_rc,
|
||
Some(0),
|
||
"synspec_rc 保留 prior 值（merge_point_summary TLUSTY-only 路径）"
|
||
);
|
||
assert_eq!(ms.synspec_sec, Some(0.3), "synspec_sec 保留 prior 值");
|
||
}
|
||
|
||
/// `get_task_tlusty_enabled`：正常任务返回 true，synspec-only 任务返回 false，
|
||
/// 不存在的 task_id 返回 None。
|
||
#[tokio::test]
|
||
async fn test_get_task_tlusty_enabled() {
|
||
let temp_dir = tempfile::tempdir().unwrap();
|
||
let db_path = temp_dir.path().join("tlusty_enabled.db");
|
||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||
|
||
let params = GridPointParams {
|
||
teff: 20000.0.into(),
|
||
logg: 5.0.into(),
|
||
loghe: 2.0.into(),
|
||
logc: (-2.0).into(),
|
||
logn: (-4.0).into(),
|
||
logo: (-4.0).into(),
|
||
};
|
||
let name = params.model_name();
|
||
db.upsert_grid_point(¶ms, 0, "wf_tle").await.unwrap();
|
||
|
||
// 正常任务（tlusty_enabled=true）
|
||
let task_normal = common::models::TaskSpec {
|
||
task_id: Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
workflow_name: Some("wf_tle".to_string()),
|
||
tlusty_config: PhaseConfig::default_tlusty(),
|
||
synspec_config: PhaseConfig::default_synspec(),
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&task_normal).await.unwrap();
|
||
let enabled = db.get_task_tlusty_enabled(&task_normal.task_id).await.unwrap();
|
||
assert_eq!(enabled, Some(true), "正常任务 tlusty_enabled 应为 true");
|
||
|
||
// synspec-only 任务（tlusty_enabled=false）
|
||
let task_synonly = common::models::TaskSpec {
|
||
task_id: Uuid::new_v4(),
|
||
point_name: name.clone(),
|
||
params: params.clone(),
|
||
workflow_name: Some("wf_tle".to_string()),
|
||
tlusty_config: PhaseConfig {
|
||
enabled: false,
|
||
..PhaseConfig::default_tlusty()
|
||
},
|
||
synspec_config: PhaseConfig::default_synspec(),
|
||
atmosphere_ref: Some(name.clone()),
|
||
..Default::default()
|
||
};
|
||
db.insert_task(&task_synonly).await.unwrap();
|
||
let enabled = db.get_task_tlusty_enabled(&task_synonly.task_id).await.unwrap();
|
||
assert_eq!(enabled, Some(false), "synspec-only 任务 tlusty_enabled 应为 false");
|
||
|
||
// 不存在的 task_id → None
|
||
let fake_id = Uuid::new_v4();
|
||
let enabled = db.get_task_tlusty_enabled(&fake_id).await.unwrap();
|
||
assert_eq!(enabled, None, "不存在的 task_id 应返回 None");
|
||
}
|
||
}
|