feat(all): 数据库模块化拆分与版本化迁移、任务引擎命名体系收敛、物理输出校验加固与用户配置接通
- server/db: 拆 4929 行 db.rs 单体为 db/ 目录，migrations.rs 引入 PRAGMA user_version
版本化迁移运行器（M1~M13）
- 任务引擎 Phase 6/7b/7c 改名收敛：EngineStageConfig→PhaseConfig、StagePolicy→ResumePolicy、
Converged→Completed、删除 task_type 列、success_method 拆 tlusty_/synspec_ 双列、
新增 tlusty_status/synspec_status 半失败阶段守卫
- 科学正确性加固：conv_check 任意行 NaN/Inf/溢出判无效（0 行容忍）、新增 spec_is_valid
校验 SYNSPEC 脏谱、itek_history 逐次迭代全量保真、fmt_abn powf 溢出饱和
- 用户配置真正接通：tlusty_chain/tlusty_input 由死字段经 调度器→TaskSpec→executor→runner
透传生效；config 加载期 validate + deny_unknown_fields + 解析失败记 warn
- 调度修复：H1 活锁（pending_strategies 跳过已失败策略）、种子查找错误不再静默降级冷启动
- dashboard: 阶段配置面板 tlusty_stage/synspec_stage、"已完成"标签、迭代诊断展示
- docs: 新增 database_refactor_design.md，同步 database/api/PIPELINE/workflow_detail
This commit is contained in:
+167
-33
@@ -1,11 +1,13 @@
|
||||
use crate::models::{EngineStageConfig, GridAxisValue};
|
||||
use crate::models::{GridAxisValue, PhaseConfig};
|
||||
use anyhow::{Context, Result};
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct GridAxesConfig {
|
||||
pub teff: Vec<GridAxisValue>,
|
||||
pub logg: Vec<GridAxisValue>,
|
||||
@@ -136,24 +138,77 @@ impl GridConfig {
|
||||
if let Some(raw_axes) = parse_grid_axes_raw(yaml) {
|
||||
cfg.grid = raw_axes;
|
||||
}
|
||||
cfg.validate()?;
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
/// 配置合法性校验（审查修复 #M2/#N3）：把配置错误从运行时逐点失败提前到加载时，
|
||||
/// 避免笔误（如 chmax: 0、logc: 400）静默产出错误输入文件、浪费算力。
|
||||
///
|
||||
/// 校验项：
|
||||
/// - `tlusty_chain` 中每个 ChainStep 的 `chmax` 若为 Some，必须 > 0（conv_check.rs
|
||||
/// 的守卫会把 chmax <= 0 判为非法、整 stage 判发散；runner.rs 的 unwrap_or(0.001)
|
||||
/// 只在字段缺失时兜底，显式写 0 不会触发）。
|
||||
/// - grid 六轴数值范围合理性（teff > 0；丰度 logc/logn/logo 物理上 ∈ [-20, 10]，
|
||||
/// 超出几乎必为笔误，如 logc: 400 会让 gen_input5 的 10^logx 溢出为 Inf 污染输入文件）。
|
||||
fn validate(&self) -> Result<()> {
|
||||
for (i, step) in self.tlusty_chain.iter().enumerate() {
|
||||
if let Some(chmax) = step.chmax {
|
||||
// 用 partial_cmp 显式判断：`!(chmax > 0.0)` 对 NaN 为 true（NaN 比较恒 false），
|
||||
// 应拒绝 NaN；改写为 `chmax <= 0.0` 会漏掉 NaN（NaN<=0 也是 false），故不用。
|
||||
if !matches!(chmax.partial_cmp(&0.0), Some(std::cmp::Ordering::Greater)) {
|
||||
anyhow::bail!(
|
||||
"tlusty_chain[{}] (label={}) 的 chmax={} 非法：必须 > 0（<=0 会被 conv_check 判为整 stage 发散）",
|
||||
i, step.label, chmax
|
||||
);
|
||||
}
|
||||
}
|
||||
// 注：niter 不校验 > 0——LTE grey start 步骤 niter=0 是合法设计
|
||||
//（runner.rs:455 显式处理 niter==0 为「不迭代，直接用 grey atmosphere 作初值」）。
|
||||
}
|
||||
// teff 必须为正（物理温度）。
|
||||
for (i, t) in self.grid.teff.iter().enumerate() {
|
||||
let v = t.value();
|
||||
// 同 chmax：`!(v > 0.0)` 保持对 NaN 的拒绝语义，用 partial_cmp 显式表达。
|
||||
if !matches!(v.partial_cmp(&0.0), Some(std::cmp::Ordering::Greater)) {
|
||||
anyhow::bail!("grid.teff[{}] = {} 非法：温度必须 > 0", i, v);
|
||||
}
|
||||
}
|
||||
// 丰度对数轴范围校验（超出 [-20, 10] 几乎必为笔误，且会令 10^logx 溢出为 Inf）。
|
||||
for (axis_name, vals) in [
|
||||
("logc", &self.grid.logc),
|
||||
("logn", &self.grid.logn),
|
||||
("logo", &self.grid.logo),
|
||||
("loghe", &self.grid.loghe),
|
||||
] {
|
||||
for (i, av) in vals.iter().enumerate() {
|
||||
let v = av.value();
|
||||
if !(-20.0..=10.0).contains(&v) {
|
||||
anyhow::bail!(
|
||||
"grid.{}[{}] = {} 超出物理合理范围 [-20, 10]：请检查是否笔误（超出会让 10^logx 溢出为 Inf 污染输入文件）",
|
||||
axis_name, i, v
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 解析 TLUSTY 阶段配置。
|
||||
///
|
||||
/// 优先级（见 docs/task_engine_decoupling_design.md §3）：
|
||||
/// 1. 新版顶层 `tlusty:` 块（EngineStageConfig）—— 显式覆盖；
|
||||
/// 1. 新版顶层 `tlusty_stage:` 块（PhaseConfig）—— 显式覆盖；
|
||||
/// 2. 旧版 `seed_step_fallback: bool` —— true → `[cold_run, seed_step]`，
|
||||
/// false → `[cold_run]`（不回退种子步进）；
|
||||
/// 3. 兜底 `default_tlusty()`。
|
||||
///
|
||||
/// 注：旧版只控制是否回退种子步进，无 enabled/policy 维度，故回退路径固定
|
||||
/// enabled=true / policy=SkipConverged（与新默认一致）。
|
||||
pub fn resolve_tlusty_config(&self) -> EngineStageConfig {
|
||||
if let Some(cfg) = &self.tlusty {
|
||||
pub fn resolve_tlusty_config(&self) -> PhaseConfig {
|
||||
if let Some(cfg) = &self.tlusty_stage {
|
||||
return cfg.clone();
|
||||
}
|
||||
let mut cfg = EngineStageConfig::default_tlusty();
|
||||
let mut cfg = PhaseConfig::default_tlusty();
|
||||
if !self.seed_step_fallback {
|
||||
cfg.strategies = vec!["cold_run".to_string()];
|
||||
}
|
||||
@@ -163,22 +218,90 @@ impl GridConfig {
|
||||
/// 解析 SYNSPEC 阶段配置。
|
||||
///
|
||||
/// 优先级：
|
||||
/// 1. 新版顶层 `synspec_stage:` 块（EngineStageConfig）—— 显式覆盖（含 enabled 开关）；
|
||||
/// 1. 新版顶层 `synspec_stage:` 块（PhaseConfig）—— 显式覆盖（含 enabled 开关）；
|
||||
/// 2. 兜底 `default_synspec()`（enabled=true，保持旧行为：有大气就跑光谱）。
|
||||
///
|
||||
/// 注：旧版 `synspec: SynspecConfig`（数值参数）不影响阶段启用/策略——它只携带
|
||||
/// 注：旧版 `synspec: SynspecInput`（数值参数）不影响阶段启用/策略——它只携带
|
||||
/// 波长范围等数值，由调度器透传到 TaskSpec.synspec_params。如需禁用 SYNSPEC，
|
||||
/// 必须用新版 `synspec_stage: { enabled: false }`。
|
||||
pub fn resolve_synspec_config(&self) -> EngineStageConfig {
|
||||
pub fn resolve_synspec_config(&self) -> PhaseConfig {
|
||||
if let Some(cfg) = &self.synspec_stage {
|
||||
return cfg.clone();
|
||||
}
|
||||
EngineStageConfig::default_synspec()
|
||||
PhaseConfig::default_synspec()
|
||||
}
|
||||
}
|
||||
|
||||
// ===== TLUSTY 输入文件（.5 + nst）全局物理参数 =====
|
||||
// 这些参数不随收敛阶段（lte/nc/nl）变化——NFREAD 频率网格、ions 能级数据表等是
|
||||
// 物理建模选择，一旦确定对整个大气计算全局生效。阶段差异参数（lte/ltgray/niter/
|
||||
// chmax/metals 等）保留在 ChainStep 里。
|
||||
// 缺省 None → 走代码内硬编码默认（gen_input5.rs/nst_writer.rs 的常量），与改动前行为一致。
|
||||
|
||||
/// 单个元素的 atoms 块配置（.5 文件的 `mode abn modpf` 行）。
|
||||
/// abn（丰度）不在此配——每网格点不同，由 GridPointParams.loghe/logc/logn/logo 计算。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct StageConfig {
|
||||
pub struct AtomConfig {
|
||||
/// 0=不计算, 1=隐式(仅 LTE Saha 算电荷，不出现在 ions 块), 2=显式(统计平衡)。
|
||||
/// None → 走代码默认（H/He/CNO=2，Li/Be/B=0）。
|
||||
pub mode: Option<i32>,
|
||||
/// partition function 模式（.5 atoms 行第 3 列 modpf）。None → 默认 0。
|
||||
#[serde(default)]
|
||||
pub modpf: Option<i32>,
|
||||
}
|
||||
|
||||
/// 单个离子的能级数据配置（.5 文件 ions 块的一行）。
|
||||
/// ilast 由 nlevs 推导（nlevs==1 → ilast=1 裸核终止标志，否则 0），不暴露。
|
||||
/// ilvlin 由 ChainStep.ilvlin 提供（nlevs==1 时强制 0），不在此配。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct IonConfig {
|
||||
/// 原子序数（1=H, 2=He, 6=C, 7=N, 8=O...）。
|
||||
pub iat: i32,
|
||||
/// 电离级（0=中性, 1=一次电离, 2=二次电离...）。
|
||||
pub iz: i32,
|
||||
/// 能级数（该离子的能级模型复杂度）。
|
||||
pub nlevs: i32,
|
||||
/// 4 字符离子标识（如 `" H 1"`、`"He 2"`），用于 tlusty 日志与诊断。
|
||||
pub typion: String,
|
||||
/// 能级数据文件路径（如 `"data/h1.dat"`），裸核（nlevs==1）用 `" "`。
|
||||
pub filei: String,
|
||||
}
|
||||
|
||||
/// TLUSTY 输入文件的全局物理参数（.5 + nst 的非阶段差异部分）。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TlustyInput {
|
||||
/// .5 frequencies 块的 NFREAD：连续频率网格点数。
|
||||
/// 取值 >0 → 从预设频率表读取（高精度、慢）；≤0 → 由 frmin/frmax 自动生成对数网格（快）。
|
||||
/// 默认 2000（gen_input5.rs 原硬编码值）。
|
||||
#[serde(default = "default_nfread")]
|
||||
pub nfread: i32,
|
||||
|
||||
/// atoms 块特定元素的 mode/modpf 覆盖。
|
||||
/// key = 元素符号（`"H"`/`"He"`/`"C"`/`"N"`/`"O"`/`"Li"`/`"Be"`/`"B"`）。
|
||||
/// 缺省的元素走代码默认 mode（H/He/CNO=2，Li/Be/B=0）。
|
||||
#[serde(default)]
|
||||
pub atoms: HashMap<String, AtomConfig>,
|
||||
|
||||
/// ions 能级数据表（完全替换默认的 H/He/C/N/O 23 行表）。
|
||||
/// 空Vec → 用 gen_input5.rs 的默认常量表。
|
||||
/// 非空时由 metals 参数按 iat 筛选参与元素。
|
||||
#[serde(default)]
|
||||
pub ions: Vec<IonConfig>,
|
||||
|
||||
/// nst 文件的额外关键字（逃逸口）：自由传入任意 `KEY=VALUE` 对。
|
||||
/// 生成 nst 时追加到末尾（每行一个），用于暴露未结构化的 220+ nst 关键字
|
||||
/// （如 FRCMAX/CUTBAL/TAU/NDGREY 等）。
|
||||
#[serde(default)]
|
||||
pub nst_extra_keys: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
fn default_nfread() -> i32 {
|
||||
2000
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ChainStep {
|
||||
pub label: String,
|
||||
#[serde(default = "default_false_str")]
|
||||
pub lte: String,
|
||||
@@ -207,8 +330,10 @@ fn default_niter() -> i32 {
|
||||
50
|
||||
}
|
||||
|
||||
/// SYNSPEC 输入文件（fort.55）的数值参数。与 `TlustyInput` 语义对称——
|
||||
/// 分别承载各阶段输入文件的物理参数（tlusty 的 .5/nst vs synspec 的 fort.55）。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SynspecConfig {
|
||||
pub struct SynspecInput {
|
||||
#[serde(default = "default_wstart")]
|
||||
pub wstart: f64,
|
||||
#[serde(default = "default_wend")]
|
||||
@@ -249,12 +374,28 @@ fn default_abs_cutoff() -> f64 {
|
||||
0.01
|
||||
}
|
||||
|
||||
/// 审查修复：`#[serde(deny_unknown_fields)]` 让旧字段名（如已重命名的 `chain`/`synspec`/
|
||||
/// `tlusty`/`results`/`itek_fallback`）在反序列化时**立即报错**，而非静默丢弃导致配置失效。
|
||||
/// 用户明确不需要向后兼容，故用严格模式让配置错误尽早暴露。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct GridConfig {
|
||||
pub grid: GridAxesConfig,
|
||||
/// TLUSTY 物理迭代步进链（lte/nc/nl 多阶段 ChainStep 数组）。
|
||||
/// 每阶段独立配置 niter/chmax/metals/ilvlin 等参数，由 scheduler 序列化进
|
||||
/// TaskSpec.tlusty_chain_params，executor 反序列化后透传给 runner.custom_chain。
|
||||
/// 空（缺省）→ executor 用 `default_chain_for_strategy` 兜底（按策略名选默认链）。
|
||||
#[serde(default)]
|
||||
pub chain: Vec<StageConfig>,
|
||||
pub synspec: Option<SynspecConfig>,
|
||||
pub tlusty_chain: Vec<ChainStep>,
|
||||
/// TLUSTY 输入文件（.5 + nst）的全局物理参数。缺省 None → 走代码内硬编码默认。
|
||||
/// 由 scheduler 序列化进 TaskSpec.tlusty_input_params，executor 反序列化后
|
||||
/// 透传给 runner → make_input5 / generate_nst_content。
|
||||
#[serde(default)]
|
||||
pub tlusty_input: Option<TlustyInput>,
|
||||
/// SYNSPEC 输入文件（fort.55）的数值参数。与 `tlusty_input` 语义对称。
|
||||
/// 由 scheduler 序列化进 TaskSpec.synspec_params，executor 反序列化后透传给 runner。
|
||||
#[serde(default)]
|
||||
pub synspec_input: Option<SynspecInput>,
|
||||
#[serde(default = "default_nworkers")]
|
||||
pub nworkers: usize,
|
||||
#[serde(default = "default_timeout")]
|
||||
@@ -263,13 +404,6 @@ pub struct GridConfig {
|
||||
pub resume: bool,
|
||||
#[serde(default = "default_true")]
|
||||
pub seed_step_fallback: bool,
|
||||
/// **已弃用的死字段**：旧版 Python 工具链遗留，无任何代码读取（实际目录以
|
||||
/// `ServerConfig.seeds_dir` / `DCTS_SEEDS_DIR` 为准）。保留以兼容旧 workflow YAML。
|
||||
#[deprecated(note = "死字段，实际目录以 DCTS_SEEDS_DIR 为准")]
|
||||
#[serde(default)]
|
||||
pub results: Option<String>,
|
||||
#[serde(default)]
|
||||
pub itek_fallback: Vec<StageConfig>,
|
||||
#[serde(default = "default_grid_niter")]
|
||||
pub niter: Option<i32>,
|
||||
pub template: Option<String>,
|
||||
@@ -277,12 +411,14 @@ pub struct GridConfig {
|
||||
pub linelist: Option<String>,
|
||||
/// TLUSTY 阶段独立配置（见 docs/task_engine_decoupling_design.md §3）。
|
||||
/// 缺省 None → `resolve_tlusty_config()` 据旧 `seed_step_fallback` 推断默认链。
|
||||
/// 命名为 `tlusty_stage` 以与同级 `synspec_stage` 对称（均带 `_stage` 后缀，
|
||||
/// 表示阶段启用/策略配置，区别于 `synspec: SynspecInput` 数值参数）。
|
||||
#[serde(default)]
|
||||
pub tlusty: Option<EngineStageConfig>,
|
||||
/// SYNSPEC 阶段独立配置。命名为 `synspec_stage` 以与上方旧 `synspec: SynspecConfig`
|
||||
pub tlusty_stage: Option<PhaseConfig>,
|
||||
/// SYNSPEC 阶段独立配置。命名为 `synspec_stage` 以与上方旧 `synspec: SynspecInput`
|
||||
///（光谱合成数值参数）区分。缺省 None → `resolve_synspec_config()` 给默认 `[standard]`。
|
||||
#[serde(default)]
|
||||
pub synspec_stage: Option<EngineStageConfig>,
|
||||
pub synspec_stage: Option<PhaseConfig>,
|
||||
}
|
||||
|
||||
fn default_grid_niter() -> Option<i32> {
|
||||
@@ -356,17 +492,16 @@ fn default_node_stale_sec() -> u64 {
|
||||
|
||||
impl Default for ServerConfig {
|
||||
fn default() -> Self {
|
||||
// Phase 7b：清理旧 CNO_* / DCTS_RESULTS_DIR 回退（TLUSTY-first 遗留命名）。
|
||||
let port = std::env::var("DCTS_PORT")
|
||||
.or_else(|_| std::env::var("CNO_PORT"))
|
||||
.or_else(|_| std::env::var("PORT"))
|
||||
.unwrap_or_else(|_| "8090".to_string());
|
||||
let db_path = std::env::var("DCTS_DB_PATH").unwrap_or_else(|_| "data/dcts.db".to_string());
|
||||
let queue_db_path = std::env::var("DCTS_QUEUE_DB_PATH")
|
||||
.unwrap_or_else(|_| "data/dcts_queue.db".to_string());
|
||||
// 种子库目录：优先 DCTS_SEEDS_DIR，回退旧 DCTS_RESULTS_DIR（已弃用，保留兼容）。
|
||||
let seeds_dir = std::env::var("DCTS_SEEDS_DIR")
|
||||
.or_else(|_| std::env::var("DCTS_RESULTS_DIR"))
|
||||
.unwrap_or_else(|_| "data/seeds".to_string());
|
||||
// 种子库目录：DCTS_SEEDS_DIR，缺省 data/seeds。
|
||||
let seeds_dir =
|
||||
std::env::var("DCTS_SEEDS_DIR").unwrap_or_else(|_| "data/seeds".to_string());
|
||||
let backup_dir =
|
||||
std::env::var("DCTS_BACKUP_DIR").unwrap_or_else(|_| "data/backups".to_string());
|
||||
let grid_config = std::env::var("DCTS_GRID_CONFIG")
|
||||
@@ -474,9 +609,9 @@ impl std::fmt::Debug for NodeConfig {
|
||||
|
||||
impl Default for NodeConfig {
|
||||
fn default() -> Self {
|
||||
// Phase 7b：清理旧 CNO_SERVER_URL 回退。
|
||||
let server_url = std::env::var("DCTS_SERVER_URL")
|
||||
.or_else(|_| std::env::var("SERVER_URL"))
|
||||
.or_else(|_| std::env::var("CNO_SERVER_URL"))
|
||||
.unwrap_or_else(|_| "http://127.0.0.1:8090".to_string());
|
||||
let node_id = std::env::var("DCTS_NODE_ID")
|
||||
.or_else(|_| std::env::var("NODE_ID"))
|
||||
@@ -496,10 +631,9 @@ impl Default for NodeConfig {
|
||||
let runtime_dir =
|
||||
std::env::var("DCTS_RUNTIME_DIR").unwrap_or_else(|_| "data/runtime".to_string());
|
||||
let work_dir = std::env::var("DCTS_WORK_DIR").unwrap_or_else(|_| "data/work".to_string());
|
||||
// 结果归档目录：优先 DCTS_RESULT_DIR，回退旧 DCTS_ARCHIVE_DIR（已弃用，保留兼容）。
|
||||
let result_dir = std::env::var("DCTS_RESULT_DIR")
|
||||
.or_else(|_| std::env::var("DCTS_ARCHIVE_DIR"))
|
||||
.unwrap_or_else(|_| "data/result".to_string());
|
||||
// 结果归档目录：DCTS_RESULT_DIR，缺省 data/result（Phase 7b 清理旧 DCTS_ARCHIVE_DIR 回退）。
|
||||
let result_dir =
|
||||
std::env::var("DCTS_RESULT_DIR").unwrap_or_else(|_| "data/result".to_string());
|
||||
let heartbeat_sec = std::env::var("DCTS_HEARTBEAT_SEC")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::models::ConvCheckResult;
|
||||
use crate::models::{ConvCheckResult, IterCheck};
|
||||
use regex::Regex;
|
||||
use std::fs::File;
|
||||
use std::io::{BufRead, BufReader};
|
||||
@@ -6,9 +6,20 @@ use std::path::Path;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
static FORT9_RE: OnceLock<Regex> = OnceLock::new();
|
||||
/// 无效数值正则模式（NaN / Inf / Infinity / Fortran 字段溢出 `***` / 超高正指数 E+300+）。
|
||||
///
|
||||
/// 抽为常量是因为 `NAN_RE` 这个 `OnceLock` 被 `atmosphere_has_nan` 与 `spec_is_valid`
|
||||
/// 共用——若两处 `get_or_init` 闭包传不同的字符串，`OnceLock` 全局只采用首次初始化的
|
||||
/// 版本，第二处闭包被静默丢弃，导致两函数行为不一致且依赖调用顺序（并发测试间歇性失败）。
|
||||
/// 常量化保证两处字面一致。超高正指数分支物理论据见 `atmosphere_has_nan` 文档注释。
|
||||
const NAN_RE_PATTERN: &str = r"(?i)(\bnan\b|\binf(?:inity)?\b|\*{3,}|[eE]\+(?:3\d{2}|[4-9]\d{2,}))";
|
||||
static NAN_RE: OnceLock<Regex> = OnceLock::new();
|
||||
/// 匹配 Fortran 无-E 科学记数法的尾数+指数部分（归一化用，见 parse_fortran_float）。
|
||||
static NO_E_EXP_RE: OnceLock<Regex> = OnceLock::new();
|
||||
/// 匹配 fort.6 中求解器发散 STOP 行（如 `**** STOP in SOLVE after ITER 8`）。
|
||||
static SOLVER_STOP_RE: OnceLock<Regex> = OnceLock::new();
|
||||
/// 匹配 call quit / stop 留言关键字。
|
||||
static QUIT_RE: OnceLock<Regex> = OnceLock::new();
|
||||
|
||||
/// 解析 fort.9 / fort.7 中的数值字符串为 f64，兼容 Fortran 的**无-E 科学记数法**。
|
||||
///
|
||||
@@ -63,6 +74,7 @@ pub fn check_fort9(path: &Path, chmax: f64) -> ConvCheckResult {
|
||||
last_iter: None,
|
||||
n_depths: 0,
|
||||
chmax,
|
||||
itek_history: Vec::new(),
|
||||
error: Some(format!(
|
||||
"非法 chmax={}（须为正有限数）：请检查工作流 YAML 中该 stage 的 chmax 配置",
|
||||
chmax
|
||||
@@ -80,6 +92,7 @@ pub fn check_fort9(path: &Path, chmax: f64) -> ConvCheckResult {
|
||||
last_iter: None,
|
||||
n_depths: 0,
|
||||
chmax,
|
||||
itek_history: Vec::new(),
|
||||
error: Some(format!("Failed to open fort.9: {}", e)),
|
||||
}
|
||||
}
|
||||
@@ -95,6 +108,9 @@ pub fn check_fort9(path: &Path, chmax: f64) -> ConvCheckResult {
|
||||
let mut last_iter: Option<i32> = None;
|
||||
let mut cur_iter: Option<i32> = None;
|
||||
let mut cur_rows: Vec<Fort9Row> = Vec::new();
|
||||
// 逐次迭代诊断（Phase 5b itek 全量保真）：每次迭代记录该次最大 |maximum|。
|
||||
let mut itek_history: Vec<IterCheck> = Vec::new();
|
||||
let mut iter_max_relc: f64 = 0.0;
|
||||
|
||||
for line in reader.lines().map_while(Result::ok) {
|
||||
if let Some(caps) = re.captures(&line) {
|
||||
@@ -112,14 +128,35 @@ pub fn check_fort9(path: &Path, chmax: f64) -> ConvCheckResult {
|
||||
};
|
||||
|
||||
if cur_iter != Some(iter) {
|
||||
// 迭代切换：结算上一拍（若有），并开启新拍。
|
||||
if let Some(prev_iter) = cur_iter {
|
||||
itek_history.push(IterCheck {
|
||||
iter: prev_iter,
|
||||
max_relc: iter_max_relc,
|
||||
n_depths: cur_rows.len(),
|
||||
});
|
||||
}
|
||||
cur_iter = Some(iter);
|
||||
cur_rows.clear();
|
||||
iter_max_relc = 0.0;
|
||||
}
|
||||
|
||||
cur_rows.push(Fort9Row { depth, maximum });
|
||||
let abs_max = maximum.abs();
|
||||
if abs_max > iter_max_relc {
|
||||
iter_max_relc = abs_max;
|
||||
}
|
||||
last_iter = Some(iter);
|
||||
}
|
||||
}
|
||||
// 收尾：结算最后一拍。
|
||||
if let Some(prev_iter) = cur_iter {
|
||||
itek_history.push(IterCheck {
|
||||
iter: prev_iter,
|
||||
max_relc: iter_max_relc,
|
||||
n_depths: cur_rows.len(),
|
||||
});
|
||||
}
|
||||
|
||||
if cur_rows.is_empty() || last_iter.is_none() {
|
||||
return ConvCheckResult {
|
||||
@@ -129,6 +166,7 @@ pub fn check_fort9(path: &Path, chmax: f64) -> ConvCheckResult {
|
||||
last_iter: None,
|
||||
n_depths: 0,
|
||||
chmax,
|
||||
itek_history: Vec::new(),
|
||||
error: Some("No valid iteration data found in fort.9".to_string()),
|
||||
};
|
||||
}
|
||||
@@ -149,6 +187,7 @@ pub fn check_fort9(path: &Path, chmax: f64) -> ConvCheckResult {
|
||||
last_iter,
|
||||
n_depths: 0,
|
||||
chmax,
|
||||
itek_history: Vec::new(),
|
||||
error: Some(
|
||||
"No valid iteration rows found when calculating maximum change".to_string(),
|
||||
),
|
||||
@@ -166,6 +205,7 @@ pub fn check_fort9(path: &Path, chmax: f64) -> ConvCheckResult {
|
||||
last_iter,
|
||||
n_depths: cur_rows.len(),
|
||||
chmax,
|
||||
itek_history,
|
||||
error: if is_valid_num {
|
||||
None
|
||||
} else {
|
||||
@@ -182,11 +222,24 @@ pub fn check_fort9(path: &Path, chmax: f64) -> ConvCheckResult {
|
||||
/// - Fortran 字段宽度溢出标记 `***`（如 `********`）：Tlusty 数值溢出发散时常以星号填满
|
||||
/// 字段而非写 NaN。历史上只检 `\bnan\b`，全溢出发散的大气会被判"无 NaN"→converged，
|
||||
/// 产出物理上完全错误的大气。
|
||||
/// - 超高指数科学记数法 `[Ee]\+(3\d{2}|[4-9]\d{2,})`（漏洞4补充盲区）：匹配 E+300 以上的
|
||||
/// 正指数值（如 `1.0E+308`、`9.99E+307`）。这是数值发散的产物——恒星大气物理量天花板
|
||||
/// 在 E+07 量级（温度/密度/布居数），E+300 在物理上无意义。gfortran 实际溢出时多写 `***`
|
||||
/// （已被上一条覆盖），但"未溢出但接近 f64 上限"的窄窗口需此分支兜底。锁定正号 `\+`
|
||||
/// 避免误伤合法的极小值（如 E-300）。真实数据集最高仅 E+07，零误报。
|
||||
///
|
||||
/// 超过 10% 的行命中任一标记即判定无效。
|
||||
/// **任意一行**命中任一标记即判定无效（0 行容忍）。物理论据：大气模型每个深度点的
|
||||
/// 物理量都是耦合求解的，任意一个深度点 NaN/Inf/溢出意味着该层解已破坏，整个大气不可用。
|
||||
///
|
||||
/// 文件缺失时返回 `false`（语义：不存在 NaN 内容）。这与“含 NaN 导致无效”是不同语义；
|
||||
/// 调用方需先自行确认文件存在性，不应将“缺失”与“含 NaN”混为一谈。
|
||||
/// 历史：曾用 10% 阈值（`bad_lines > total*0.1`），但单行 NaN（如表层发散）会被放过，
|
||||
/// 导致部分坏大气被误判为可用。见 `docs/tlusty&synspec收敛性判断.md` §4 漏洞 4。
|
||||
///
|
||||
/// 文件缺失时返回 `false`（语义：不存在 NaN 内容）。这与"含 NaN 导致无效"是不同语义；
|
||||
/// 调用方需先自行确认文件存在性，不应将"缺失"与"含 NaN"混为一谈。
|
||||
///
|
||||
/// **空文件（存在但 0 行数据）返回 `true`**（语义：无效大气）。0 行意味着大气数据
|
||||
/// 缺失/损坏（如 TLUSTY 启动后立即崩溃、写出的文件截断），不应被当作"无 NaN 的可用
|
||||
/// 大气"。runner 凭此判定 final_converged=false，行为安全。
|
||||
pub fn atmosphere_has_nan(path: &Path) -> bool {
|
||||
let file = match File::open(path) {
|
||||
Ok(f) => f,
|
||||
@@ -195,8 +248,7 @@ pub fn atmosphere_has_nan(path: &Path) -> bool {
|
||||
let reader = BufReader::new(file);
|
||||
let mut total_lines = 0;
|
||||
let mut bad_lines = 0;
|
||||
let nan_re =
|
||||
NAN_RE.get_or_init(|| Regex::new(r"(?i)(\bnan\b|\binf(?:inity)?\b|\*{3,})").unwrap());
|
||||
let nan_re = NAN_RE.get_or_init(|| Regex::new(NAN_RE_PATTERN).unwrap());
|
||||
|
||||
for line in reader.lines().map_while(Result::ok) {
|
||||
total_lines += 1;
|
||||
@@ -209,7 +261,120 @@ pub fn atmosphere_has_nan(path: &Path) -> bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
(bad_lines as f64) > (total_lines as f64 * 0.1)
|
||||
// 漏洞4修复：0 行容忍——任意一行 NaN/Inf/溢出/超高指数即判定大气不可用。
|
||||
bad_lines > 0
|
||||
}
|
||||
|
||||
/// 校验 SYNSPEC 产出的 `.spec` 光谱文件内容是否有效。
|
||||
///
|
||||
/// # 背景（漏洞 1，P0）
|
||||
/// gfortran 下 SYNSPEC 几乎所有错误路径都用裸 `STOP`（rc=0），且即便发散也会写出
|
||||
/// 一个"看起来存在"的 `.spec`。旧代码只做 `is_file()` 存在性检查，导致脏谱（含
|
||||
/// NaN/Inf、行数极少、流量全零）被当作 `Completed` 归档——这是全链路最大的科学
|
||||
/// 正确性风险。见 `docs/tlusty&synspec收敛性判断.md` §4 漏洞 1。
|
||||
///
|
||||
/// # 校验规则（按序短路）
|
||||
/// 1. 文件缺失/无法读取 → `Some("...")`
|
||||
/// 2. 逐行扫描累计：含 NaN/Inf/`***` 的 bad_lines、含 ≥2 个可解析数值 token 的有效行、
|
||||
/// 流量列（第 2 列）非零的有效行
|
||||
/// 3. 空文件 → `Some("spec 为空")`
|
||||
/// 4. bad_lines > 0 → `Some("spec 含 NaN/Inf/溢出行 (共 N 行)")`
|
||||
/// 5. 有效行数 < 10 → `Some("spec 有效行数不足 (N<10)")`
|
||||
/// 6. 非零流量行数 == 0 → `Some("spec 流量全为零")`
|
||||
/// 7. 全通过 → `None`
|
||||
///
|
||||
/// 返回 `None` 表示有效，`Some(原因)` 表示无效（调用方据此置 `synspec_rc`/`synspec_error`）。
|
||||
pub fn spec_is_valid(path: &Path) -> Option<String> {
|
||||
let file = match File::open(path) {
|
||||
Ok(f) => f,
|
||||
Err(_) => return Some("spec 文件缺失或无法读取".to_string()),
|
||||
};
|
||||
let reader = BufReader::new(file);
|
||||
let nan_re = NAN_RE.get_or_init(|| Regex::new(NAN_RE_PATTERN).unwrap());
|
||||
|
||||
let mut total_lines = 0;
|
||||
let mut bad_lines = 0;
|
||||
let mut valid_lines = 0; // 含 ≥2 个可解析数值 token 的行
|
||||
let mut nonzero_flux_lines = 0; // 流量列（第 2 个 token）非零的有效行
|
||||
|
||||
for line in reader.lines().map_while(Result::ok) {
|
||||
total_lines += 1;
|
||||
if nan_re.is_match(&line) {
|
||||
bad_lines += 1;
|
||||
continue;
|
||||
}
|
||||
// 拆 token，尝试解析为数值。.spec 每行 2 列：波长、流量（FLAM）。
|
||||
let tokens: Vec<&str> = line.split_whitespace().collect();
|
||||
let parsed: Vec<f64> = tokens
|
||||
.iter()
|
||||
.filter_map(|t| parse_fortran_float(t))
|
||||
.filter(|v| v.is_finite())
|
||||
.collect();
|
||||
if parsed.len() >= 2 {
|
||||
valid_lines += 1;
|
||||
// 第 2 个数值列是流量；流量非零才算有效行（避免全零谱）。
|
||||
if parsed[1].abs() > 0.0 {
|
||||
nonzero_flux_lines += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if total_lines == 0 {
|
||||
return Some("spec 为空".to_string());
|
||||
}
|
||||
if bad_lines > 0 {
|
||||
return Some(format!("spec 含 NaN/Inf/溢出行 (共 {} 行)", bad_lines));
|
||||
}
|
||||
if valid_lines < 10 {
|
||||
return Some(format!("spec 有效行数不足 ({}<10)", valid_lines));
|
||||
}
|
||||
if nonzero_flux_lines == 0 {
|
||||
return Some("spec 流量全为零".to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// 从 fort.6（TLUSTY stdout 日志）提取失败诊断提示。
|
||||
///
|
||||
/// # 背景（漏洞 5，P2）
|
||||
/// TLUSTY 失败时旧 note 统一记 `"tlusty rc=N or missing fort.7"`，丢失"为何未收敛"线索。
|
||||
/// 求解器发散时 fort.6 会印 `**** STOP in SOLVE/SOLVES/RYBSOL after ITER N`（来自
|
||||
/// `tlusty208.f:14731/15077/47598`，FORMAT 610）；fort.7 缺失（输入错误、temp 越界等
|
||||
/// `call quit`）时 fort.6 尾部会有 `stop 'msg'` 留言。两类信息都能显著提升归因质量。
|
||||
/// 见 `docs/tlusty&synspec收敛性判断.md` §4 漏洞 5。
|
||||
///
|
||||
/// # 规则
|
||||
/// 1. 优先全文匹配 `STOP in (SOLVE|SOLVES|RYBSOL) after ITER N`（发散求解器名），返回该行。
|
||||
/// 2. 否则取最后 5 行，找含 `stop|quit|error`（忽略大小写）的行返回。
|
||||
/// 3. 都没有 → `None`。
|
||||
pub fn extract_failure_hint(fort6_path: &Path) -> Option<String> {
|
||||
let file = File::open(fort6_path).ok()?;
|
||||
let reader = BufReader::new(file);
|
||||
let solver_re = SOLVER_STOP_RE.get_or_init(|| {
|
||||
Regex::new(r"(?i)STOP\s+in\s+(SOLVES?|RYBSOL)\s+after\s+ITER\s+\d+").unwrap()
|
||||
});
|
||||
// 单词边界 \b 避免子串误报（如 "stopping criterion"/"no error detected"）。
|
||||
// 仅在已确认失败的语境下提取 hint，误报后果仅是 note 多一行提示，不影响 converged 判定。
|
||||
let quit_re = QUIT_RE.get_or_init(|| Regex::new(r"(?i)\b(stop|quit|error)\b").unwrap());
|
||||
|
||||
let mut all_lines: Vec<String> = Vec::new();
|
||||
for line in reader.lines().map_while(Result::ok) {
|
||||
all_lines.push(line);
|
||||
}
|
||||
|
||||
// 1. 全文找发散求解器 STOP 行（取最后一次出现）。
|
||||
for line in all_lines.iter().rev() {
|
||||
if solver_re.is_match(line) {
|
||||
return Some(line.trim().to_string());
|
||||
}
|
||||
}
|
||||
// 2. 尾部 5 行找 call quit / stop / error 留言（取最后一条）。
|
||||
for line in all_lines.iter().rev().take(5) {
|
||||
if quit_re.is_match(line) {
|
||||
return Some(line.trim().to_string());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -245,6 +410,215 @@ mod tests {
|
||||
let inf_file_path = dir.path().join("inf.7");
|
||||
std::fs::write(&inf_file_path, "Inf 2 3\nInfinity 5 6\n7 8 9\n").unwrap();
|
||||
assert!(atmosphere_has_nan(&inf_file_path));
|
||||
|
||||
// 漏洞4修复：0 行容忍——单行 NaN（1/3 行）也应判定无效。
|
||||
let single_nan_path = dir.path().join("single_nan.7");
|
||||
std::fs::write(&single_nan_path, "NaN 2 3\n4 5 6\n7 8 9\n").unwrap();
|
||||
assert!(
|
||||
atmosphere_has_nan(&single_nan_path),
|
||||
"单行 NaN（表层发散的典型形态）应被判为无效"
|
||||
);
|
||||
|
||||
// 漏洞4补充盲区：超高指数科学记数法（E+300 以上）是数值发散产物，物理上无意义。
|
||||
// 正常大气物理量天花板在 E+07，应被判无效。
|
||||
let huge_exp_path = dir.path().join("huge_exp.7");
|
||||
std::fs::write(
|
||||
&huge_exp_path,
|
||||
"1.0E+07 2.0 3.0\n9.99E+308 5.0 6.0\n7.0 8.0 9.0\n",
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
atmosphere_has_nan(&huge_exp_path),
|
||||
"超高指数 E+308 应被判为无效（数值发散产物）"
|
||||
);
|
||||
|
||||
// 边界：合法的极高正指数（E+50，仍远超物理上限但未被超高指数分支命中）
|
||||
// —— 此测试确认正则只匹配 E+300+，不误伤。
|
||||
// 注：E+50 在物理上无意义但 regex 不拦，由物理论据留给将来收紧。
|
||||
let high_but_ok_path = dir.path().join("high_ok.7");
|
||||
std::fs::write(&high_but_ok_path, "1.0E+50 2.0 3.0\n4.0 5.0 6.0\n").unwrap();
|
||||
assert!(
|
||||
!atmosphere_has_nan(&high_but_ok_path),
|
||||
"E+50 不在 E+300+ 检测范围，不应被超高指数分支误判"
|
||||
);
|
||||
|
||||
// 边界：合法的极小值（E-300）不得被误伤（正则锁定正号 \\+）。
|
||||
let tiny_path = dir.path().join("tiny.7");
|
||||
std::fs::write(&tiny_path, "1.0E-300 2.0 3.0\n4.0 5.0 6.0\n").unwrap();
|
||||
assert!(
|
||||
!atmosphere_has_nan(&tiny_path),
|
||||
"极小值 E-300（合法）不得被超高指数分支误伤"
|
||||
);
|
||||
}
|
||||
|
||||
/// 漏洞1修复：SYNSPEC `.spec` 内容校验。
|
||||
#[test]
|
||||
fn test_spec_validation() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
// 1. 正常谱：≥10 行、每行 2 数值列、流量非零 → None（有效）
|
||||
let good_spec = dir.path().join("good.spec");
|
||||
let mut content = String::new();
|
||||
for i in 0..15 {
|
||||
content.push_str(&format!(
|
||||
" {:.5} {:.6E}\n",
|
||||
1400.0 + i as f64 * 0.5,
|
||||
1e-12
|
||||
));
|
||||
}
|
||||
std::fs::write(&good_spec, &content).unwrap();
|
||||
assert!(
|
||||
spec_is_valid(&good_spec).is_none(),
|
||||
"正常多行数值谱应判有效"
|
||||
);
|
||||
|
||||
// 2. 脏谱（仓库内真实失败样本形态）：仅 2 行 + Infinity/NaN → Some
|
||||
let dirty_spec = dir.path().join("dirty.spec");
|
||||
std::fs::write(
|
||||
&dirty_spec,
|
||||
" Infinity NaN\n 1410.00005 NaN\n",
|
||||
)
|
||||
.unwrap();
|
||||
let reason = spec_is_valid(&dirty_spec).expect("脏谱应判无效");
|
||||
assert!(
|
||||
reason.contains("NaN") || reason.contains("Inf"),
|
||||
"脏谱原因应提及 NaN/Inf，实际：{}",
|
||||
reason
|
||||
);
|
||||
|
||||
// 3. 行数不足（< 10）→ Some
|
||||
let short_spec = dir.path().join("short.spec");
|
||||
std::fs::write(
|
||||
&short_spec,
|
||||
" 1400.0 1.0E-12\n 1401.0 1.0E-12\n 1402.0 1.0E-12\n",
|
||||
)
|
||||
.unwrap();
|
||||
let reason = spec_is_valid(&short_spec).expect("行数不足应判无效");
|
||||
assert!(
|
||||
reason.contains("行数不足"),
|
||||
"行数不足原因，实际：{}",
|
||||
reason
|
||||
);
|
||||
|
||||
// 4. 全零流量 → Some
|
||||
let zero_spec = dir.path().join("zero.spec");
|
||||
let mut content = String::new();
|
||||
for i in 0..15 {
|
||||
content.push_str(&format!(" {:.5} {:.6E}\n", 1400.0 + i as f64 * 0.5, 0.0));
|
||||
}
|
||||
std::fs::write(&zero_spec, &content).unwrap();
|
||||
let reason = spec_is_valid(&zero_spec).expect("全零谱应判无效");
|
||||
assert!(reason.contains("全为零"), "全零谱原因，实际：{}", reason);
|
||||
|
||||
// 5. 文件缺失 → Some
|
||||
let missing_spec = dir.path().join("missing.spec");
|
||||
let reason = spec_is_valid(&missing_spec).expect("文件缺失应判无效");
|
||||
assert!(reason.contains("缺失"), "文件缺失原因，实际：{}", reason);
|
||||
|
||||
// 6. Fortran 字段溢出 *** → Some
|
||||
let overflow_spec = dir.path().join("overflow.spec");
|
||||
let mut content = String::new();
|
||||
for i in 0..15 {
|
||||
content.push_str(&format!(
|
||||
" {:.5} {:.6E}\n",
|
||||
1400.0 + i as f64 * 0.5,
|
||||
1e-12
|
||||
));
|
||||
}
|
||||
// 第 3 行混入溢出行
|
||||
let mut lines: Vec<&str> = content.lines().collect();
|
||||
if lines.len() > 2 {
|
||||
lines[2] = " 1401.0 ********";
|
||||
}
|
||||
std::fs::write(&overflow_spec, lines.join("\n") + "\n").unwrap();
|
||||
let reason = spec_is_valid(&overflow_spec).expect("含溢出标记的谱应判无效");
|
||||
assert!(reason.contains("溢出"), "溢出标记原因，实际：{}", reason);
|
||||
|
||||
// 7. 超高指数（E+308，数值发散产物）→ Some。
|
||||
// 双重作用：(a) 验证 spec_is_valid 拦截超高指数；
|
||||
// (b) 守护 NAN_RE OnceLock 一致性——spec_is_valid 与 atmosphere_has_nan
|
||||
// 共用 NAN_RE，若两处正则字符串不一致（OnceLock 竞态），此断言会
|
||||
// 在多线程测试时间歇性失败。
|
||||
let huge_exp_spec = dir.path().join("huge_exp.spec");
|
||||
let mut content = String::new();
|
||||
for i in 0..15 {
|
||||
content.push_str(&format!(
|
||||
" {:.5} {:.6E}\n",
|
||||
1400.0 + i as f64 * 0.5,
|
||||
1e-12
|
||||
));
|
||||
}
|
||||
let mut lines: Vec<String> = content.lines().map(|s| s.to_string()).collect();
|
||||
if lines.len() > 2 {
|
||||
lines[2] = " 1401.0 9.99E+308".to_string();
|
||||
}
|
||||
std::fs::write(&huge_exp_spec, lines.join("\n") + "\n").unwrap();
|
||||
let reason = spec_is_valid(&huge_exp_spec).expect("含超高指数的谱应判无效");
|
||||
assert!(
|
||||
reason.contains("NaN") || reason.contains("Inf") || reason.contains("溢出"),
|
||||
"超高指数应被识别为无效数值，实际原因：{}",
|
||||
reason
|
||||
);
|
||||
}
|
||||
|
||||
/// 漏洞5修复：fort.6 失败诊断提示提取。
|
||||
#[test]
|
||||
fn test_extract_failure_hint() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
// 1. 发散求解器 STOP 行 → 提取该行
|
||||
let solve_log = dir.path().join("solve.6");
|
||||
std::fs::write(
|
||||
&solve_log,
|
||||
" KANTOROVICH acceleration: ITER 7\n\
|
||||
**** STOP in SOLVE after ITER 8\n\
|
||||
Max change: 3.59E+20\n",
|
||||
)
|
||||
.unwrap();
|
||||
let hint = extract_failure_hint(&solve_log).expect("发散日志应返回 hint");
|
||||
assert!(
|
||||
hint.contains("STOP in SOLVE"),
|
||||
"应提取求解器 STOP 行，实际：{}",
|
||||
hint
|
||||
);
|
||||
|
||||
// RYBSOL 路径
|
||||
let rybsol_log = dir.path().join("rybsol.6");
|
||||
std::fs::write(&rybsol_log, "**** STOP in RYBSOL after ITER 3\n").unwrap();
|
||||
let hint = extract_failure_hint(&rybsol_log).expect("RYBSOL 日志应返回 hint");
|
||||
assert!(hint.contains("RYBSOL"));
|
||||
|
||||
// 2. call quit 留言（fort.7 缺失场景，如 temp 越界）→ 提取尾部
|
||||
let quit_log = dir.path().join("quit.6");
|
||||
std::fs::write(
|
||||
&quit_log,
|
||||
" some normal output\n\
|
||||
partf; temp<1000 K\n\
|
||||
stop 'partf; temp<1000 K'\n",
|
||||
)
|
||||
.unwrap();
|
||||
let hint = extract_failure_hint(&quit_log).expect("call quit 日志应返回 hint");
|
||||
assert!(
|
||||
hint.contains("partf") || hint.to_lowercase().contains("stop"),
|
||||
"应提取 quit 留言，实际：{}",
|
||||
hint
|
||||
);
|
||||
|
||||
// 3. 无关日志（正常收敛，无 STOP/quit）→ None
|
||||
let clean_log = dir.path().join("clean.6");
|
||||
std::fs::write(
|
||||
&clean_log,
|
||||
" KANTOROVICH acceleration: ITER 10\n Converged.\n",
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
extract_failure_hint(&clean_log).is_none(),
|
||||
"正常日志不应返回 hint"
|
||||
);
|
||||
|
||||
// 4. 文件缺失 → None
|
||||
let missing = dir.path().join("missing.6");
|
||||
assert!(extract_failure_hint(&missing).is_none());
|
||||
}
|
||||
|
||||
/// 验证 parse_fortran_float 对各种数值格式（含 Fortran 无-E 记数法）的解析。
|
||||
@@ -253,7 +627,7 @@ mod tests {
|
||||
// 标准 parse 能覆盖的
|
||||
assert_eq!(parse_fortran_float("100"), Some(100.0));
|
||||
assert_eq!(parse_fortran_float("-0.001"), Some(-0.001));
|
||||
assert_eq!(parse_fortran_float("3.14"), Some(3.14));
|
||||
assert_eq!(parse_fortran_float("2.5"), Some(2.5));
|
||||
assert_eq!(parse_fortran_float("-5.42E+72"), Some(-5.42e72));
|
||||
assert_eq!(parse_fortran_float("1.5e-99"), Some(1.5e-99));
|
||||
assert_eq!(parse_fortran_float("0"), Some(0.0));
|
||||
@@ -336,5 +710,35 @@ mod tests {
|
||||
let res = check_fort9(&fort9, 0.001);
|
||||
assert!(res.converged, "正常收敛行应判为收敛");
|
||||
assert!((res.max_relc - 1.0e-5).abs() < 1e-15);
|
||||
assert_eq!(res.itek_history.len(), 1, "单迭代应只有一拍");
|
||||
assert_eq!(res.itek_history[0].iter, 10);
|
||||
}
|
||||
|
||||
/// Phase 5b itek 全量保真：多迭代 fort.9 → itek_history 逐拍记录（iter → 该次最大 |maximum|）。
|
||||
#[test]
|
||||
fn test_fort9_itek_history_per_iteration() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let fort9 = dir.path().join("multi_iter.9");
|
||||
// 两拍：iter1 最深 max_relc=2.0E-2（depth2）、iter2 最深 max_relc=5.0E-4（depth1）。
|
||||
std::fs::write(
|
||||
&fort9,
|
||||
" 1 1 0 1.0E-3 1.0E-3 1.0E-3 1.0E-2 5 10\n\
|
||||
1 2 0 1.0E-3 1.0E-3 1.0E-3 2.0E-2 5 10\n\
|
||||
2 1 0 1.0E-3 1.0E-3 1.0E-3 5.0E-4 5 10\n\
|
||||
2 2 0 1.0E-3 1.0E-3 1.0E-3 3.0E-4 5 10\n",
|
||||
)
|
||||
.unwrap();
|
||||
let res = check_fort9(&fort9, 0.001);
|
||||
// 末拍（iter2）收敛。
|
||||
assert!(res.converged);
|
||||
assert!((res.max_relc - 5.0e-4).abs() < 1e-15);
|
||||
// itek_history 逐拍完整。
|
||||
assert_eq!(res.itek_history.len(), 2);
|
||||
assert_eq!(res.itek_history[0].iter, 1);
|
||||
assert!((res.itek_history[0].max_relc - 2.0e-2).abs() < 1e-15);
|
||||
assert_eq!(res.itek_history[0].n_depths, 2);
|
||||
assert_eq!(res.itek_history[1].iter, 2);
|
||||
assert!((res.itek_history[1].max_relc - 5.0e-4).abs() < 1e-15);
|
||||
assert_eq!(res.itek_history[1].n_depths, 2);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use crate::config::SynspecConfig;
|
||||
use crate::config::SynspecInput;
|
||||
|
||||
/// Dynamic generator for SYNSPEC fort.55 parameter control file
|
||||
pub fn generate_fort55_content(cfg: &SynspecConfig) -> String {
|
||||
pub fn generate_fort55_content(cfg: &SynspecInput) -> String {
|
||||
let line1 = format!(" {} {} {}", cfg.imode, cfg.idrv, cfg.ifreq);
|
||||
let line2 = " 1 0 0 0";
|
||||
let line3 = " 0 0 0 0 0";
|
||||
@@ -25,7 +25,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_fort55_generation() {
|
||||
let cfg = SynspecConfig {
|
||||
let cfg = SynspecInput {
|
||||
wstart: 3000.0,
|
||||
wend: 7000.0,
|
||||
imode: 0,
|
||||
|
||||
+290
-37
@@ -1,4 +1,6 @@
|
||||
use crate::config::{AtomConfig, IonConfig, TlustyInput};
|
||||
use crate::models::GridPointParams;
|
||||
use std::collections::HashMap;
|
||||
|
||||
struct IonDef {
|
||||
iat: i32,
|
||||
@@ -177,63 +179,193 @@ const IONS_O: &[IonDef] = &[
|
||||
},
|
||||
];
|
||||
|
||||
/// 把对数丰度 logx 格式化为 10^logx 的科学计数法字符串。
|
||||
///
|
||||
/// 审查修复 #N3：深度防御 powf 溢出——config.validate 已把丰度轴限制在 [-20, 10]，
|
||||
/// 但 fmt_abn 也可能被 tlusty_chain 里的元素丰度等其它路径调用。若 logx 超出约 [-308, 308]，
|
||||
/// `10f64.powf(logx)` 会溢出为 Inf/0，写出 `inf` 污染 .5 输入文件导致 TLUSTY 行为未定义。
|
||||
/// 配置层校验在前，此处仅作最终防线：溢出时 saturate 到 f64 可表示范围并按原格式输出。
|
||||
fn fmt_abn(logx: f64) -> String {
|
||||
format!("{:.4E}", 10.0f64.powf(logx))
|
||||
let val = if logx.abs() < 300.0 {
|
||||
10.0f64.powf(logx)
|
||||
} else {
|
||||
// 超出安全范围：clamp 到 0（丰度对数极小）或 f64::MAX（丰度对数极大），
|
||||
// 避免写出 inf。配置层应已拦截，这里不会成为正常路径。
|
||||
if logx < 0.0 {
|
||||
0.0
|
||||
} else {
|
||||
f64::MAX
|
||||
}
|
||||
};
|
||||
format!("{:.4E}", val)
|
||||
}
|
||||
|
||||
/// 元素符号首字母大写归一（"h"→"H", "he"→"He", "HE"→"He"）。
|
||||
/// 容忍用户在 YAML atoms 块用小写/全大写写元素符号。
|
||||
fn capitalize_first(s: &str) -> String {
|
||||
let mut c = s.chars();
|
||||
match c.next() {
|
||||
Some(f) => f.to_uppercase().collect::<String>() + c.as_str().to_lowercase().as_str(),
|
||||
None => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// 默认 ions 能级数据表（与历史 IONS_* 常量等价）。
|
||||
/// 用户未在 `tlusty_input.ions` 配置时使用此表，按 metals 筛选参与元素。
|
||||
fn default_ions(metals: &str) -> Vec<IonConfig> {
|
||||
let mt = metals.to_lowercase();
|
||||
let has_c = mt.contains('c');
|
||||
let has_n = mt.contains('n');
|
||||
let has_o = mt.contains('o');
|
||||
let mut ions: Vec<IonConfig> = Vec::new();
|
||||
// H（恒在）
|
||||
for d in IONS_H {
|
||||
ions.push(IonConfig {
|
||||
iat: d.iat,
|
||||
iz: d.iz,
|
||||
nlevs: d.nlevs,
|
||||
typion: d.typion.to_string(),
|
||||
filei: d.filei.to_string(),
|
||||
});
|
||||
}
|
||||
// He（恒在）
|
||||
for d in IONS_HE {
|
||||
ions.push(IonConfig {
|
||||
iat: d.iat,
|
||||
iz: d.iz,
|
||||
nlevs: d.nlevs,
|
||||
typion: d.typion.to_string(),
|
||||
filei: d.filei.to_string(),
|
||||
});
|
||||
}
|
||||
let extend = |ions: &mut Vec<IonConfig>, table: &[IonDef]| {
|
||||
for d in table {
|
||||
ions.push(IonConfig {
|
||||
iat: d.iat,
|
||||
iz: d.iz,
|
||||
nlevs: d.nlevs,
|
||||
typion: d.typion.to_string(),
|
||||
filei: d.filei.to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
if has_c {
|
||||
extend(&mut ions, IONS_C);
|
||||
}
|
||||
if has_n {
|
||||
extend(&mut ions, IONS_N);
|
||||
}
|
||||
if has_o {
|
||||
extend(&mut ions, IONS_O);
|
||||
}
|
||||
ions
|
||||
}
|
||||
|
||||
/// Constructs the complete text of a `.5` input file for TLUSTY
|
||||
///
|
||||
/// `input_cfg`：全局输入参数（NFREAD / atoms.mode / ions 表）。
|
||||
/// None → 走代码内硬编码默认（与改动前行为完全一致，向后兼容）。
|
||||
pub fn make_input5(
|
||||
params: &GridPointParams,
|
||||
lte: &str,
|
||||
ltgray: &str,
|
||||
metals: &str,
|
||||
ilvlin: i32,
|
||||
input_cfg: Option<&TlustyInput>,
|
||||
) -> String {
|
||||
let mt = metals.to_lowercase();
|
||||
let has_c = mt.contains('c');
|
||||
let has_n = mt.contains('n');
|
||||
let has_o = mt.contains('o');
|
||||
|
||||
// Atoms block
|
||||
let mut atom_rows: Vec<(i32, String)> = vec![
|
||||
(2, "0.".to_string()), // 1 H
|
||||
(2, fmt_abn(*params.loghe)), // 2 He
|
||||
(0, "0.".to_string()), // 3 Li
|
||||
(0, "0.".to_string()), // 4 Be
|
||||
(0, "0.".to_string()), // 5 B
|
||||
];
|
||||
// 元素符号 → (mode, abn)。mode 优先取 input_cfg.atoms 覆写，否则走默认。
|
||||
// abn（丰度）永远由 GridPointParams 计算——每网格点不同，不可全局配置。
|
||||
// 查找时对 key 做首字母大写归一（H/He/Li/Be/B/C/N/O），容忍用户写小写 "h"/"he"。
|
||||
//
|
||||
// M1 修复：config.rs 的 `atoms` 是 Serde 反序列化的 `HashMap<String, AtomConfig>`，
|
||||
// 键按 YAML 原样存储（无规范化）。若用户写 `he:`/`HE:`，原实现只归一探针、不归一存储键，
|
||||
// 查不到规范键 "He" → mode 覆写静默回落默认（错误物理：想要的 LTE-only 变统计平衡）。
|
||||
// 这里把用户键也按同一 capitalize_first 规则归一一次，保证任意大小写都能命中。
|
||||
let atoms_normalized: Option<HashMap<String, &AtomConfig>> = input_cfg.map(|c| {
|
||||
let mut m = HashMap::with_capacity(c.atoms.len());
|
||||
for (k, v) in &c.atoms {
|
||||
let canon = capitalize_first(k);
|
||||
// 归一撞键检测：用户同时写 "C" 与 "c"（或 "He"/"HE"）等大小写变体时，两键归一为
|
||||
// 同一规范键。原 collect() 静默覆盖（迭代序非确定 → 取哪个任选）。现显式提示
|
||||
// 让歧义可见，避免 mode 覆写悄悄取到任意一个变体。
|
||||
if let Some(_prev) = m.insert(canon.clone(), v) {
|
||||
tracing::warn!(
|
||||
"atoms 键 {:?} 与已有键（归一后同为 {:?}）冲突：取最后读取项，请改用单一规范键",
|
||||
k,
|
||||
canon
|
||||
);
|
||||
}
|
||||
}
|
||||
m
|
||||
});
|
||||
let atom_mode = |sym: &str, default_mode: i32| -> i32 {
|
||||
atoms_normalized
|
||||
.as_ref()
|
||||
.and_then(|m| m.get(&capitalize_first(sym)))
|
||||
.and_then(|a| a.mode)
|
||||
.unwrap_or(default_mode)
|
||||
};
|
||||
let atom_modpf = |sym: &str| -> i32 {
|
||||
atoms_normalized
|
||||
.as_ref()
|
||||
.and_then(|m| m.get(&capitalize_first(sym)))
|
||||
.and_then(|a| a.modpf)
|
||||
.unwrap_or(0)
|
||||
};
|
||||
|
||||
// Atoms block：H/He 恒在；Li/Be/B 恒在（mode=0 不参与）；C/N/O 按 metals。
|
||||
let mut atom_rows: Vec<(i32, String, i32)> = vec![
|
||||
(atom_mode("H", 2), "0.".to_string(), atom_modpf("H")), // 1 H
|
||||
(atom_mode("He", 2), fmt_abn(*params.loghe), atom_modpf("He")), // 2 He
|
||||
(atom_mode("Li", 0), "0.".to_string(), atom_modpf("Li")), // 3 Li
|
||||
(atom_mode("Be", 0), "0.".to_string(), atom_modpf("Be")), // 4 Be
|
||||
(atom_mode("B", 0), "0.".to_string(), atom_modpf("B")), // 5 B
|
||||
];
|
||||
if has_c {
|
||||
atom_rows.push((2, fmt_abn(*params.logc))); // 6 C
|
||||
atom_rows.push((atom_mode("C", 2), fmt_abn(*params.logc), atom_modpf("C")));
|
||||
// 6 C
|
||||
}
|
||||
if has_n {
|
||||
atom_rows.push((2, fmt_abn(*params.logn))); // 7 N
|
||||
atom_rows.push((atom_mode("N", 2), fmt_abn(*params.logn), atom_modpf("N")));
|
||||
// 7 N
|
||||
}
|
||||
if has_o {
|
||||
atom_rows.push((2, fmt_abn(*params.logo))); // 8 O
|
||||
atom_rows.push((atom_mode("O", 2), fmt_abn(*params.logo), atom_modpf("O")));
|
||||
// 8 O
|
||||
}
|
||||
|
||||
let natoms =
|
||||
5 + (if has_c { 1 } else { 0 }) + (if has_n { 1 } else { 0 }) + (if has_o { 1 } else { 0 });
|
||||
let natoms = atom_rows.len() as i32;
|
||||
|
||||
let mut atoms_block = format!(" {}\n* mode abn modpf\n", natoms);
|
||||
for (mode, abn) in &atom_rows {
|
||||
atoms_block.push_str(&format!(" {} {} 0\n", mode, abn));
|
||||
for (mode, abn, modpf) in &atom_rows {
|
||||
atoms_block.push_str(&format!(" {} {} {}\n", mode, abn, modpf));
|
||||
}
|
||||
|
||||
// Ions block
|
||||
let mut ions: Vec<&IonDef> = Vec::new();
|
||||
ions.extend(IONS_H.iter());
|
||||
ions.extend(IONS_HE.iter());
|
||||
if has_c {
|
||||
ions.extend(IONS_C.iter());
|
||||
}
|
||||
if has_n {
|
||||
ions.extend(IONS_N.iter());
|
||||
}
|
||||
if has_o {
|
||||
ions.extend(IONS_O.iter());
|
||||
}
|
||||
// Ions block：用户配置非空时完全替换默认表，仍按 metals 筛选 iat。
|
||||
let ions: Vec<IonConfig> = if let Some(cfg) = input_cfg {
|
||||
if cfg.ions.is_empty() {
|
||||
default_ions(metals)
|
||||
} else {
|
||||
// 用户自定义表——按 metals 筛选参与元素（H/He 恒在，C/N/O 按 metals）。
|
||||
cfg.ions
|
||||
.iter()
|
||||
.filter(|ion| match ion.iat {
|
||||
1 | 2 => true, // H, He 恒在
|
||||
6 => has_c,
|
||||
7 => has_n,
|
||||
8 => has_o,
|
||||
_ => true, // 用户自定义元素不筛
|
||||
})
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
} else {
|
||||
default_ions(metals)
|
||||
};
|
||||
|
||||
let mut ions_block = "*iat iz nlevs ilast ilvlin nonstd typion filei\n*\n".to_string();
|
||||
for ion in &ions {
|
||||
@@ -243,9 +375,6 @@ pub fn make_input5(
|
||||
// (tests/tlusty/hhe/fort.5) 逐字节一致：
|
||||
// iat 结束于 col4(|iat|=4), iz col10(+6), nlevs col16(+6),
|
||||
// ilast col23(+7), ilvl col30(+7), nonstd col37(+7)。
|
||||
// 实测：此宽列宽与窄列宽对 tlusty 输出（fort.7/9 等）完全相同（list-directed I/O
|
||||
// 列宽无关），但对齐真实文件便于与历史参考 diff、符合 tlusty 官方输入惯例。
|
||||
// 历史上曾用 `" {} {:2} {:5}..."`（窄列宽，与 Python 旧实现一致但偏离真实 fort.5）。
|
||||
ions_block.push_str(&format!(
|
||||
"{:>4}{:>6}{:>6}{:>7}{:>7}{:>7} '{}' '{}'\n",
|
||||
ion.iat, ion.iz, ion.nlevs, ilast, ilvl, 0, ion.typion, ion.filei
|
||||
@@ -257,13 +386,16 @@ pub fn make_input5(
|
||||
0, 0, 0, -1, 0, 0, " ", " "
|
||||
));
|
||||
|
||||
// NFREAD：从配置读，None → 默认 2000。
|
||||
let nfread = input_cfg.map(|c| c.nfread).unwrap_or(2000);
|
||||
|
||||
format!(
|
||||
"{:.1} {:.1} ! TEFF, GRAV\n \
|
||||
{} {} ! LTE, LTGRAY\n \
|
||||
'nst' ! name of file containing non-standard flags\n\
|
||||
*-----------------------------------------------------------------\n\
|
||||
* frequencies\n \
|
||||
2000 ! NFREAD\n\
|
||||
{} ! NFREAD\n\
|
||||
*-----------------------------------------------------------------\n\
|
||||
* data for atoms\n\
|
||||
{}\
|
||||
@@ -271,7 +403,7 @@ pub fn make_input5(
|
||||
* data for ions\n*\n\
|
||||
{}\
|
||||
*\n* end\n",
|
||||
params.teff, params.logg, lte, ltgray, atoms_block, ions_block
|
||||
params.teff, params.logg, lte, ltgray, nfread, atoms_block, ions_block
|
||||
)
|
||||
}
|
||||
|
||||
@@ -289,7 +421,7 @@ mod tests {
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
};
|
||||
let input5 = make_input5(¶ms, "F", "F", "cno", 100);
|
||||
let input5 = make_input5(¶ms, "F", "F", "cno", 100, None);
|
||||
assert!(input5.contains("35000.0 5.5"));
|
||||
assert!(input5.contains("data/h1.dat"));
|
||||
assert!(input5.contains("data/c1.dat"));
|
||||
@@ -312,7 +444,7 @@ mod tests {
|
||||
logn: (-1.0).into(),
|
||||
logo: (-1.0).into(),
|
||||
};
|
||||
let input5 = make_input5(¶ms, "T", "T", "", 100);
|
||||
let input5 = make_input5(¶ms, "T", "T", "", 100, None);
|
||||
let lines: Vec<&str> = input5.lines().collect();
|
||||
|
||||
// 真实 fort.5 的 ions 数据行（数值部分 + typion/filei）。
|
||||
@@ -346,4 +478,125 @@ mod tests {
|
||||
"ions 行数值部分必须与真实 fort.5 逐字节一致（宽列宽）"
|
||||
);
|
||||
}
|
||||
|
||||
/// 回归守护：None 配置时 NFREAD=2000、atoms.mode 走默认（H/He=2，Li/Be/B=0）。
|
||||
#[test]
|
||||
fn test_make_input5_none_cfg_defaults() {
|
||||
let params = GridPointParams {
|
||||
teff: 35000.0.into(),
|
||||
logg: 5.5.into(),
|
||||
loghe: (-1.0).into(),
|
||||
logc: (-2.0).into(),
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
};
|
||||
let input5 = make_input5(¶ms, "F", "F", "cno", 100, None);
|
||||
// NFREAD 默认 2000
|
||||
assert!(input5.contains("2000"), "None 配置时 NFREAD 应为默认 2000");
|
||||
// H 的 mode=2（显式）
|
||||
assert!(input5.contains(" 2 0. 0"), "H 的 mode 应为默认 2");
|
||||
}
|
||||
|
||||
/// 用户配置生效：nfread 覆写、atoms.mode 覆写、ions 自定义表。
|
||||
#[test]
|
||||
fn test_make_input5_user_cfg_override() {
|
||||
let params = GridPointParams {
|
||||
teff: 35000.0.into(),
|
||||
logg: 5.5.into(),
|
||||
loghe: (-1.0).into(),
|
||||
logc: (-2.0).into(),
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
};
|
||||
let mut atoms = std::collections::HashMap::new();
|
||||
atoms.insert(
|
||||
"C".to_string(),
|
||||
crate::config::AtomConfig {
|
||||
mode: Some(1),
|
||||
modpf: None,
|
||||
},
|
||||
);
|
||||
let cfg = TlustyInput {
|
||||
nfread: 500,
|
||||
atoms,
|
||||
ions: vec![],
|
||||
nst_extra_keys: vec![],
|
||||
};
|
||||
let input5 = make_input5(¶ms, "F", "F", "cno", 100, Some(&cfg));
|
||||
// NFREAD 被覆写为 500
|
||||
assert!(
|
||||
input5.contains("500") && !input5.contains("2000"),
|
||||
"NFREAD 应被用户配置覆写为 500"
|
||||
);
|
||||
// C 的 mode 被覆写为 1（隐式），而非默认 2
|
||||
// atoms 行格式 " {mode} {abn} {modpf}"，C 的 abn 是 fmt_abn(logc)
|
||||
let c_line = input5
|
||||
.lines()
|
||||
.find(|l| l.contains(&format!("{:.4E}", 10.0f64.powf(-2.0))))
|
||||
.expect("应找到 C 的 atoms 行");
|
||||
assert!(
|
||||
c_line.trim_start().starts_with("1"),
|
||||
"C 的 mode 应被覆写为 1（隐式），实际: {}",
|
||||
c_line
|
||||
);
|
||||
}
|
||||
|
||||
/// M1 回归：`atoms` 键大小写不敏感查找。用户写小写/全大写元素键（`c:`/`HE:`）时，
|
||||
/// 与规范键（`C`/`He`）必须同样命中覆写，否则 mode 覆写会静默回落默认（错误物理）。
|
||||
#[test]
|
||||
fn test_atoms_key_case_insensitive() {
|
||||
let params = GridPointParams {
|
||||
teff: 35000.0.into(),
|
||||
logg: 5.5.into(),
|
||||
loghe: (-1.0).into(),
|
||||
logc: (-2.0).into(),
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
};
|
||||
// 用户用小写 "c" 与全大写 "HE" 覆写 mode。
|
||||
let mut atoms = std::collections::HashMap::new();
|
||||
atoms.insert(
|
||||
"c".to_string(),
|
||||
crate::config::AtomConfig {
|
||||
mode: Some(1),
|
||||
modpf: None,
|
||||
},
|
||||
);
|
||||
atoms.insert(
|
||||
"HE".to_string(),
|
||||
crate::config::AtomConfig {
|
||||
mode: Some(0),
|
||||
modpf: None,
|
||||
},
|
||||
);
|
||||
let cfg = TlustyInput {
|
||||
nfread: 2000,
|
||||
atoms,
|
||||
ions: vec![],
|
||||
nst_extra_keys: vec![],
|
||||
};
|
||||
let input5 = make_input5(¶ms, "F", "F", "cno", 100, Some(&cfg));
|
||||
// C（键 "c" 小写）应命中得到 mode=1，而非默认 2。
|
||||
let c_abn = format!("{:.4E}", 10.0f64.powf(-2.0));
|
||||
let c_line = input5
|
||||
.lines()
|
||||
.find(|l| l.contains(&c_abn))
|
||||
.expect("应找到 C 的 atoms 行");
|
||||
assert!(
|
||||
c_line.trim_start().starts_with("1"),
|
||||
"小写键 c 应覆写 C 的 mode 为 1，实际: {}",
|
||||
c_line
|
||||
);
|
||||
// He（键 "HE" 全大写）应命中得到 mode=0，而非默认 2。
|
||||
let he_abn = format!("{:.4E}", 10.0f64.powf(-1.0));
|
||||
let he_line = input5
|
||||
.lines()
|
||||
.find(|l| l.contains(&he_abn))
|
||||
.expect("应找到 He 的 atoms 行");
|
||||
assert!(
|
||||
he_line.trim_start().starts_with("0"),
|
||||
"全大写键 HE 应覆写 He 的 mode 为 0，实际: {}",
|
||||
he_line
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+149
-125
@@ -242,7 +242,10 @@ pub struct GridPoint {
|
||||
pub wave: i32,
|
||||
pub status: GridPointStatus,
|
||||
pub attempt_count: i32,
|
||||
pub success_method: Option<String>,
|
||||
/// TLUSTY 阶段收敛策略（TLUSTY 禁用为 NULL）。
|
||||
pub tlusty_success_method: Option<String>,
|
||||
/// 光谱阶段收敛策略（TLUSTY-only 为 NULL）。
|
||||
pub synspec_success_method: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
@@ -251,7 +254,9 @@ pub enum GridPointStatus {
|
||||
Pending,
|
||||
Queued,
|
||||
Running,
|
||||
Converged,
|
||||
/// Phase 7c：由 `Converged` 改名——点级"管线完成"（大气收敛 + 光谱合成），
|
||||
/// 消除 TLUSTY-first 的"大气收敛"误读。DB 值 'completed' 经 M9 迁为 'completed'。
|
||||
Completed,
|
||||
Failed,
|
||||
}
|
||||
|
||||
@@ -261,7 +266,7 @@ impl std::fmt::Display for GridPointStatus {
|
||||
GridPointStatus::Pending => "pending",
|
||||
GridPointStatus::Queued => "queued",
|
||||
GridPointStatus::Running => "running",
|
||||
GridPointStatus::Converged => "converged",
|
||||
GridPointStatus::Completed => "completed",
|
||||
GridPointStatus::Failed => "failed",
|
||||
};
|
||||
write!(f, "{}", s)
|
||||
@@ -273,7 +278,8 @@ impl From<&str> for GridPointStatus {
|
||||
match s {
|
||||
"queued" => GridPointStatus::Queued,
|
||||
"running" => GridPointStatus::Running,
|
||||
"converged" | "done" => GridPointStatus::Converged,
|
||||
// 7c：'completed' 是权威值；'converged'/'done' 为 legacy 别名（M9 迁移前旧数据/旧代码）。
|
||||
"completed" | "converged" | "done" => GridPointStatus::Completed,
|
||||
"failed" => GridPointStatus::Failed,
|
||||
_ => GridPointStatus::Pending,
|
||||
}
|
||||
@@ -288,7 +294,7 @@ impl From<&str> for GridPointStatus {
|
||||
/// 的处理；失败后的策略链回退**只由启动时的策略链（回退优先级排序）驱动**，不受策略门控。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum StagePolicy {
|
||||
pub enum ResumePolicy {
|
||||
/// 跳过已收敛、重试已失败：启动时把已失败点打回 pending 重试，收敛点保留（增量+重试失败）。
|
||||
/// 默认值。
|
||||
#[default]
|
||||
@@ -299,22 +305,22 @@ pub enum StagePolicy {
|
||||
SkipFailed,
|
||||
}
|
||||
|
||||
impl StagePolicy {
|
||||
impl ResumePolicy {
|
||||
/// 序列化为 DB 文本列存储用的 snake_case 字符串。
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
StagePolicy::SkipConverged => "skip_converged",
|
||||
StagePolicy::ForceRecompute => "force_recompute",
|
||||
StagePolicy::SkipFailed => "skip_failed",
|
||||
ResumePolicy::SkipConverged => "skip_converged",
|
||||
ResumePolicy::ForceRecompute => "force_recompute",
|
||||
ResumePolicy::SkipFailed => "skip_failed",
|
||||
}
|
||||
}
|
||||
|
||||
/// 从 DB 文本列回读；非法值兜底为默认 SkipConverged（防注入与脏数据）。
|
||||
pub fn from_str_lossy(s: &str) -> Self {
|
||||
match s {
|
||||
"force_recompute" => StagePolicy::ForceRecompute,
|
||||
"skip_failed" => StagePolicy::SkipFailed,
|
||||
_ => StagePolicy::SkipConverged,
|
||||
"force_recompute" => ResumePolicy::ForceRecompute,
|
||||
"skip_failed" => ResumePolicy::SkipFailed,
|
||||
_ => ResumePolicy::SkipConverged,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -324,16 +330,16 @@ impl StagePolicy {
|
||||
/// 见 docs/task_engine_decoupling_design.md §3：嵌套式单阶段配置模型，
|
||||
/// 包含三个正交维度：enabled / policy / strategies。
|
||||
///
|
||||
/// 为避免与 `common::config::StageConfig`（迭代步进参数）同名冲突，命名为
|
||||
/// `EngineStageConfig`。
|
||||
/// 为避免与 `common::config::ChainStep`（迭代步进参数）同名冲突，命名为
|
||||
/// `PhaseConfig`。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct EngineStageConfig {
|
||||
pub struct PhaseConfig {
|
||||
/// 是否在当前计算流中启用该阶段。
|
||||
#[serde(default = "default_engine_stage_enabled")]
|
||||
pub enabled: bool,
|
||||
/// 决定如何处理历史记录。
|
||||
#[serde(default)]
|
||||
pub policy: StagePolicy,
|
||||
pub policy: ResumePolicy,
|
||||
/// 策略链队列（按回退优先级排序），如 `["cold_run", "seed_step"]`。
|
||||
/// 节点总是执行 `strategies[0]`；失败后由服务端弹出首项，下一顺位顶上。
|
||||
#[serde(default)]
|
||||
@@ -344,12 +350,12 @@ fn default_engine_stage_enabled() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
impl EngineStageConfig {
|
||||
impl PhaseConfig {
|
||||
/// TLUSTY 阶段默认配置：启用、增量、策略链 `[cold_run, seed_step]`。
|
||||
pub fn default_tlusty() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
policy: StagePolicy::SkipConverged,
|
||||
policy: ResumePolicy::SkipConverged,
|
||||
strategies: vec!["cold_run".to_string(), "seed_step".to_string()],
|
||||
}
|
||||
}
|
||||
@@ -358,7 +364,7 @@ impl EngineStageConfig {
|
||||
pub fn default_synspec() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
policy: StagePolicy::SkipConverged,
|
||||
policy: ResumePolicy::SkipConverged,
|
||||
strategies: vec!["standard".to_string()],
|
||||
}
|
||||
}
|
||||
@@ -382,20 +388,18 @@ impl EngineStageConfig {
|
||||
|
||||
/// Task execution specification sent to Node
|
||||
///
|
||||
/// 注：`EngineStageConfig` 刻意**不实现 `Default`**——阶段默认值随阶段而异（TLUSTY
|
||||
/// 注：`PhaseConfig` 刻意**不实现 `Default`**——阶段默认值随阶段而异（TLUSTY
|
||||
/// `[cold_run, seed_step]` vs SYNSPEC `[standard]`），无中立的默认语义。构造某阶段的配置请用
|
||||
/// `..EngineStageConfig::default_tlusty()` / `..EngineStageConfig::default_synspec()`，
|
||||
/// `..PhaseConfig::default_tlusty()` / `..PhaseConfig::default_synspec()`，
|
||||
/// 避免把 TLUSTY 默认链误用到 synspec。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TaskSpec {
|
||||
pub task_id: Uuid,
|
||||
pub point_name: String,
|
||||
pub params: GridPointParams,
|
||||
/// **已废弃**：保留以兼容历史 MQ 在途消息与旧节点。新代码应读
|
||||
/// `tlusty_config.strategies[0]` 判定当前 TLUSTY 策略。
|
||||
/// 该字段仍是必填（serde 反序列化要求），调度器在派发时会据
|
||||
/// `tlusty_config.strategies[0]` 同步设置它，保证旧节点能正常工作。
|
||||
pub task_type: TaskType,
|
||||
/// **双义**（Phase 7a 标注）：TLUSTY 启用时 = seed_step 热启动的近邻种子点；
|
||||
/// SYNSPEC-only（TLUSTY 关闭）时 = 光谱输入大气来源点（由 atmosphere_ref/point_name 决定，
|
||||
/// 本字段此时恒 None）。executor 以 `tlusty_config.enabled` 区分两种语义。
|
||||
pub seed_point_name: Option<String>,
|
||||
pub timeout_sec: u64,
|
||||
/// 所属工作流名称，用于按工作流隔离队列清理（stop_workflow 只清当前工作流的任务）。
|
||||
@@ -408,46 +412,37 @@ pub struct TaskSpec {
|
||||
pub wave: i32,
|
||||
/// TLUSTY 阶段独立配置（见 docs/task_engine_decoupling_design.md §3）。
|
||||
/// 旧 payload 反序列化时缺省为 `default_tlusty()`。
|
||||
#[serde(default = "EngineStageConfig::default_tlusty")]
|
||||
pub tlusty_config: EngineStageConfig,
|
||||
#[serde(default = "PhaseConfig::default_tlusty")]
|
||||
pub tlusty_config: PhaseConfig,
|
||||
/// SYNSPEC 阶段独立配置。旧 payload 反序列化时缺省为 `default_synspec()`。
|
||||
#[serde(default = "EngineStageConfig::default_synspec")]
|
||||
pub synspec_config: EngineStageConfig,
|
||||
/// SYNSPEC 数值参数（波长范围等，对应 `config::SynspecConfig`）。
|
||||
#[serde(default = "PhaseConfig::default_synspec")]
|
||||
pub synspec_config: PhaseConfig,
|
||||
/// SYNSPEC 数值参数（波长范围等，对应 `config::SynspecInput`）。
|
||||
/// 以 `serde_json::Value` 携带避免 models ↔ config 循环依赖；executor 侧
|
||||
/// 反序列化为 `SynspecConfig` 后透传给 runner。None → runner 用硬编码默认。
|
||||
/// 反序列化为 `SynspecInput` 后透传给 runner。None → runner 用硬编码默认。
|
||||
/// 旧 payload 反序列化时缺省为 None（旧节点本就用默认，无回归）。
|
||||
#[serde(default)]
|
||||
pub synspec_params: Option<serde_json::Value>,
|
||||
/// TLUSTY 物理迭代步进链（lte/nc/nl 多阶段 `config::ChainStep` 数组）。
|
||||
/// 同样以 `serde_json::Value` 携带避免循环依赖；executor 反序列化为
|
||||
/// `Vec<ChainStep>` 后透传给 runner 的 custom_chain 参数。
|
||||
/// None/空 → executor 用 `default_chain_for_strategy` 兜底（按策略名选默认链）。
|
||||
/// 旧 payload 反序列化时缺省为 None（旧节点本就用 default 链，无回归）。
|
||||
#[serde(default)]
|
||||
pub tlusty_chain_params: Option<serde_json::Value>,
|
||||
/// TLUSTY 输入文件（.5 + nst）的全局物理参数（`config::TlustyInput`）。
|
||||
/// NFREAD 频率网格、ions 能级表、nst extra_keys 等不随阶段变化的参数。
|
||||
/// None → runner 用代码内硬编码默认（gen_input5.rs/nst_writer.rs 的常量）。
|
||||
/// 旧 payload 反序列化时缺省为 None（旧节点本就用默认，无回归）。
|
||||
#[serde(default)]
|
||||
pub tlusty_input_params: Option<serde_json::Value>,
|
||||
/// 仅 SYNSPEC-only 场景（TLUSTY 关闭）拉取大气用：显式关联大气网格点名。
|
||||
#[serde(default)]
|
||||
pub atmosphere_ref: Option<String>,
|
||||
}
|
||||
|
||||
impl TaskSpec {
|
||||
/// 旧版兼容归一化：据废弃的 `task_type` 回填 `tlusty_config.strategies` 首项。
|
||||
///
|
||||
/// 修复（审查 #8）：serde default 已把 strategies 填为完整默认链 `[cold_run, seed_step]`，
|
||||
/// 故仅判 `is_empty` 无法覆盖「旧 seed_step 消息被误判为 cold_run」的场景。
|
||||
/// 现据 task_type 把首项校正为对应的单策略链（旧消息的 task_type 是权威来源）：
|
||||
/// - task_type=SeedStep → `[seed_step]`（旧热启动消息不应被当冷启动重跑）；
|
||||
/// - task_type=ColdRun → 保持默认链（cold_run 本就是默认首项）。
|
||||
pub fn normalize_compat(&mut self) {
|
||||
let legacy_first = match self.task_type {
|
||||
TaskType::ColdRun => "cold_run",
|
||||
TaskType::SeedStep => "seed_step",
|
||||
};
|
||||
// 仅当当前 strategies 首项与 task_type 不一致时校正（避免覆盖显式配置）。
|
||||
let needs_fix =
|
||||
self.tlusty_config.strategies.first().map(|s| s.as_str()) != Some(legacy_first);
|
||||
if needs_fix {
|
||||
self.tlusty_config.strategies = vec![legacy_first.to_string()];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 仅供测试夹具构造便利：`TaskSpec::default()` 给出合法占位（task_id 零值、
|
||||
/// 空点/参数、ColdRun、默认阶段配置）。生产代码应显式构造所有字段，避免依赖占位。
|
||||
/// 空点/参数、默认阶段配置）。生产代码应显式构造所有字段，避免依赖占位。
|
||||
impl Default for TaskSpec {
|
||||
fn default() -> Self {
|
||||
TaskSpec {
|
||||
@@ -461,26 +456,20 @@ impl Default for TaskSpec {
|
||||
logn: 0.0.into(),
|
||||
logo: 0.0.into(),
|
||||
},
|
||||
task_type: TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 3600,
|
||||
workflow_name: None,
|
||||
wave: 0,
|
||||
tlusty_config: EngineStageConfig::default_tlusty(),
|
||||
synspec_config: EngineStageConfig::default_synspec(),
|
||||
tlusty_config: PhaseConfig::default_tlusty(),
|
||||
synspec_config: PhaseConfig::default_synspec(),
|
||||
synspec_params: None,
|
||||
tlusty_chain_params: None,
|
||||
tlusty_input_params: None,
|
||||
atmosphere_ref: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TaskType {
|
||||
ColdRun,
|
||||
SeedStep,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TaskStatus {
|
||||
@@ -499,10 +488,20 @@ pub struct TaskReport {
|
||||
#[serde(default)]
|
||||
pub params: Option<GridPointParams>,
|
||||
pub node_id: String,
|
||||
/// 任务整体成败（`derive_report_status`：converged && 无 synspec 错误 → Completed，
|
||||
/// 半失败 = Failed）。整体语义，非阶段成败。
|
||||
pub status: TaskStatus,
|
||||
pub converged: bool,
|
||||
/// **双义**（Phase 7a/7b 标注）：TLUSTY 启用时 = 大气收敛标志（"本次大气产物是否可用"）；
|
||||
/// SYNSPEC-only（TLUSTY 关闭）被重写为管线成功。7b 改名 `result_valid` 消除字段名误读——
|
||||
/// 阶段成败请用 `failed_stage` / `summary_json.synspec_*`。
|
||||
///
|
||||
/// 旧节点仍以字段名 `converged` 上报，serde alias 兼容（支持滚动升级）。
|
||||
#[serde(alias = "converged")]
|
||||
pub result_valid: bool,
|
||||
/// 仅 TLUSTY 大气迭代有效（SYNSPEC-only 任务此量为 None 或大气来源值）。
|
||||
pub max_relc: Option<f64>,
|
||||
pub atmosphere_has_nan: bool,
|
||||
/// 单点总墙钟耗时（秒，含 TLUSTY + SYNSPEC；`synspec_sec` 是其子集）。
|
||||
pub elapsed_sec: f64,
|
||||
pub error_message: Option<String>,
|
||||
pub summary_json: String,
|
||||
@@ -570,11 +569,25 @@ pub struct ConvCheckResult {
|
||||
pub n_depths: usize,
|
||||
pub chmax: f64,
|
||||
pub error: Option<String>,
|
||||
/// 逐次迭代诊断（iter → 该次最大相对变化；Phase 5b 起由 fort.9 全量解析）。
|
||||
/// 完整收敛轨迹：17 次迭代缓降 vs 顶着 NITER 上限勉强的发散轨迹一眼可辨。
|
||||
#[serde(default)]
|
||||
pub itek_history: Vec<IterCheck>,
|
||||
}
|
||||
|
||||
/// 单次迭代的收敛诊断（fort.9 每迭代一拍）。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct IterCheck {
|
||||
pub iter: i32,
|
||||
/// 该次迭代的最大相对变化（所有深度点 |maximum| 的最大值）。
|
||||
pub max_relc: f64,
|
||||
/// 该次迭代参与解析的深度点行数。
|
||||
pub n_depths: usize,
|
||||
}
|
||||
|
||||
/// Convergence stage summary recorded in conv.json
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct StageSummary {
|
||||
pub struct StepSummary {
|
||||
pub label: String,
|
||||
pub chmax: Option<f64>,
|
||||
pub lte: String,
|
||||
@@ -592,6 +605,10 @@ pub struct StageSummary {
|
||||
/// 深度点总数，诊断用。
|
||||
#[serde(default)]
|
||||
pub n_depths: Option<usize>,
|
||||
/// 逐次迭代诊断（iter → 该次最大相对变化）。**全量保真**：summary_json 与 conv.json
|
||||
/// 同源于 runner 的 StepSummary，故完整收敛轨迹随 summary_json 落库，不再仅存磁盘。
|
||||
#[serde(default)]
|
||||
pub itek_history: Vec<IterCheck>,
|
||||
}
|
||||
|
||||
/// Full execution summary for a grid point
|
||||
@@ -599,8 +616,15 @@ pub struct StageSummary {
|
||||
pub struct ModelSummary {
|
||||
pub name: String,
|
||||
pub params: GridPointParams,
|
||||
pub stages: Vec<StageSummary>,
|
||||
pub converged: bool,
|
||||
/// TLUSTY 收敛链子步骤摘要（lte/nc/nl 或 seed_nc/nl），**不是** TLUSTY/SYNSPEC 管线大阶段。
|
||||
pub stages: Vec<StepSummary>,
|
||||
/// 本次结果是否可用（P9 拆分，与 `TaskReport.result_valid` 对齐）：
|
||||
/// TLUSTY 启用时 = 大气收敛；SYNSPEC-only（TLUSTY 关闭）被 reporter 重写为管线成功。
|
||||
/// 读方不能仅凭字段名判断是哪个阶段——整体成败请用 `TaskReport.status`。
|
||||
/// 旧 `conv.json`/`summary_json` 序列化的键名是 `converged`，`#[serde(alias)]` 兼容读取。
|
||||
#[serde(alias = "converged")]
|
||||
pub result_valid: bool,
|
||||
/// 仅 TLUSTY 大气迭代有效（最大相对修正；SYNSPEC 阶段无此量）。
|
||||
pub final_max_relc: Option<f64>,
|
||||
pub final_chmax: Option<f64>,
|
||||
pub seed: Option<String>,
|
||||
@@ -608,8 +632,9 @@ pub struct ModelSummary {
|
||||
pub synspec_rc: Option<i32>,
|
||||
pub synspec_error: Option<String>,
|
||||
pub synspec_sec: Option<f64>,
|
||||
/// 单点总墙钟耗时（秒）。极旧版 conv.json 可能缺此字段，default 0.0 兜底
|
||||
/// （展示层把 ≤0 视为"无数据"）；现版 run_one.py 总是写入。
|
||||
/// 单点总墙钟耗时（秒，含 TLUSTY + SYNSPEC）。**包含** `synspec_sec`（子集）：
|
||||
/// `elapsed_sec ≥ synspec_sec` 恒成立（synspec 为空时 synspec_sec=None）。
|
||||
/// 极旧版 conv.json 可能缺此字段，default 0.0 兜底（展示层把 ≤0 视为"无数据"）。
|
||||
#[serde(default)]
|
||||
pub elapsed_sec: f64,
|
||||
pub note: Option<String>,
|
||||
@@ -619,7 +644,7 @@ pub struct ModelSummary {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// StageSummary 新增迭代诊断字段的向后兼容：
|
||||
/// StepSummary 新增迭代诊断字段的向后兼容：
|
||||
/// 旧 conv.json（无 last_iter/worst_depth/n_depths）必须能正常反序列化为 null，
|
||||
/// 新写入的 conv.json 往返保真。
|
||||
#[test]
|
||||
@@ -628,12 +653,12 @@ mod tests {
|
||||
"label": "nl", "chmax": 0.001, "lte": "F", "converged": true,
|
||||
"best_max_relc": 0.0005, "elapsed_sec": 64.0, "note": null
|
||||
}"#;
|
||||
let st: StageSummary = serde_json::from_str(legacy).unwrap();
|
||||
let st: StepSummary = serde_json::from_str(legacy).unwrap();
|
||||
assert_eq!(st.last_iter, None);
|
||||
assert_eq!(st.worst_depth, None);
|
||||
assert_eq!(st.n_depths, None);
|
||||
|
||||
let full = StageSummary {
|
||||
let full = StepSummary {
|
||||
label: "nl".to_string(),
|
||||
chmax: Some(0.001),
|
||||
lte: "F".to_string(),
|
||||
@@ -644,8 +669,13 @@ mod tests {
|
||||
last_iter: Some(17),
|
||||
worst_depth: Some(1),
|
||||
n_depths: Some(50),
|
||||
itek_history: vec![IterCheck {
|
||||
iter: 1,
|
||||
max_relc: 0.5,
|
||||
n_depths: 50,
|
||||
}],
|
||||
};
|
||||
let round: StageSummary =
|
||||
let round: StepSummary =
|
||||
serde_json::from_str(&serde_json::to_string(&full).unwrap()).unwrap();
|
||||
assert_eq!(round.last_iter, Some(17));
|
||||
assert_eq!(round.worst_depth, Some(1));
|
||||
@@ -657,7 +687,7 @@ mod tests {
|
||||
///
|
||||
/// 载荷严格复刻 run_one.py 的真实输出形态：stage 含 `itek_attempts`/`final` 嵌套
|
||||
/// dict、`note`、可选 `best_max_relc`，顶层含 `final_chmax`/`synspec_*`/`seed` 等。
|
||||
/// Rust StageSummary 未声明的字段（itek_attempts/final）应被 serde 静默忽略。
|
||||
/// Rust StepSummary 未声明的字段（itek_attempts/final）应被 serde 静默忽略。
|
||||
#[test]
|
||||
fn test_model_summary_parses_python_legacy_conv_json() {
|
||||
let legacy = r#"{
|
||||
@@ -699,7 +729,8 @@ mod tests {
|
||||
let s: ModelSummary = serde_json::from_str(legacy).expect("旧版 conv.json 应可解析");
|
||||
assert_eq!(s.name, "t20000_g5.0_he-2_c-4_n-4_o-4");
|
||||
assert_eq!(*s.params.teff, 20000.0);
|
||||
assert!(s.converged);
|
||||
// 旧版键名 "converged" 经 #[serde(alias)] 兼容读入 result_valid。
|
||||
assert!(s.result_valid);
|
||||
assert!(!s.atmosphere_has_nan);
|
||||
assert_eq!(s.final_max_relc, Some(0.0069));
|
||||
assert_eq!(s.elapsed_sec, 715.0);
|
||||
@@ -852,75 +883,80 @@ mod tests {
|
||||
assert_eq!(GridPointStatus::Pending.to_string(), "pending");
|
||||
assert_eq!(GridPointStatus::Queued.to_string(), "queued");
|
||||
assert_eq!(GridPointStatus::Running.to_string(), "running");
|
||||
assert_eq!(GridPointStatus::Converged.to_string(), "converged");
|
||||
assert_eq!(GridPointStatus::Completed.to_string(), "completed");
|
||||
assert_eq!(GridPointStatus::Failed.to_string(), "failed");
|
||||
|
||||
assert_eq!(GridPointStatus::from("queued"), GridPointStatus::Queued);
|
||||
// 7c：'completed' 权威值；'converged'/'done' 为 legacy 别名。
|
||||
assert_eq!(
|
||||
GridPointStatus::from("completed"),
|
||||
GridPointStatus::Completed
|
||||
);
|
||||
assert_eq!(
|
||||
GridPointStatus::from("converged"),
|
||||
GridPointStatus::Converged
|
||||
GridPointStatus::Completed
|
||||
);
|
||||
assert_eq!(GridPointStatus::from("done"), GridPointStatus::Converged);
|
||||
assert_eq!(GridPointStatus::from("done"), GridPointStatus::Completed);
|
||||
assert_eq!(GridPointStatus::from("failed"), GridPointStatus::Failed);
|
||||
assert_eq!(GridPointStatus::from("unknown"), GridPointStatus::Pending);
|
||||
}
|
||||
|
||||
/// `StagePolicy` 的 snake_case serde 往返 + DB 文本兜底。
|
||||
/// `ResumePolicy` 的 snake_case serde 往返 + DB 文本兜底。
|
||||
#[test]
|
||||
fn test_stage_policy_serde_roundtrip() {
|
||||
for p in [
|
||||
StagePolicy::SkipConverged,
|
||||
StagePolicy::ForceRecompute,
|
||||
StagePolicy::SkipFailed,
|
||||
ResumePolicy::SkipConverged,
|
||||
ResumePolicy::ForceRecompute,
|
||||
ResumePolicy::SkipFailed,
|
||||
] {
|
||||
let s = serde_json::to_string(&p).unwrap();
|
||||
let back: StagePolicy = serde_json::from_str(&s).unwrap();
|
||||
let back: ResumePolicy = serde_json::from_str(&s).unwrap();
|
||||
assert_eq!(p, back);
|
||||
}
|
||||
// snake_case 形态锁定（前端 payload 与 DB 列口径）
|
||||
assert_eq!(
|
||||
serde_json::to_string(&StagePolicy::SkipConverged).unwrap(),
|
||||
serde_json::to_string(&ResumePolicy::SkipConverged).unwrap(),
|
||||
"\"skip_converged\""
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&StagePolicy::ForceRecompute).unwrap(),
|
||||
serde_json::to_string(&ResumePolicy::ForceRecompute).unwrap(),
|
||||
"\"force_recompute\""
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&StagePolicy::SkipFailed).unwrap(),
|
||||
serde_json::to_string(&ResumePolicy::SkipFailed).unwrap(),
|
||||
"\"skip_failed\""
|
||||
);
|
||||
// as_str/from_str_lossy 互逆（非法值兜底 SkipConverged）
|
||||
assert_eq!(
|
||||
StagePolicy::from_str_lossy("skip_converged"),
|
||||
StagePolicy::SkipConverged
|
||||
ResumePolicy::from_str_lossy("skip_converged"),
|
||||
ResumePolicy::SkipConverged
|
||||
);
|
||||
assert_eq!(
|
||||
StagePolicy::from_str_lossy("garbage"),
|
||||
StagePolicy::SkipConverged
|
||||
ResumePolicy::from_str_lossy("garbage"),
|
||||
ResumePolicy::SkipConverged
|
||||
);
|
||||
assert_eq!(
|
||||
StagePolicy::from_str_lossy(StagePolicy::ForceRecompute.as_str()),
|
||||
StagePolicy::ForceRecompute
|
||||
ResumePolicy::from_str_lossy(ResumePolicy::ForceRecompute.as_str()),
|
||||
ResumePolicy::ForceRecompute
|
||||
);
|
||||
}
|
||||
|
||||
/// `EngineStageConfig` serde 往返 + 默认值（缺字段时 serde default 兜底）。
|
||||
/// `PhaseConfig` serde 往返 + 默认值（缺字段时 serde default 兜底）。
|
||||
#[test]
|
||||
fn test_engine_stage_config_serde_and_defaults() {
|
||||
let cfg = EngineStageConfig {
|
||||
let cfg = PhaseConfig {
|
||||
enabled: false,
|
||||
policy: StagePolicy::ForceRecompute,
|
||||
policy: ResumePolicy::ForceRecompute,
|
||||
strategies: vec!["cold_run".to_string(), "seed_step".to_string()],
|
||||
};
|
||||
let json = serde_json::to_string(&cfg).unwrap();
|
||||
let back: EngineStageConfig = serde_json::from_str(&json).unwrap();
|
||||
let back: PhaseConfig = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(cfg, back);
|
||||
|
||||
// 空 payload 应产出默认值（enabled=true, policy=skip_converged, strategies=[]）
|
||||
let empty: EngineStageConfig = serde_json::from_str("{}").unwrap();
|
||||
let empty: PhaseConfig = serde_json::from_str("{}").unwrap();
|
||||
assert!(empty.enabled);
|
||||
assert_eq!(empty.policy, StagePolicy::SkipConverged);
|
||||
assert_eq!(empty.policy, ResumePolicy::SkipConverged);
|
||||
assert!(empty.strategies.is_empty());
|
||||
|
||||
// current_strategy 空链兜底
|
||||
@@ -930,43 +966,31 @@ mod tests {
|
||||
assert!(!cfg.has_strategy("standard"));
|
||||
}
|
||||
|
||||
/// 旧版 MQ 在途消息（仅含 task_type，无 tlusty_config）经 `#[serde(default)]`
|
||||
/// 反序列化后，`normalize_compat()` 应据 task_type 回填 strategies。
|
||||
/// Phase 6（P8）删除 task_type 后，`normalize_compat` 与旧版单策略链校正机制整体移除。
|
||||
/// 升级前需确认队列为空（docs/database_refactor_design.md §8.8）：残留旧 payload 的
|
||||
/// strategies 会按 serde default 填成默认链 [cold_run, seed_step]，无 task_type 可校正。
|
||||
#[test]
|
||||
fn test_task_spec_normalize_compat_from_legacy_task_type() {
|
||||
fn test_legacy_payload_without_task_type_uses_default_strategy_chain() {
|
||||
let legacy_json = r#"{
|
||||
"task_id": "00000000-0000-0000-0000-000000000001",
|
||||
"point_name": "t20000_g5.0_he-2_c-4_n-4_o-4",
|
||||
"params": {"teff": 20000.0, "logg": 5.0, "loghe": -2.0, "logc": -4.0, "logn": -4.0, "logo": -4.0},
|
||||
"task_type": "seed_step",
|
||||
"seed_point_name": "neighbor",
|
||||
"timeout_sec": 7200,
|
||||
"workflow_name": "wf_a",
|
||||
"wave": 0
|
||||
}"#;
|
||||
let mut spec: TaskSpec = serde_json::from_str(legacy_json).unwrap();
|
||||
// 修复后 normalize_compat 据 task_type 校正首项：旧 seed_step 消息的 strategies
|
||||
// 首项应被校正为 seed_step（而非保留默认链的 cold_run 首项，否则会被误当冷启动）。
|
||||
spec.normalize_compat();
|
||||
// task_type 字段已被移除；旧 payload 即使仍携带该键也会被 serde 忽略（未知字段）。
|
||||
// tlusty_config 缺省回落到默认链，首项 cold_run。
|
||||
let spec: TaskSpec = serde_json::from_str(legacy_json).unwrap();
|
||||
assert_eq!(
|
||||
spec.tlusty_config.current_strategy("cold_run"),
|
||||
"cold_run",
|
||||
"无显式 strategies 的旧 payload 回落到默认链首项"
|
||||
);
|
||||
assert_eq!(
|
||||
spec.tlusty_config.strategies,
|
||||
vec!["seed_step".to_string()],
|
||||
"旧 seed_step 消息应校正为 [seed_step] 单策略链"
|
||||
);
|
||||
assert_eq!(spec.tlusty_config.current_strategy("cold_run"), "seed_step");
|
||||
|
||||
// 对照：旧 cold_run 消息 → 首项已是 cold_run（默认链首项），无需校正。
|
||||
let mut cold_spec = TaskSpec::default();
|
||||
cold_spec.task_type = TaskType::ColdRun;
|
||||
cold_spec.normalize_compat();
|
||||
assert_eq!(
|
||||
cold_spec
|
||||
.tlusty_config
|
||||
.strategies
|
||||
.first()
|
||||
.map(|s| s.as_str()),
|
||||
Some("cold_run"),
|
||||
"旧 cold_run 消息保持默认链"
|
||||
vec!["cold_run".to_string(), "seed_step".to_string()]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,17 @@
|
||||
use crate::config::StageConfig;
|
||||
use crate::config::{ChainStep, TlustyInput};
|
||||
|
||||
pub fn generate_nst_content(stage: &StageConfig) -> String {
|
||||
/// 生成 TLUSTY 的 nst（非标准标志）文件内容。
|
||||
///
|
||||
/// nst 文件由 `KEY=VALUE` 对组成，TLUSTY 的 NSTPAR 子程序逐行解析（无行数限制）。
|
||||
/// 内容来源（优先级从高到低）：
|
||||
/// 1. `input_cfg.nst_extra_keys`：用户自由传入的任意 KEY=VALUE（逃逸口），追加到末尾。
|
||||
/// 2. `stage`（ChainStep）：CHMAX/ITEK/NITER/ORELAX/IDLTE/IACC/ICHANG 等阶段差异参数。
|
||||
/// 3. 硬编码默认：ND/NLAMBD/VTB/ISPODF/DDNU/CNU1/IELCOR（input_cfg 无对应字段时）。
|
||||
///
|
||||
/// `input_cfg` 为 None 时走全默认（与改动前行为完全一致，向后兼容）。
|
||||
pub fn generate_nst_content(stage: &ChainStep, input_cfg: Option<&TlustyInput>) -> String {
|
||||
// 第 1 行：深度点数/角度数/湍速/ODF 等物理网格参数 + 收敛控制。
|
||||
// ND/NLAMBD/VTB/ISPODF/DDNU/CNU1 当前无结构化字段，保留硬编码（如需覆写用 extra_keys）。
|
||||
let mut line1_parts = vec![
|
||||
"ND=50".to_string(),
|
||||
"NLAMBD=3".to_string(),
|
||||
@@ -18,6 +29,7 @@ pub fn generate_nst_content(stage: &StageConfig) -> String {
|
||||
}
|
||||
line1_parts.push(format!("NITER={}", stage.niter));
|
||||
|
||||
// 第 2 行：加速/收敛控制开关。
|
||||
let mut line2_parts = Vec::new();
|
||||
if let Some(orelax) = stage.orelax {
|
||||
line2_parts.push(format!("ORELAX={}", orelax));
|
||||
@@ -33,7 +45,38 @@ pub fn generate_nst_content(stage: &StageConfig) -> String {
|
||||
}
|
||||
line2_parts.push("IELCOR=-1".to_string());
|
||||
|
||||
format!("{}\n{}\n", line1_parts.join(","), line2_parts.join(","))
|
||||
let mut out = format!("{}\n{}\n", line1_parts.join(","), line2_parts.join(","));
|
||||
|
||||
// 第 3 行起：用户自由传入的额外 nst 关键字（逃逸口）。
|
||||
// 每行一个 KEY=VALUE，追加到末尾。用于暴露未结构化的 220+ nst 关键字
|
||||
// （如 FRCMAX/CUTBAL/TAU/NDGREY 等）。
|
||||
// 多行格式经 TLUSTY 源码验证（tlusty208.f:1819 NSTPAR 用 `READ(INPFI,500,END=70)`
|
||||
// + `GO TO 10` 逐行循环读至 EOF，每行用 GETWRD 解析 KEY=VALUE），追加行可被正确解析。
|
||||
// 安全校验：key/value 含逗号/换行/等号会破坏 nst 的逗号分隔或 KEY=VALUE 解析，
|
||||
// 跳过非法项并记 warn（避免生成损坏的 nst 导致 tlusty 行为异常）。
|
||||
// L1 修复：**key 与 value 都校验**——key 含 `=`/`,`/空白/换行会产出畸形 `KEY=VALUE`
|
||||
// 行，TLUSTY 的 GETWRD 解析器可能误读。key 必须是合法标识符（非空、无上述分隔符）。
|
||||
if let Some(cfg) = input_cfg {
|
||||
for (key, value) in &cfg.nst_extra_keys {
|
||||
let bad_key = key.is_empty()
|
||||
|| key.contains(',')
|
||||
|| key.contains('\n')
|
||||
|| key.contains('=')
|
||||
|| key.chars().any(char::is_whitespace);
|
||||
let bad_value = value.contains(',') || value.contains('\n') || value.contains('=');
|
||||
if bad_key || bad_value {
|
||||
tracing::warn!(
|
||||
"跳过非法 nst_extra_keys 项 {:?}={:?}: 含逗号/换行/等号/空白会破坏 nst 解析",
|
||||
key,
|
||||
value
|
||||
);
|
||||
continue;
|
||||
}
|
||||
out.push_str(&format!("{}={}\n", key, value));
|
||||
}
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -42,7 +85,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_nst_generation() {
|
||||
let stage = StageConfig {
|
||||
let stage = ChainStep {
|
||||
label: "nc".to_string(),
|
||||
lte: "F".to_string(),
|
||||
ltgray: "F".to_string(),
|
||||
@@ -57,9 +100,75 @@ mod tests {
|
||||
iacc: None,
|
||||
orelax: None,
|
||||
};
|
||||
let content = generate_nst_content(&stage);
|
||||
let content = generate_nst_content(&stage, None);
|
||||
assert!(content.contains("ND=50"));
|
||||
assert!(content.contains("NITER=10"));
|
||||
assert!(content.contains("IELCOR=-1"));
|
||||
}
|
||||
|
||||
/// extra_keys 追加到 nst 末尾（每行一个 KEY=VALUE）。
|
||||
#[test]
|
||||
fn test_nst_extra_keys() {
|
||||
let stage = ChainStep {
|
||||
label: "nl".to_string(),
|
||||
lte: "F".to_string(),
|
||||
ltgray: "F".to_string(),
|
||||
ilvlin: 100,
|
||||
require_converged: true,
|
||||
niter: 100,
|
||||
chmax: Some(0.001),
|
||||
itek: None,
|
||||
metals: None,
|
||||
ichang: None,
|
||||
idlte: None,
|
||||
iacc: None,
|
||||
orelax: None,
|
||||
};
|
||||
let cfg = TlustyInput {
|
||||
nfread: 2000,
|
||||
atoms: Default::default(),
|
||||
ions: vec![],
|
||||
nst_extra_keys: vec![
|
||||
("FRCMAX".to_string(), "0.01".to_string()),
|
||||
("CUTBAL".to_string(), "0.3".to_string()),
|
||||
],
|
||||
};
|
||||
let content = generate_nst_content(&stage, Some(&cfg));
|
||||
assert!(content.contains("FRCMAX=0.01"), "extra_keys 应追加到 nst");
|
||||
assert!(content.contains("CUTBAL=0.3"));
|
||||
// 原有内容仍存在
|
||||
assert!(content.contains("NITER=100"));
|
||||
assert!(content.contains("CHMAX=0.001"));
|
||||
}
|
||||
|
||||
/// None 配置时与改动前行为一致（无 extra_keys 行）。
|
||||
#[test]
|
||||
fn test_nst_none_cfg_backward_compat() {
|
||||
let stage = ChainStep {
|
||||
label: "lte".to_string(),
|
||||
lte: "T".to_string(),
|
||||
ltgray: "T".to_string(),
|
||||
ilvlin: 0,
|
||||
require_converged: false,
|
||||
niter: 0,
|
||||
chmax: None,
|
||||
itek: None,
|
||||
metals: None,
|
||||
ichang: None,
|
||||
idlte: None,
|
||||
iacc: None,
|
||||
orelax: None,
|
||||
};
|
||||
let content = generate_nst_content(&stage, None);
|
||||
// None 配置时只有 2 行（第1行 ND/NITER 等 + 第2行 IELCOR 等），无 extra_keys 追加行。
|
||||
let non_empty_lines: Vec<&str> = content.lines().filter(|l| !l.is_empty()).collect();
|
||||
assert_eq!(
|
||||
non_empty_lines.len(),
|
||||
2,
|
||||
"None 配置时 nst 应只有 2 行，实际 {} 行: {:?}",
|
||||
non_empty_lines.len(),
|
||||
non_empty_lines
|
||||
);
|
||||
assert!(!content.contains("FRCMAX"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,8 @@
|
||||
//! `conv.json`、`fort.8`（synspec 输入大气）、`fort.55`（synspec 控制卡）
|
||||
//! 2. **科学核心**：`<name>.7`、`<name>.spec`、`<name>.cont`、`<name>.iden`、`<name>.log`、
|
||||
//! `<name>.bfac`（TLUSTY 最终 b 因子/非 LTE 偏离因子）、`<name>.emflux`（TLUSTY 最终出射谱 λ–Fλ）
|
||||
//! 3. **阶段快照**：`<name>.<label>.5/.6/.err/.nst/.7`（label ∈ lte/nc/nl/seed_nc）
|
||||
//! 3. **阶段快照**：`<name>.<label>.5/.6/.err/.nst/.7`（label 为任意单一标识符，
|
||||
//! 含默认链 lte/nc/nl/seed_nc 与用户 `tlusty_chain` 自定义标签——M3 起不再硬编码白名单）
|
||||
//! 4. **收敛诊断**：`<name>.<label>_chmax*.9`（**唯一保留的 .9**；裸 `<name>.<label>.9`
|
||||
//! 已在 runner 源头停止写出，因其与 `_chmax*.9` 内容完全重复）
|
||||
//!
|
||||
@@ -30,15 +31,6 @@ const SCIENCE_SUFFIXES: &[&str] = &["7", "spec", "cont", "iden", "log", "bfac",
|
||||
/// 阶段快照的文件名后缀（挂在 `<name>.<label>.` 之后）。
|
||||
const STAGE_SNAPSHOT_SUFFIXES: &[&str] = &["5", "6", "err", "nst", "7"];
|
||||
|
||||
/// 合法阶段标签（来自 `default_cold_chain` / `default_seed_chain` 的 label）。
|
||||
/// 阶段标签由 workflow 配置保证唯一，不会与科学后缀或 synspec 产物冲突。
|
||||
///
|
||||
/// **约束**：此处硬编码了默认链的 4 个标签。runner 的 `run_model_with_timeout`
|
||||
/// 虽接受 `custom_chain`（可含任意 label），但当前唯一生产调用方（executor）传 `None`
|
||||
/// 走默认链，故白名单覆盖安全。若将来启用自定义 chain 且引入新标签，需同步加入此处，
|
||||
/// 否则带新标签的阶段快照（`.5/.6/.err/.nst/.7`）和 `_chmax*.9` 会被白名单静默丢弃。
|
||||
const STAGE_LABELS: &[&str] = &["lte", "nc", "nl", "seed_nc"];
|
||||
|
||||
/// 有独立语义、保留的裸文件名（不以 model_name 为前缀）。
|
||||
const BARE_KEEPS: &[&str] = &["conv.json", "fort.8", "fort.55"];
|
||||
|
||||
@@ -86,21 +78,25 @@ pub fn is_result_worthy(fname: &str, model_name: &str) -> bool {
|
||||
}
|
||||
|
||||
// 3. 阶段快照：`<label>.<suffix>`（如 `nl.7`、`nc.nst`）。
|
||||
// 用 split_once('.', label/suffix) 切一刀；label 必须在 STAGE_LABELS 内，
|
||||
// suffix 必须在 STAGE_SNAPSHOT_SUFFIXES 内。这样能精确排除 `<name>.nl.9`
|
||||
// （suffix=9 不在快照后缀集）等。
|
||||
// M3 修复：不再硬编码 label 白名单——runner 的 `custom_chain` 允许用户在 YAML
|
||||
// `tlusty_chain` 配置任意阶段标签，硬编码白名单（lte/nc/nl/seed_nc）会把自定义
|
||||
// 标签的阶段快照静默丢弃。改按**结构**识别：`<label>.<suffix>`，label 为不含
|
||||
// '.' 的单一标识符，suffix 限定在快照后缀集内。这样自定义标签（如 `grey`/`base`）
|
||||
// 与默认标签同等归档；代价是 `<name>.<任意>.7` 这类罕见杂散文件也会被保留
|
||||
// （低风险，宁可多留一份也不丢科学产物）。
|
||||
if let Some((label, suffix)) = rest.split_once('.') {
|
||||
if STAGE_LABELS.contains(&label) && STAGE_SNAPSHOT_SUFFIXES.contains(&suffix) {
|
||||
if !label.is_empty() && !label.contains('.') && STAGE_SNAPSHOT_SUFFIXES.contains(&suffix) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// 4. 收敛诊断：`<label>_chmax*.9`（如 `nl_chmax0.001.9`）。
|
||||
// rest 以 `<label>_chmax` 开头，以 `.9` 结尾。这是唯一保留的 .9 形态。
|
||||
if rest.ends_with(".9") {
|
||||
for label in STAGE_LABELS {
|
||||
let tag = format!("{}_chmax", label);
|
||||
if rest.starts_with(&tag) && rest.ends_with(".9") {
|
||||
// 4. 收敛诊断：`<label>_chmax<value>.9`（如 `nl_chmax0.001.9`）。唯一保留的 .9 形态。
|
||||
// 同样按结构识别：以 `.9` 结尾 + 去掉 `.9` 后含 `_chmax` + 标签为非空单 token。
|
||||
// 精确排除冗余的 `<name>.<label>.9`（无 `_chmax`，与此完全重复）。
|
||||
if let Some(stem) = rest.strip_suffix(".9") {
|
||||
if let Some(pos) = stem.rfind("_chmax") {
|
||||
let label = &stem[..pos];
|
||||
if !label.is_empty() && !label.contains('.') {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -114,6 +110,8 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
const NAME: &str = "t20000_g5.0_he-2_c-4_n-4_o-4";
|
||||
// 默认链标签（测试沿用历史白名单；生产逻辑已改为结构识别，见 is_result_worthy）。
|
||||
const STAGE_LABELS: &[&str] = &["lte", "nc", "nl", "seed_nc"];
|
||||
|
||||
#[test]
|
||||
fn test_bare_keeps() {
|
||||
@@ -201,10 +199,29 @@ mod tests {
|
||||
assert!(!is_result_worthy(&format!("{}.nl.foo", NAME), NAME));
|
||||
}
|
||||
|
||||
/// M3 回归：用户自定义阶段标签（非默认 lte/nc/nl/seed_nc）的阶段快照必须归档。
|
||||
/// 硬编码 STAGE_LABELS 白名单会把自定义标签的 `.5/.6/.err/.nst/.7` 与 `_chmax*.9`
|
||||
/// 静默丢弃；现改为结构识别，任意单一标识符标签均保留。
|
||||
#[test]
|
||||
fn test_unknown_stage_label_skipped() {
|
||||
// 未知的阶段标签不归档（防御性：未来若引入新标签需显式加入 STAGE_LABELS）
|
||||
assert!(!is_result_worthy(&format!("{}.xxx.7", NAME), NAME));
|
||||
assert!(!is_result_worthy(&format!("{}.xxx.nst", NAME), NAME));
|
||||
fn test_custom_stage_label_kept() {
|
||||
for label in ["grey", "base", "myscenario"] {
|
||||
for s in ["5", "6", "err", "nst", "7"] {
|
||||
let f = format!("{}.{}.{}", NAME, label, s);
|
||||
assert!(
|
||||
is_result_worthy(&f, NAME),
|
||||
"自定义标签 {} 的阶段快照 {} 应归档",
|
||||
label,
|
||||
f
|
||||
);
|
||||
}
|
||||
let chmax = format!("{}.{}_chmax0.001.9", NAME, label);
|
||||
assert!(
|
||||
is_result_worthy(&chmax, NAME),
|
||||
"自定义标签 {} 的 chmax 诊断应归档",
|
||||
label
|
||||
);
|
||||
}
|
||||
// 冗余的 `<name>.<label>.9`（无 _chmax）仍不应归档（与 _chmax.9 重复）。
|
||||
assert!(!is_result_worthy(&format!("{}.grey.9", NAME), NAME));
|
||||
}
|
||||
}
|
||||
|
||||
+142
-38
@@ -1,9 +1,9 @@
|
||||
use crate::config::{StageConfig, SynspecConfig};
|
||||
use crate::conv_check::{atmosphere_has_nan, check_fort9};
|
||||
use crate::config::{ChainStep, SynspecInput, TlustyInput};
|
||||
use crate::conv_check::{atmosphere_has_nan, check_fort9, extract_failure_hint, spec_is_valid};
|
||||
use crate::embedded::RuntimePaths;
|
||||
use crate::fort55_writer::generate_fort55_content;
|
||||
use crate::gen_input5::make_input5;
|
||||
use crate::models::{GridPointParams, ModelSummary, StageSummary, TaskType};
|
||||
use crate::models::{GridPointParams, ModelSummary, StepSummary};
|
||||
use crate::nst_writer::generate_nst_content;
|
||||
use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -13,9 +13,9 @@ use tokio::fs::File;
|
||||
use tokio::process::Command as AsyncCommand;
|
||||
use tracing::{info, warn};
|
||||
|
||||
pub fn default_cold_chain() -> Vec<StageConfig> {
|
||||
pub fn default_cold_chain() -> Vec<ChainStep> {
|
||||
vec![
|
||||
StageConfig {
|
||||
ChainStep {
|
||||
label: "lte".to_string(),
|
||||
lte: "T".to_string(),
|
||||
ltgray: "T".to_string(),
|
||||
@@ -30,7 +30,7 @@ pub fn default_cold_chain() -> Vec<StageConfig> {
|
||||
iacc: None,
|
||||
orelax: None,
|
||||
},
|
||||
StageConfig {
|
||||
ChainStep {
|
||||
label: "nc".to_string(),
|
||||
lte: "F".to_string(),
|
||||
ltgray: "F".to_string(),
|
||||
@@ -45,7 +45,7 @@ pub fn default_cold_chain() -> Vec<StageConfig> {
|
||||
iacc: None,
|
||||
orelax: None,
|
||||
},
|
||||
StageConfig {
|
||||
ChainStep {
|
||||
label: "nl".to_string(),
|
||||
lte: "F".to_string(),
|
||||
ltgray: "F".to_string(),
|
||||
@@ -63,9 +63,9 @@ pub fn default_cold_chain() -> Vec<StageConfig> {
|
||||
]
|
||||
}
|
||||
|
||||
pub fn default_seed_chain() -> Vec<StageConfig> {
|
||||
pub fn default_seed_chain() -> Vec<ChainStep> {
|
||||
vec![
|
||||
StageConfig {
|
||||
ChainStep {
|
||||
label: "seed_nc".to_string(),
|
||||
lte: "F".to_string(),
|
||||
ltgray: "F".to_string(),
|
||||
@@ -80,7 +80,7 @@ pub fn default_seed_chain() -> Vec<StageConfig> {
|
||||
iacc: None,
|
||||
orelax: None,
|
||||
},
|
||||
StageConfig {
|
||||
ChainStep {
|
||||
label: "nl".to_string(),
|
||||
lte: "F".to_string(),
|
||||
ltgray: "F".to_string(),
|
||||
@@ -98,6 +98,19 @@ pub fn default_seed_chain() -> Vec<StageConfig> {
|
||||
]
|
||||
}
|
||||
|
||||
/// 按当前策略选默认执行链（`custom_chain` 为 None/空时的兜底）。
|
||||
///
|
||||
/// Phase 6（P8）起取代废弃的 task_type 匹配：`"seed_step"` → 种子热启动链，其余策略
|
||||
/// （`cold_run` 等）→ 冷启动链。executor 现优先使用 TaskSpec.tlusty_chain_params
|
||||
/// （用户 YAML `tlusty_chain:` 配置），None/空才回退本函数的默认链。
|
||||
pub fn default_chain_for_strategy(current_strategy: &str) -> Vec<ChainStep> {
|
||||
if current_strategy == "seed_step" {
|
||||
default_seed_chain()
|
||||
} else {
|
||||
default_cold_chain()
|
||||
}
|
||||
}
|
||||
|
||||
/// 运行子进程，带超时与优雅退出（shutdown）感知。
|
||||
///
|
||||
/// 三种终止路径：
|
||||
@@ -199,19 +212,21 @@ impl<'a> ExecutionRunner<'a> {
|
||||
Self { runtime, work_dir }
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)] // 透传全参给 run_model_with_timeout（后者同 allow）
|
||||
pub async fn run_model(
|
||||
&self,
|
||||
params: &GridPointParams,
|
||||
name: &str,
|
||||
task_type: TaskType,
|
||||
custom_chain: Option<Vec<StageConfig>>,
|
||||
current_strategy: &str,
|
||||
custom_chain: Option<Vec<ChainStep>>,
|
||||
seed_atmos: Option<&Path>,
|
||||
synspec_cfg: Option<&SynspecConfig>,
|
||||
synspec_cfg: Option<&SynspecInput>,
|
||||
tlusty_input: Option<&TlustyInput>,
|
||||
) -> Result<ModelSummary> {
|
||||
self.run_model_with_timeout(
|
||||
params,
|
||||
name,
|
||||
task_type,
|
||||
current_strategy,
|
||||
custom_chain,
|
||||
seed_atmos,
|
||||
synspec_cfg,
|
||||
@@ -219,6 +234,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
true,
|
||||
7200,
|
||||
None,
|
||||
tlusty_input,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -228,19 +244,24 @@ impl<'a> ExecutionRunner<'a> {
|
||||
/// `tlusty_enabled` / `synspec_enabled` 控制各阶段是否运行：
|
||||
/// - TLUSTY 关闭：跳过 chain 循环，直接以 seed_atmos（或单独拉取的大气）作 final_7；
|
||||
/// - SYNSPEC 关闭：跳过光谱合成块（即便 final_7 存在）。
|
||||
///
|
||||
/// Phase 6（P8）：`current_strategy` 取代废弃的 `task_type`——执行链由
|
||||
/// `custom_chain`（executor 按 `tlusty_config.strategies[0]` 显式推导）决定；
|
||||
/// 该参数仅用于日志与 custom_chain=None 时的兜底（"seed_step"→种子链，否则冷启动链）。
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn run_model_with_timeout(
|
||||
&self,
|
||||
params: &GridPointParams,
|
||||
name: &str,
|
||||
task_type: TaskType,
|
||||
custom_chain: Option<Vec<StageConfig>>,
|
||||
current_strategy: &str,
|
||||
custom_chain: Option<Vec<ChainStep>>,
|
||||
seed_atmos: Option<&Path>,
|
||||
synspec_cfg: Option<&SynspecConfig>,
|
||||
synspec_cfg: Option<&SynspecInput>,
|
||||
tlusty_enabled: bool,
|
||||
synspec_enabled: bool,
|
||||
timeout_sec: u64,
|
||||
shutdown: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
|
||||
tlusty_input: Option<&TlustyInput>,
|
||||
) -> Result<ModelSummary> {
|
||||
// `name` 取自权威的 TaskSpec.point_name（DB 的 grid_points.name 列，源精度正确），
|
||||
// 而非 params.model_name()。原因：服务端把 GridPointParams 存成 6 个 REAL 数值列，
|
||||
@@ -255,7 +276,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
let model_dir = self.work_dir.join(name);
|
||||
tokio::fs::create_dir_all(&model_dir).await?;
|
||||
|
||||
info!("开始物理计算网格模型 {} (类型: {:?})", name, task_type);
|
||||
info!("开始物理计算网格模型 {} (策略: {})", name, current_strategy);
|
||||
let t0 = Instant::now();
|
||||
|
||||
// 1. Data directory symlink setup
|
||||
@@ -296,10 +317,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
let _ = tokio::fs::remove_file(&fort84).await;
|
||||
}
|
||||
|
||||
let chain = custom_chain.unwrap_or_else(|| match task_type {
|
||||
TaskType::ColdRun => default_cold_chain(),
|
||||
TaskType::SeedStep => default_seed_chain(),
|
||||
});
|
||||
let chain = custom_chain.unwrap_or_else(|| default_chain_for_strategy(current_strategy));
|
||||
|
||||
let mut stage_summaries = Vec::new();
|
||||
let mut current_seed: Option<PathBuf> = seed_atmos.map(|p| p.to_path_buf());
|
||||
@@ -329,13 +347,14 @@ impl<'a> ExecutionRunner<'a> {
|
||||
&stage_def.ltgray,
|
||||
metals,
|
||||
stage_def.ilvlin,
|
||||
tlusty_input,
|
||||
);
|
||||
|
||||
let input5_path = model_dir.join(format!("{}.5", name));
|
||||
tokio::fs::write(&input5_path, &input5_text).await?;
|
||||
|
||||
// Write nst file
|
||||
let nst_text = generate_nst_content(stage_def);
|
||||
let nst_text = generate_nst_content(stage_def, tlusty_input);
|
||||
tokio::fs::write(model_dir.join("nst"), &nst_text).await?;
|
||||
|
||||
// Prepare fort.8 for this stage
|
||||
@@ -386,7 +405,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
let fort9 = model_dir.join("fort.9");
|
||||
let fort7 = model_dir.join("fort.7");
|
||||
|
||||
let mut stage_summary = StageSummary {
|
||||
let mut stage_summary = StepSummary {
|
||||
label: stage_def.label.clone(),
|
||||
chmax: stage_def.chmax,
|
||||
lte: stage_def.lte.clone(),
|
||||
@@ -397,6 +416,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
last_iter: None,
|
||||
worst_depth: None,
|
||||
n_depths: None,
|
||||
itek_history: Vec::new(),
|
||||
};
|
||||
|
||||
if rc == 0 && fort7.is_file() {
|
||||
@@ -410,15 +430,50 @@ impl<'a> ExecutionRunner<'a> {
|
||||
stage_summary.last_iter = res.last_iter;
|
||||
stage_summary.worst_depth = Some(res.worst_depth);
|
||||
stage_summary.n_depths = Some(res.n_depths);
|
||||
// itek 全量保真（Phase 5b）：逐次迭代诊断随 summary_json/conv.json 落库。
|
||||
stage_summary.itek_history = res.itek_history;
|
||||
|
||||
// 漏洞5修复：发散时从 fort.6 提取求解器 STOP 行（SOLVE/SOLVES/RYBSOL）
|
||||
// 作为 note，提升归因质量。仅未收敛且无既有 note 时补（避免覆盖错误信息）。
|
||||
if !res.converged && stage_summary.note.is_none() {
|
||||
let fort6 = model_dir.join(format!("{}.6", name));
|
||||
if let Some(h) = extract_failure_hint(&fort6) {
|
||||
stage_summary.note = Some(format!("未收敛 [{}]", h));
|
||||
}
|
||||
}
|
||||
|
||||
// Save fort.9 snapshot
|
||||
let snap_name = format!("{}.{}_chmax{}.9", name, stage_def.label, eff_chmax);
|
||||
let _ = tokio::fs::copy(&fort9, model_dir.join(snap_name)).await;
|
||||
} else {
|
||||
// NITER=0 grey start without fort.9
|
||||
stage_summary.converged = true;
|
||||
stage_summary.best_max_relc = Some(0.0);
|
||||
stage_summary.note = Some("NITER=0 grey start".to_string());
|
||||
// fort.9 缺失：按 stage_def.niter 区分两种场景（漏洞2进阶修复）。
|
||||
// - niter==0：合法 grey start（lte 阶段不迭代，TLUSTY 不写 fort.9）。
|
||||
// converged=true 保留 grey start 语义；best_max_relc=None 不虚构
|
||||
// （避免污染 final_max_relc/种子选择）。
|
||||
// - niter>0：异常——配了迭代却无 fort.9，通常是 TLUSTY 启动失败
|
||||
// （call quit，如 temp 越界）或 IO 异常。判 converged=false，
|
||||
// 避免把崩溃误判为收敛。此前两种场景共用无校验分支无法区分。
|
||||
if stage_def.niter == 0 {
|
||||
stage_summary.converged = true;
|
||||
stage_summary.best_max_relc = None;
|
||||
stage_summary.note = Some("NITER=0 grey start".to_string());
|
||||
} else {
|
||||
stage_summary.converged = false;
|
||||
stage_summary.best_max_relc = None;
|
||||
// 补 fort.6 失败诊断（call quit 留言），便于排查启动失败原因。
|
||||
let fort6 = model_dir.join(format!("{}.6", name));
|
||||
let hint = extract_failure_hint(&fort6);
|
||||
stage_summary.note = Some(match hint {
|
||||
Some(h) => format!(
|
||||
"stage {} 配置 NITER={} 但 fort.9 缺失 [{}]",
|
||||
stage_def.label, stage_def.niter, h
|
||||
),
|
||||
None => format!(
|
||||
"stage {} 配置 NITER={} 但 fort.9 缺失（TLUSTY 未完成迭代）",
|
||||
stage_def.label, stage_def.niter
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Copy fort.7 as stage seed
|
||||
@@ -426,7 +481,14 @@ impl<'a> ExecutionRunner<'a> {
|
||||
let _ = tokio::fs::copy(&fort7, &stage_seed_path).await;
|
||||
current_seed = Some(stage_seed_path);
|
||||
} else {
|
||||
stage_summary.note = Some(format!("tlusty rc={} or missing fort.7", rc));
|
||||
// 漏洞5修复：fort.7 缺失分支（输入错误、temp 越界等 call quit 场景），
|
||||
// 从 fort.6 尾部提取 call quit / stop 留言补进 note，便于排查。
|
||||
let fort6 = model_dir.join(format!("{}.6", name));
|
||||
let hint = extract_failure_hint(&fort6);
|
||||
stage_summary.note = Some(match hint {
|
||||
Some(h) => format!("tlusty rc={} or missing fort.7 [{}]", rc, h),
|
||||
None => format!("tlusty rc={} or missing fort.7", rc),
|
||||
});
|
||||
}
|
||||
|
||||
// 快照本阶段的同名输入/输出文件，带阶段标签保留。
|
||||
@@ -485,7 +547,15 @@ impl<'a> ExecutionRunner<'a> {
|
||||
// fort.12/14 不存在，函数内按文件是否存在静默跳过。
|
||||
snapshot_tlusty_outputs(&model_dir, name).await;
|
||||
|
||||
let atmo_has_nan = atmosphere_has_nan(&final_7);
|
||||
// L2 修复：`atmosphere_has_nan` 对**缺失**文件返回 false（语义是"无 NaN"而非"有效"），
|
||||
// 与**空文件返回 true** 语义不对称。缺失最终大气 = 无可判定收敛的干净大气 →
|
||||
// 在此显式判定为无效（NaN），与空文件语义对齐。调用前提：final_7 应在收敛链产出；
|
||||
// 若缺失（如种子拷贝失败、TLUSTY 崩溃未写 fort.7），本守卫强制最终不收敛。
|
||||
let atmo_has_nan = if final_7.is_file() {
|
||||
atmosphere_has_nan(&final_7)
|
||||
} else {
|
||||
true
|
||||
};
|
||||
if atmo_has_nan {
|
||||
final_converged = false;
|
||||
}
|
||||
@@ -518,7 +588,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
let _ = tokio::fs::remove_file(&fort55_path).await;
|
||||
let _ = tokio::fs::remove_file(&fort19_path).await;
|
||||
|
||||
let default_cfg = SynspecConfig {
|
||||
let default_cfg = SynspecInput {
|
||||
wstart: 1400.0,
|
||||
wend: 1410.0,
|
||||
imode: 0,
|
||||
@@ -577,11 +647,22 @@ impl<'a> ExecutionRunner<'a> {
|
||||
|
||||
// Copy/move outputs: fort.7 (Synspec spectrum) -> .spec, fort.17 -> .cont, fort.12 -> .iden
|
||||
if model_dir.join("fort.7").is_file() {
|
||||
let _ = tokio::fs::rename(
|
||||
model_dir.join("fort.7"),
|
||||
model_dir.join(format!("{}.spec", name)),
|
||||
)
|
||||
.await;
|
||||
let spec_path = model_dir.join(format!("{}.spec", name));
|
||||
let _ = tokio::fs::rename(model_dir.join("fort.7"), &spec_path).await;
|
||||
|
||||
// 漏洞1修复（P0）：SYNSPEC .spec 内容校验。
|
||||
// gfortran 下 SYNSPEC 几乎所有错误路径 rc=0，旧代码只做 is_file() 存在性
|
||||
// 检查，导致脏谱（NaN/Inf/行数不足/全零）被当作 Completed 归档——全链路
|
||||
// 最大的科学正确性风险。命中无效则置 synspec_rc 非零 + synspec_error 描述，
|
||||
// 让 reporter 判 Failed 并触发 synspec 策略链回退。
|
||||
// 守卫 synspec_err.is_none()：避免覆盖上游 fort.8/fort.55 复制失败的既有 err。
|
||||
if synspec_err.is_none() {
|
||||
if let Some(reason) = spec_is_valid(&spec_path) {
|
||||
warn!("spec 校验失败: {}", reason);
|
||||
synspec_rc = Some(1);
|
||||
synspec_err = Some(reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
if model_dir.join("fort.17").is_file() {
|
||||
let _ = tokio::fs::copy(
|
||||
@@ -642,7 +723,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
let note = {
|
||||
let mut notes: Vec<String> = Vec::new();
|
||||
if atmo_has_nan {
|
||||
notes.push("Invalidated: atmosphere contains >10% NaN lines".to_string());
|
||||
notes.push("Invalidated: atmosphere contains NaN/Inf lines".to_string());
|
||||
}
|
||||
if let Some(ref err) = synspec_err {
|
||||
notes.push(format!("synspec error: {}", err));
|
||||
@@ -662,7 +743,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
name: name.to_string(),
|
||||
params: params.clone(),
|
||||
stages: stage_summaries,
|
||||
converged: final_converged,
|
||||
result_valid: final_converged,
|
||||
final_max_relc,
|
||||
final_chmax,
|
||||
seed: seed_atmos.map(|p| p.to_string_lossy().to_string()),
|
||||
@@ -687,6 +768,29 @@ mod tests {
|
||||
use super::snapshot_tlusty_outputs;
|
||||
use crate::models::{GridAxisValue, GridPointParams};
|
||||
|
||||
/// Phase 6（P8）：执行链按当前策略派生——seed_step 走种子热启动链，其余走冷启动链。
|
||||
#[test]
|
||||
fn test_default_chain_for_strategy() {
|
||||
let labels = |chain: Vec<super::ChainStep>| -> Vec<String> {
|
||||
chain.into_iter().map(|s| s.label).collect()
|
||||
};
|
||||
// seed_step → 种子热启动链（seed_nc/nl）。
|
||||
assert_eq!(
|
||||
labels(super::default_chain_for_strategy("seed_step")),
|
||||
vec!["seed_nc".to_string(), "nl".to_string()]
|
||||
);
|
||||
// 其余策略（cold_run / 未知如 standard）→ 冷启动链（lte/nc/nl）。
|
||||
assert_eq!(
|
||||
labels(super::default_chain_for_strategy("cold_run")),
|
||||
vec!["lte".to_string(), "nc".to_string(), "nl".to_string()]
|
||||
);
|
||||
assert_eq!(
|
||||
labels(super::default_chain_for_strategy("standard")),
|
||||
vec!["lte".to_string(), "nc".to_string(), "nl".to_string()],
|
||||
"未知策略兜底冷启动链（synspec-only strategies[0] 等）"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_synspec_timeout_calculation() {
|
||||
let long_tlusty_timeout: u64 = 7200;
|
||||
|
||||
@@ -15,7 +15,7 @@ pub const MAX_GLOBAL_SEED_DISTANCE: f64 = 3.0;
|
||||
/// **数据标定依据**——对历史 1191 个真实 seed_step（种子,目标）配对的成败统计：
|
||||
/// - 贫金属方向（种子更富、目标往贫走，delta=目标−种子 < 0）：成功率 **42–54%**
|
||||
/// - 富金属方向（目标更富、delta > 0）：成功率仅 **3–11%**
|
||||
/// （每个 loghe 分层该规律独立成立，he=−4 时贫方向 54% vs 富方向 3%，差 18 倍）
|
||||
/// （每个 loghe 分层该规律独立成立，he=−4 时贫方向 54% vs 富方向 3%，差 18 倍）
|
||||
///
|
||||
/// **物理解释**：从高金属丰度的收敛解出发**减少**金属（贫方向）是稳定微扰；
|
||||
/// 反过来从贫金属种子**增加**金属（富方向），新增的紫外谱线辐射驱动会破坏已建立的
|
||||
|
||||
@@ -212,14 +212,9 @@ impl SqliteTaskQueue {
|
||||
}
|
||||
};
|
||||
|
||||
// 旧版兼容归一化（见 models.rs::normalize_compat 文档）：
|
||||
// 旧 MQ 在途消息可能只含 task_type、无 tlusty_config 字段。serde default
|
||||
// 会把 tlusty_config.strategies 填为完整默认链 [cold_run, seed_step]，
|
||||
// 导致旧 SeedStep 热启动消息首项被误判为 cold_run。normalize_compat
|
||||
// 据 task_type 校正首项策略。在 pop 出队后立即调用，保证节点拿到的是
|
||||
// 语义正确的策略链。对新版消息（已显式设置 tlusty_config）无副作用。
|
||||
let mut task = task;
|
||||
task.normalize_compat();
|
||||
// Phase 6（P8）：task_type 兼容字段与 normalize_compat 整体废弃。
|
||||
// 升级前须确认队列为空（docs/database_refactor_design.md §8.8）——残留
|
||||
// 旧 payload 的 strategies 会按 serde default 填成默认链，无法再校正。
|
||||
|
||||
// 记录任务归属：claim 时写入领用方 node_id，供 report 阶段校验，
|
||||
// 杜绝「节点 A 领用、节点 B 上报」的跨节点伪造结果投毒。
|
||||
@@ -391,13 +386,10 @@ impl SqliteTaskQueue {
|
||||
let mut stmt = conn.prepare(
|
||||
"DELETE FROM task_queue WHERE workflow_name = ?1 AND status = 'pending' RETURNING task_id",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![wf_owned], |row| row.get::<_, String>(0))?;
|
||||
let mut ids = Vec::new();
|
||||
for r in rows {
|
||||
if let Ok(id) = r {
|
||||
ids.push(id);
|
||||
}
|
||||
}
|
||||
let ids: Vec<String> = stmt
|
||||
.query_map(params![wf_owned], |row| row.get::<_, String>(0))?
|
||||
.filter_map(Result::ok)
|
||||
.collect();
|
||||
Ok(ids)
|
||||
})
|
||||
.await??;
|
||||
@@ -431,7 +423,7 @@ impl SqliteTaskQueue {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use common::models::{GridPointParams, TaskType};
|
||||
use common::models::GridPointParams;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// H1 修复：队列库迁移必须把历史遗留的 NULL workflow_name 行回填为 '__legacy__'
|
||||
@@ -467,7 +459,9 @@ mod tests {
|
||||
}
|
||||
|
||||
// 打开队列库（触发迁移：补列检查 + NULL workflow_name 回填）。
|
||||
let queue = SqliteTaskQueue::new(&db_path.to_string_lossy()).await.unwrap();
|
||||
let queue = SqliteTaskQueue::new(&db_path.to_string_lossy())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let wf: String = {
|
||||
let conn = queue.pool.get().unwrap();
|
||||
@@ -503,7 +497,6 @@ mod tests {
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
},
|
||||
task_type: TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 3600,
|
||||
workflow_name: Some("test_wf".to_string()),
|
||||
@@ -552,7 +545,6 @@ mod tests {
|
||||
task_id,
|
||||
point_name: params.model_name(),
|
||||
params: params.clone(),
|
||||
task_type: TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 60,
|
||||
workflow_name: None,
|
||||
@@ -616,7 +608,6 @@ mod tests {
|
||||
task_id,
|
||||
point_name: params.model_name(),
|
||||
params: params.clone(),
|
||||
task_type: TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 7200,
|
||||
workflow_name: Some("wf_rq".to_string()),
|
||||
@@ -650,7 +641,7 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
claim_a.map(|(p, w)| (p, w)),
|
||||
claim_a,
|
||||
Some((params.model_name(), Some("wf_rq".to_string()))),
|
||||
"requeue 后原节点仍持归属，应放行"
|
||||
);
|
||||
@@ -690,7 +681,6 @@ mod tests {
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
},
|
||||
task_type: TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 60,
|
||||
workflow_name: Some(wf.to_string()),
|
||||
@@ -803,20 +793,18 @@ mod tests {
|
||||
assert!(queue.pop_task("n1").await.unwrap().is_none());
|
||||
}
|
||||
|
||||
/// 旧版 MQ 在途消息兼容（二次审查 critical 修复验证）：
|
||||
/// 旧消息只含 task_type=seed_step、无 tlusty_config 字段。serde default 会把
|
||||
/// strategies 填为 [cold_run, seed_step]，首项 cold_run 与 task_type 不符。
|
||||
/// pop_task 出队时须调用 normalize_compat() 把首项校正为 seed_step，
|
||||
/// 否则旧热启动消息会被节点误当冷启动执行。
|
||||
/// Phase 6（P8）：旧版消息兼容（normalize_compat）整体废弃。升级前确认队列为空
|
||||
/// （docs/database_refactor_design.md §8.8）后，旧 payload 的 task_type 键被 serde 忽略，
|
||||
/// strategies 回落到默认链——出队仍正常，只是不再有 task_type 校正。
|
||||
#[tokio::test]
|
||||
async fn test_pop_normalizes_legacy_seed_step_message() {
|
||||
async fn test_pop_legacy_payload_without_task_type_correction() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let db_path = temp_dir.path().join("legacy.db");
|
||||
let queue = SqliteTaskQueue::new(&db_path.to_string_lossy())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// 直接写一行旧版 payload（仅 task_type，无 tlusty_config/synspec_config 字段）。
|
||||
// 旧版 payload 仍携带 task_type 键（已被忽略）+ 无 tlusty_config（回落默认链）。
|
||||
let legacy_task_id = uuid::Uuid::new_v4();
|
||||
let legacy_payload = serde_json::json!({
|
||||
"task_id": legacy_task_id.to_string(),
|
||||
@@ -848,16 +836,11 @@ mod tests {
|
||||
|
||||
let popped = queue.pop_task("n1").await.unwrap();
|
||||
let task = popped.expect("旧版消息应能正常出队");
|
||||
// normalize_compat 应把 strategies 校正为 [seed_step]（与 task_type 一致）。
|
||||
assert_eq!(
|
||||
task.tlusty_config.strategies,
|
||||
vec!["seed_step".to_string()],
|
||||
"旧 seed_step 消息经 normalize_compat 后首项应为 seed_step"
|
||||
);
|
||||
// 无 task_type 校正：strategies 为 serde default 默认链，首项 cold_run。
|
||||
assert_eq!(
|
||||
task.tlusty_config.current_strategy("cold_run"),
|
||||
"seed_step",
|
||||
"current_strategy 应为 seed_step（而非默认链的 cold_run）"
|
||||
"cold_run",
|
||||
"旧 payload 无显式 strategies，回落到默认链首项"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
+132
-20
@@ -1,5 +1,5 @@
|
||||
use anyhow::Result;
|
||||
use common::config::SynspecConfig;
|
||||
use common::config::{ChainStep, SynspecInput, TlustyInput};
|
||||
use common::embedded::{ensure_specific_data_files, RuntimePaths};
|
||||
use common::models::{ModelSummary, TaskSpec};
|
||||
use common::result_filter::is_result_worthy;
|
||||
@@ -157,22 +157,49 @@ pub async fn execute_task(
|
||||
// 3. （slot_work_dir 已在种子下载前提前创建，种子私有副本亦已落盘于沙盒内。）
|
||||
|
||||
// 反序列化工作流携带的 SYNSPEC 数值参数（波长范围等）。None → runner 用硬编码默认。
|
||||
let synspec_cfg: Option<SynspecConfig> = task
|
||||
let synspec_cfg: Option<SynspecInput> = task
|
||||
.synspec_params
|
||||
.as_ref()
|
||||
.and_then(|v| serde_json::from_value::<SynspecConfig>(v.clone()).ok());
|
||||
.and_then(|v| serde_json::from_value::<SynspecInput>(v.clone()).ok());
|
||||
|
||||
let runner = ExecutionRunner::new(runtime, slot_work_dir.clone());
|
||||
// 执行链来源（优先级）：
|
||||
// 1. TaskSpec.tlusty_chain_params（用户在 YAML `tlusty_chain:` 配置的多阶段 ChainStep
|
||||
// 数组，由 scheduler 序列化注入）——非空时优先使用，使用户能细粒度控制 niter/chmax/
|
||||
// metals 等阶段参数。
|
||||
// 2. default_chain_for_strategy(current_strategy) 兜底——按策略名（cold_run/seed_step）
|
||||
// 选预设默认链（runner.rs 的 default_cold_chain / default_seed_chain）。
|
||||
// 历史：Phase 6 起仅用 default 链（用户 config.chain 被忽略，是死字段）；本次接通后
|
||||
// 用户配置真正生效，default 链降级为兜底。旧 MQ payload（无 tlusty_chain_params 字段）
|
||||
// 反序列化为 None → 回退 default 链，行为与旧版完全一致（向后兼容）。
|
||||
let chain = resolve_execution_chain(
|
||||
current_strategy,
|
||||
&task.tlusty_chain_params,
|
||||
&task.task_id.to_string(),
|
||||
);
|
||||
// TLUSTY 输入文件全局参数（NFREAD/ions 表/nst extra_keys 等）。
|
||||
// None → runner 用代码内硬编码默认（向后兼容）。
|
||||
let tlusty_input =
|
||||
task.tlusty_input_params.as_ref().and_then(|v| {
|
||||
match serde_json::from_value::<TlustyInput>(v.clone()) {
|
||||
Ok(t) => Some(t),
|
||||
Err(e) => {
|
||||
warn!(
|
||||
"任务 {} 的 tlusty_input_params 反序列化失败，回退默认输入: {}",
|
||||
task.task_id, e
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
});
|
||||
let summary = runner
|
||||
.run_model_with_timeout(
|
||||
&task.params,
|
||||
// 用权威的 point_name（DB grid_points.name 列，源精度正确）作为模型名，
|
||||
// 而非 task.params.model_name()（后者经 DB REAL 列回读已丢精度 "5.0"→"5"）。
|
||||
&task.point_name,
|
||||
task.task_type.clone(),
|
||||
// custom_chain 恒为 None：执行链由 task_type 决定（ColdRun→default_cold_chain、
|
||||
// SeedStep→default_seed_chain），节点端不再做「缺种子回退冷启动链」的降级。
|
||||
None,
|
||||
current_strategy,
|
||||
Some(chain),
|
||||
seed_atmos_path.as_deref(),
|
||||
synspec_cfg.as_ref(),
|
||||
// 阶段独立配置开关（见 docs/task_engine_decoupling_design.md §5）。
|
||||
@@ -180,17 +207,18 @@ pub async fn execute_task(
|
||||
task.synspec_config.enabled,
|
||||
task.timeout_sec,
|
||||
shutdown,
|
||||
tlusty_input.as_ref(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
info!(
|
||||
"完成计算任务 {} (网格点: {}, 收敛状态: {})",
|
||||
task.task_id, task.point_name, summary.converged
|
||||
"完成计算任务 {} (网格点: {}, 结果可用: {})",
|
||||
task.task_id, task.point_name, summary.result_valid
|
||||
);
|
||||
|
||||
// Read seed bytes if converged and clean
|
||||
// Read seed bytes if result usable and clean
|
||||
let mut seed_bytes: Option<Vec<u8>> = None;
|
||||
if summary.converged && !summary.atmosphere_has_nan {
|
||||
if summary.result_valid && !summary.atmosphere_has_nan {
|
||||
let model_sub_dir = slot_work_dir.join(&summary.name);
|
||||
let candidates = [
|
||||
model_sub_dir.join(format!("{}.7", summary.name)),
|
||||
@@ -350,15 +378,23 @@ pub async fn save_result_artifacts(result_dir: &Path, slot_work_dir: &Path, name
|
||||
);
|
||||
}
|
||||
|
||||
/// 归档目录不做数量上限治理：所有已算网格点的完整产物（.spec/.cont/.iden/各阶段
|
||||
/// 快照/日志/种子二进制等）一律永久保留，避免 LRU 淘汰导致科学产物丢失
|
||||
/// （2026-08-02 修正：撤销 1bfa240 引入的 MAX_RESULT_MODELS=200 LRU 上限）。
|
||||
// 归档目录不做数量上限治理：所有已算网格点的完整产物（.spec/.cont/.iden/各阶段
|
||||
// 快照/日志/种子二进制等）一律永久保留，避免 LRU 淘汰导致科学产物丢失
|
||||
// （2026-08-02 修正：撤销 1bfa240 引入的 MAX_RESULT_MODELS=200 LRU 上限）。
|
||||
|
||||
/// `.seed_cache/` 内保留的 `.seed.7` 文件上限。超过则按 mtime 删除最旧的。
|
||||
/// 典型网格内活跃种子点数量有限，8 足以覆盖常用邻域且把磁盘占用控制在 ~8 个种子文件。
|
||||
const MAX_SEED_CACHE_FILES: usize = 8;
|
||||
///
|
||||
/// 审查修复 #N5：上限可经环境变量 `DCTS_SEED_CACHE_MAX` 覆盖（默认 8）。多工作流或密集
|
||||
/// 网格下常用邻域种子可能超过 8 个，硬编码上限会导致反复从 server 下载，增加负载。
|
||||
fn seed_cache_max_files() -> usize {
|
||||
std::env::var("DCTS_SEED_CACHE_MAX")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.filter(|n: &usize| *n > 0)
|
||||
.unwrap_or(8)
|
||||
}
|
||||
|
||||
/// LRU 清理种子缓存目录：当 `.seed.7` 文件数超过 `MAX_SEED_CACHE_FILES` 时，
|
||||
/// LRU 清理种子缓存目录：当 `.seed.7` 文件数超过上限（`DCTS_SEED_CACHE_MAX`，默认 8）时，
|
||||
/// 按 mtime 升序删除最旧的若干个，直到不超过上限。仅统计 `.seed.7`，忽略 `.tmp` 中间文件。
|
||||
/// 任何 IO 错误均降级为 warn，不阻断主流程。
|
||||
pub async fn cleanup_seed_cache(seed_dir: &Path) {
|
||||
@@ -391,13 +427,14 @@ pub async fn cleanup_seed_cache(seed_dir: &Path) {
|
||||
Err(_) => return,
|
||||
};
|
||||
|
||||
if entries.len() <= MAX_SEED_CACHE_FILES {
|
||||
let max_files = seed_cache_max_files();
|
||||
if entries.len() <= max_files {
|
||||
return;
|
||||
}
|
||||
|
||||
// 按 mtime 升序（最旧在前），删除超出上限的最旧文件
|
||||
entries.sort_by_key(|(mtime, _)| *mtime);
|
||||
let to_remove = entries.len().saturating_sub(MAX_SEED_CACHE_FILES);
|
||||
let to_remove = entries.len().saturating_sub(max_files);
|
||||
for (_, path) in entries.into_iter().take(to_remove) {
|
||||
match tokio::fs::remove_file(&path).await {
|
||||
Ok(()) => info!("LRU 清理种子缓存文件: {}", path.display()),
|
||||
@@ -411,6 +448,43 @@ pub async fn cleanup_seed_cache(seed_dir: &Path) {
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析任务要执行的大气链（冷启动链 / 种子热启动链）。
|
||||
///
|
||||
/// 优先级（修复回归）：
|
||||
/// - `current_strategy == "seed_step"` → 强制 `default_seed_chain()`(seed_nc→nl)。
|
||||
/// 自定义 `tlusty_chain` 是冷启动链：首步 lte 的 `ltgray=T` 会删除 fort.8、丢弃已下载的
|
||||
/// 热启动种子(runner.rs 阶段 fort.8 准备逻辑)。scheduler 在派发与回退两条路径都注入
|
||||
/// 同一个 `tlusty_chain`，若 seed_step 也沿用自定义链，会把种子回退退化成本地冷启动，
|
||||
/// 丢失热启动语义。故种子链固定走内置默认，仅在 cold_run 等冷策略下信任用户自定义链。
|
||||
/// - 其余策略 → 优先 TaskSpec.tlusty_chain_params(用户 YAML `tlusty_chain:` 配置)，非空即用；
|
||||
/// 为空/反序列化失败 → `default_chain_for_strategy(current_strategy)` 兜底。
|
||||
fn resolve_execution_chain(
|
||||
current_strategy: &str,
|
||||
tlusty_chain_params: &Option<serde_json::Value>,
|
||||
task_id: &str,
|
||||
) -> Vec<ChainStep> {
|
||||
if current_strategy == "seed_step" {
|
||||
common::runner::default_seed_chain()
|
||||
} else {
|
||||
tlusty_chain_params
|
||||
.as_ref()
|
||||
.and_then(
|
||||
|v| match serde_json::from_value::<Vec<ChainStep>>(v.clone()) {
|
||||
Ok(c) => Some(c),
|
||||
Err(e) => {
|
||||
warn!(
|
||||
"任务 {} 的 tlusty_chain_params 反序列化失败，回退 default 链: {}",
|
||||
task_id, e
|
||||
);
|
||||
None
|
||||
}
|
||||
},
|
||||
)
|
||||
.filter(|c| !c.is_empty())
|
||||
.unwrap_or_else(|| common::runner::default_chain_for_strategy(current_strategy))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -429,7 +503,7 @@ mod tests {
|
||||
name: params.model_name(),
|
||||
params,
|
||||
stages: vec![],
|
||||
converged: true,
|
||||
result_valid: true,
|
||||
final_max_relc: Some(0.0005),
|
||||
final_chmax: None,
|
||||
seed: None,
|
||||
@@ -442,6 +516,44 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn labels(chain: &[ChainStep]) -> Vec<String> {
|
||||
chain.iter().map(|s| s.label.clone()).collect()
|
||||
}
|
||||
|
||||
/// P1 回归防护：seed_step 即使注入自定义冷启动链，也必须强制走种子热启动默认链
|
||||
/// （seed_nc→nl），否则首步 lte(ltgray=T) 会删 fort.8、丢弃已下载种子，回退退化成本地冷启动。
|
||||
#[test]
|
||||
fn seed_step_ignores_custom_cold_chain() {
|
||||
let custom = serde_json::json!([
|
||||
{"label": "lte", "lte": "T", "ltgray": "T", "ilvlin": 0, "niter": 0},
|
||||
{"label": "nc", "lte": "F", "ltgray": "F", "ilvlin": 0, "niter": 10},
|
||||
{"label": "nl", "lte": "F", "ltgray": "F", "ilvlin": 100, "niter": 100},
|
||||
]);
|
||||
|
||||
let chain = resolve_execution_chain("seed_step", &Some(custom), "t1");
|
||||
assert_eq!(labels(&chain), vec!["seed_nc", "nl"]);
|
||||
// 首步必须是非灰 LTE（ltgray=F），否则会删 fort.8 丢弃种子。
|
||||
assert_eq!(chain[0].ltgray, "F");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cold_run_uses_custom_chain_when_provided() {
|
||||
let custom = serde_json::json!([
|
||||
{"label": "lte", "lte": "T", "ltgray": "T", "ilvlin": 0, "niter": 0},
|
||||
{"label": "nc", "lte": "F", "ltgray": "F", "ilvlin": 0, "niter": 10},
|
||||
{"label": "nl", "lte": "F", "ltgray": "F", "ilvlin": 100, "niter": 100},
|
||||
]);
|
||||
|
||||
let chain = resolve_execution_chain("cold_run", &Some(custom), "t2");
|
||||
assert_eq!(labels(&chain), vec!["lte", "nc", "nl"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cold_run_falls_back_to_default_when_no_custom_chain() {
|
||||
let chain = resolve_execution_chain("cold_run", &None, "t3");
|
||||
assert_eq!(labels(&chain), vec!["lte", "nc", "nl"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cleanup_slot_work_dir() {
|
||||
let temp_dir =
|
||||
|
||||
@@ -12,8 +12,14 @@ use tracing::{info, warn};
|
||||
/// 失败导致 synspec 无输入"的级联场景）；TLUSTY 关闭 → 归因 SYNSPEC（synspec-only 任务
|
||||
/// 的 converged 由 synspec_rc 决定）。
|
||||
/// - `converged=true` 但 synspec 有错误/非零 rc：归因 SYNSPEC（大气已收敛、光谱失败）。
|
||||
///
|
||||
/// 已知盲区（审查 #N1，文档化取舍）：TLUSTY 关闭 + `converged=false` 时归因 SYNSPEC，
|
||||
/// 但此时大气来自外部既有产物（非本任务计算），`converged=false` 更可能意味着外部大气
|
||||
/// 文件损坏/NaN/缺失，而非光谱合成本身的问题。归因到 synspec 触发的 synspec 链重试对
|
||||
/// 「外部大气损坏」无济于事（重试仍用同一损坏大气）。当前仍按 synspec 归因是保守选择
|
||||
/// （至少触发一次重试暴露问题），运维需结合 note 里的 atmosphere_has_nan 标志甄别。
|
||||
fn infer_failed_stage(task: &TaskSpec, summary: &ModelSummary) -> Option<String> {
|
||||
if !summary.converged {
|
||||
if !summary.result_valid {
|
||||
return if task.tlusty_config.enabled {
|
||||
Some("tlusty".to_string())
|
||||
} else {
|
||||
@@ -37,7 +43,7 @@ fn infer_failed_stage(task: &TaskSpec, summary: &ModelSummary) -> Option<String>
|
||||
/// 保存 .7 作种子），但 status=Failed 让服务端弹 synspec 链重试，且错误对运维可见。
|
||||
fn derive_report_status(s: &ModelSummary) -> TaskStatus {
|
||||
let synspec_failed = s.synspec_error.is_some() || matches!(s.synspec_rc, Some(rc) if rc != 0);
|
||||
if s.converged && !synspec_failed {
|
||||
if s.result_valid && !synspec_failed {
|
||||
TaskStatus::Completed
|
||||
} else {
|
||||
TaskStatus::Failed
|
||||
@@ -66,7 +72,7 @@ pub async fn report_result(
|
||||
) = match exec_res {
|
||||
Ok((s, s_bytes)) => (
|
||||
derive_report_status(&s),
|
||||
s.converged,
|
||||
s.result_valid,
|
||||
s.final_max_relc,
|
||||
s.atmosphere_has_nan,
|
||||
s.elapsed_sec,
|
||||
@@ -94,7 +100,7 @@ pub async fn report_result(
|
||||
params: Some(task.params.clone()),
|
||||
node_id: node_id.to_string(),
|
||||
status,
|
||||
converged,
|
||||
result_valid: converged,
|
||||
max_relc,
|
||||
atmosphere_has_nan: atmo_has_nan,
|
||||
elapsed_sec,
|
||||
@@ -196,16 +202,16 @@ mod tests {
|
||||
logn: (-4.0).into(),
|
||||
logo: (-4.0).into(),
|
||||
},
|
||||
tlusty_config: common::models::EngineStageConfig {
|
||||
tlusty_config: common::models::PhaseConfig {
|
||||
enabled: tlusty_enabled,
|
||||
..common::models::EngineStageConfig::default_tlusty()
|
||||
..common::models::PhaseConfig::default_tlusty()
|
||||
},
|
||||
..TaskSpec::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn mk_summary(
|
||||
converged: bool,
|
||||
result_valid: bool,
|
||||
synspec_rc: Option<i32>,
|
||||
synspec_error: Option<&str>,
|
||||
) -> ModelSummary {
|
||||
@@ -220,7 +226,7 @@ mod tests {
|
||||
logo: (-4.0).into(),
|
||||
},
|
||||
stages: vec![],
|
||||
converged,
|
||||
result_valid,
|
||||
final_max_relc: None,
|
||||
final_chmax: None,
|
||||
seed: None,
|
||||
@@ -302,4 +308,27 @@ mod tests {
|
||||
TaskStatus::Failed
|
||||
);
|
||||
}
|
||||
|
||||
/// 漏洞1修复验证（端到端判定链路）：.spec 内容校验失败时，runner 同时设置
|
||||
/// synspec_rc=Some(1) + synspec_error=Some("spec...")。验证 derive_report_status 和
|
||||
/// infer_failed_stage 在双字段同时非空/非零时都判 synspec 失败 → Failed + 归因 synspec。
|
||||
/// 这是漏洞1失败信号从 runner 一路流到 reporter 判定的关键节点。
|
||||
#[test]
|
||||
fn test_spec_invalid_drives_failed() {
|
||||
// 模拟 spec_is_valid 命中：converged=true（大气正常）+ spec 脏（双字段）
|
||||
let dirty_spec_summary =
|
||||
mk_summary(true, Some(1), Some("spec 含 NaN/Inf/溢出行 (共 2 行)"));
|
||||
// 任务整体判 Failed（不再被误报 Completed → 脏谱归档）
|
||||
assert_eq!(
|
||||
derive_report_status(&dirty_spec_summary),
|
||||
TaskStatus::Failed,
|
||||
"spec 校验失败（双字段）应判 Failed"
|
||||
);
|
||||
// 归因 synspec（触发 synspec 策略链回退）
|
||||
assert_eq!(
|
||||
infer_failed_stage(&mk_task(true), &dirty_spec_summary),
|
||||
Some("synspec".to_string()),
|
||||
"spec 校验失败应归因 synspec 以触发回退"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+117
-9
@@ -6,9 +6,11 @@ use common::embedded::RuntimePaths;
|
||||
use common::models::{NodeHeartbeatRequest, NodeHeartbeatResponse, NodeRegisterRequest, TaskSpec};
|
||||
use reqwest::Client;
|
||||
use serde_json::Value;
|
||||
use std::collections::HashSet;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicI32, AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::time::{sleep, Duration};
|
||||
use tracing::{info, warn};
|
||||
|
||||
@@ -32,10 +34,65 @@ impl Drop for SlotGuard {
|
||||
}
|
||||
}
|
||||
|
||||
/// 后台沙盒 GC 间隔（秒）：长寿命 Worker 运行期间周期清理孤儿 `task_*` 目录。
|
||||
/// 孤儿来源：`cleanup_slot_work_dir` 反复失败（权限/文件锁）或任务进程异常残留。
|
||||
/// 启动清理（run() 开头）只处理「上次崩溃」的残留；运行期累积靠本 GC 兜底。
|
||||
/// 精确跳过 `active_tasks` 中的在途任务，故无需担心误删运行中沙盒。
|
||||
const SANDBOX_GC_INTERVAL_SECS: u64 = 3600;
|
||||
|
||||
/// 周期清理 work_dir 下非活跃的 `task_*` 沙盒目录。
|
||||
///
|
||||
/// - 目录名 `task_{uuid}`，后缀精确匹配 `active_tasks`（在途任务 task_id 集合）；
|
||||
/// 命中的在途任务跳过，其余视为孤儿删除（best-effort，失败 warn 下轮重试）。
|
||||
/// - 与启动清理（run() 开头删全部）互补：运行期若单 slot 清理反复失败，孤儿子目录
|
||||
/// 不会累积写满磁盘，由本任务周期性回收。
|
||||
/// - 仅在 shutdown 标志置位时退出（每 tick 检查一次），随节点优雅退出。
|
||||
async fn sandbox_gc_loop(
|
||||
work_dir: PathBuf,
|
||||
active_tasks: Arc<Mutex<HashSet<String>>>,
|
||||
shutdown: Arc<AtomicBool>,
|
||||
) {
|
||||
let mut ticker = tokio::time::interval(Duration::from_secs(SANDBOX_GC_INTERVAL_SECS));
|
||||
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
loop {
|
||||
ticker.tick().await;
|
||||
if shutdown.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
let active: HashSet<String> = active_tasks.lock().await.iter().cloned().collect();
|
||||
if let Ok(mut rd) = tokio::fs::read_dir(&work_dir).await {
|
||||
while let Ok(Some(entry)) = rd.next_entry().await {
|
||||
let name = entry.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
if is_orphan_task_dir(&name_str, &active) {
|
||||
let p = entry.path();
|
||||
match tokio::fs::remove_dir_all(&p).await {
|
||||
Ok(_) => warn!("GC 清理孤儿沙盒: {}", p.display()),
|
||||
Err(e) => warn!("GC 清理孤儿沙盒 {} 失败（下轮重试）: {}", p.display(), e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 判断 work_dir 下的一个条目是否为「应回收的孤儿沙盒」。
|
||||
/// `task_{uuid}` 且 uuid 不在 `active_tasks`（在途任务）→ 孤儿；其它一律非孤儿。
|
||||
fn is_orphan_task_dir(entry_name: &str, active: &HashSet<String>) -> bool {
|
||||
match entry_name.strip_prefix("task_") {
|
||||
Some(suffix) => !active.contains(suffix),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 领用请求的归一化结果。
|
||||
///
|
||||
/// 区分「被管理员停用」与「暂无任务」：前者节点保持存活、空闲待命（拉长轮询），
|
||||
/// 后者按常规节奏轮询。服务端返回 `{"status":"disabled"}` 映射为 `Disabled`。
|
||||
///
|
||||
/// `TaskSpec` 体积远大于空枚举项，属可接受的形态（node 端一次性消费，非热循环持有）；
|
||||
/// 装箱会改变所有 match 点的解构方式且无实际收益，故允许该 lint。
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
enum ClaimOutcome {
|
||||
/// 成功领用到任务。
|
||||
Task(TaskSpec),
|
||||
@@ -62,6 +119,12 @@ pub struct NodeWorker {
|
||||
/// （见 docs/dynamic_cpu_slots_design.md §3.2）。用 `Arc<AtomicUsize>` 在心跳线程与
|
||||
/// 领用线程间共享，无需 Mutex：单写者（心跳线程）单读者（领用线程），原子读写即可。
|
||||
effective_max_slots: Arc<AtomicUsize>,
|
||||
/// 全局优雅退出标志：信号线程、心跳线程、claim_task 任一发现需要终止时置 true，
|
||||
/// 主循环据此停止领用新任务并走优雅退出（等待在途任务 ≤30s + 归档沙盒）。
|
||||
///
|
||||
/// 审查修复 #M5：此前心跳/claim_task 遇 token 失效直接 `std::process::exit(1)`，
|
||||
/// 跳过主循环的优雅退出逻辑，导致在途长任务结果静默丢失。改为置此标志让主循环感知。
|
||||
shutting_down: Arc<std::sync::atomic::AtomicBool>,
|
||||
}
|
||||
|
||||
impl NodeWorker {
|
||||
@@ -73,6 +136,7 @@ impl NodeWorker {
|
||||
runtime,
|
||||
active_slots: Arc::new(AtomicI32::new(0)),
|
||||
effective_max_slots: Arc::new(AtomicUsize::new(physical)),
|
||||
shutting_down: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -218,6 +282,7 @@ impl NodeWorker {
|
||||
let hb_physical = self.config.max_slots;
|
||||
let hb_interval = self.config.heartbeat_sec;
|
||||
let hb_runtime_dir = self.config.runtime_dir.clone();
|
||||
let hb_shutting_down = self.shutting_down.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let sys_arc = std::sync::Arc::new(std::sync::Mutex::new(sysinfo::System::new_all()));
|
||||
@@ -275,20 +340,22 @@ impl NodeWorker {
|
||||
match hb_client.post(&hb_url).json(&req).send().await {
|
||||
Ok(resp) => {
|
||||
let status = resp.status();
|
||||
// 401/403：token 失效（被重发覆盖）。与 claim_task 口径统一：直接退出进程，
|
||||
// 避免心跳线程持续发被拒请求刷日志、占用服务端限流计数。心跳通常比
|
||||
// claim 更高频，往往先于 claim_task 发现 token 失效。
|
||||
// 401/403：token 失效（被重发覆盖）。与 claim_task 口径统一：
|
||||
// 审查修复 #M5：原 `std::process::exit(1)` 跳过主循环优雅退出，
|
||||
// 在途长任务结果静默丢失。现置 shutting_down 让主循环停止领用新任务、
|
||||
// 等待在途任务完成（≤30s）并归档沙盒后再退出。
|
||||
if status.as_u16() == 401 || status.as_u16() == 403 {
|
||||
tracing::error!(
|
||||
"节点 {} 心跳被服务端拒绝 (HTTP {})：node token 已失效（已被重发覆盖）。\n\
|
||||
恢复方式：把管理员重发的新 token 明文写入 {} 后重启节点，\n\
|
||||
或删除该文件后重启以重新提交注册申请等待审批。\n\
|
||||
进程将退出，依赖编排系统重启。",
|
||||
已触发优雅退出：等待在途任务完成后进程退出，依赖编排系统重启。",
|
||||
hb_node_id,
|
||||
status,
|
||||
token_file_display(&hb_runtime_dir)
|
||||
);
|
||||
std::process::exit(1);
|
||||
hb_shutting_down.store(true, Ordering::Release);
|
||||
break;
|
||||
}
|
||||
// 解析心跳响应体，提取管理员配额（见 docs/dynamic_cpu_slots_design.md
|
||||
// §3.2）：计算生效配额并更新 effective_max_slots。反序列化失败不致命
|
||||
@@ -348,7 +415,18 @@ impl NodeWorker {
|
||||
}
|
||||
}
|
||||
|
||||
let shutting_down = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
// 运行期沙盒 GC：追踪在途任务 task_id 集合，后台周期清理孤儿 task_* 目录
|
||||
//（cleanup_slot_work_dir 反复失败/进程异常残留的累积治理，见 sandbox_gc_loop）。
|
||||
let active_tasks: Arc<Mutex<HashSet<String>>> = Arc::new(Mutex::new(HashSet::new()));
|
||||
tokio::spawn(sandbox_gc_loop(
|
||||
work_dir.clone(),
|
||||
active_tasks.clone(),
|
||||
self.shutting_down.clone(),
|
||||
));
|
||||
|
||||
// 复用 self.shutting_down（结构体字段）：心跳线程、claim_task、信号线程、主循环
|
||||
// 共享同一标志。任一来源触发优雅退出，主循环都能感知（审查修复 #M5）。
|
||||
let shutting_down = self.shutting_down.clone();
|
||||
let shutdown_signal = shutting_down.clone();
|
||||
|
||||
// 信号处理：同时监听 SIGINT（Ctrl+C）与 SIGTERM。
|
||||
@@ -428,6 +506,8 @@ impl NodeWorker {
|
||||
let result_dir = result_dir.clone();
|
||||
let slots_counter = self.active_slots.clone();
|
||||
let shutdown = shutting_down.clone();
|
||||
// 在途任务 task_id 集合（沙盒 GC 跳过用）：spawn 前登记、任务结束移除。
|
||||
let active_tasks = active_tasks.clone();
|
||||
|
||||
// 在 spawn 前同步自增，与容量检查紧邻成原子操作：避免
|
||||
// 「检查通过 → spawn 排队 → 回循环再检查时计数尚未自增」的竞态
|
||||
@@ -438,6 +518,8 @@ impl NodeWorker {
|
||||
let slot_guard = SlotGuard {
|
||||
counter: slots_counter.clone(),
|
||||
};
|
||||
// GC 活跃集登记（在 spawn 前，避免 GC 误删尚未建目录的在途任务）。
|
||||
active_tasks.lock().await.insert(task.task_id.to_string());
|
||||
|
||||
tokio::spawn(async move {
|
||||
// 把守卫移入 future，确保任务结束（含 panic）时归还 slot。
|
||||
@@ -509,6 +591,9 @@ impl NodeWorker {
|
||||
);
|
||||
}
|
||||
}
|
||||
// 任务结束（含 panic/清理失败）：从 GC 活跃集移除，下轮 GC 可回收
|
||||
// 清理失败残留的孤儿沙盒。
|
||||
active_tasks.lock().await.remove(&task.task_id.to_string());
|
||||
// _slot_guard 在此作用域结束时 drop，归还活动 slot 计数。
|
||||
});
|
||||
}
|
||||
@@ -581,11 +666,14 @@ impl NodeWorker {
|
||||
"领用任务被服务端拒绝 (HTTP {})：node token 已失效（已被重发覆盖）。\n\
|
||||
恢复方式：把管理员重发的新 token 明文写入 {} 后重启节点，\n\
|
||||
或删除该文件后重启以重新提交注册申请等待审批。\n\
|
||||
进程将退出，依赖编排系统重启。",
|
||||
已触发优雅退出：等待在途任务完成后进程退出，依赖编排系统重启。",
|
||||
status,
|
||||
token_file_display(&self.config.runtime_dir)
|
||||
);
|
||||
std::process::exit(1);
|
||||
// 审查修复 #M5：与心跳线程口径统一，置 shutting_down 让主循环走优雅退出，
|
||||
// 而非 exit(1) 跳过在途任务的结果上报与沙盒归档。
|
||||
self.shutting_down.store(true, Ordering::Release);
|
||||
return Ok(ClaimOutcome::Empty);
|
||||
}
|
||||
|
||||
if status.is_server_error() {
|
||||
@@ -612,3 +700,23 @@ impl NodeWorker {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// 沙盒 GC 孤儿判定（#4）：`task_{uuid}` 且 uuid 不在活跃集 → 孤儿；在途/非沙盒目录 → 否。
|
||||
#[test]
|
||||
fn orphan_gc_predicate_skips_active_and_non_sandbox() {
|
||||
let active: HashSet<String> = ["task-1111-aaaa".to_string()].into_iter().collect();
|
||||
// 在途任务 → 非孤儿（GC 跳过）。
|
||||
assert!(!is_orphan_task_dir("task_task-1111-aaaa", &active));
|
||||
// 孤儿（uuid 不在活跃集）→ 回收。
|
||||
assert!(is_orphan_task_dir("task_task-2222-bbbb", &active));
|
||||
// 非沙盒目录（.seed_cache / 其它）→ 永不动。
|
||||
assert!(!is_orphan_task_dir(".seed_cache", &active));
|
||||
assert!(!is_orphan_task_dir("result_dir", &active));
|
||||
// 空活跃集：所有 task_* 都是孤儿（等价重启清理）。
|
||||
assert!(is_orphan_task_dir("task_abc", &HashSet::new()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,16 @@ pub async fn healthz() -> Result<impl IntoResponse, crate::api::AppError> {
|
||||
pub async fn get_status(
|
||||
State(state): State<AppState>,
|
||||
) -> Result<impl IntoResponse, crate::api::AppError> {
|
||||
let nodes = state.db.get_active_nodes().await.unwrap_or_default();
|
||||
// 审查修复 #N6：原 unwrap_or_default 吞掉 DB 错误，前端显示「0 节点在线」掩盖真实故障。
|
||||
// 现至少记 warn，让运维能在日志里定位「dashboard 为何显示无节点」。仍降级为空列表
|
||||
// （返回 500 会让 dashboard 整个 status 面板不可用，空列表是更安全的退化）。
|
||||
let nodes = match state.db.get_active_nodes().await {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
tracing::warn!("get_status 查询活跃节点失败，降级为空列表: {}", e);
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
let total_active_slots: i32 = nodes.iter().map(|n| n.active_slots).sum();
|
||||
let total_max_slots: i32 = nodes.iter().map(|n| n.max_slots).sum();
|
||||
|
||||
@@ -24,8 +33,8 @@ pub async fn get_status(
|
||||
.get_grid_summary_stats(None)
|
||||
.await
|
||||
.unwrap_or(serde_json::json!({
|
||||
"total": 0, "pending": 0, "queued": 0, "running": 0, "converged": 0, "failed": 0,
|
||||
"cold_run_converged": 0, "seed_step_converged": 0
|
||||
"total": 0, "pending": 0, "queued": 0, "running": 0, "completed": 0, "failed": 0,
|
||||
"cold_run_converged": 0, "seed_step_converged": 0, "synspec_converged": 0
|
||||
}));
|
||||
|
||||
Ok(Json(json!({
|
||||
|
||||
@@ -46,7 +46,11 @@ pub async fn claim_task(
|
||||
// 旧版在途任务 payload 无 workflow_name（None）→ 归一到主库迁移回填的
|
||||
// '__legacy__' 标记，使 mark_grid_point_running 能命中 legacy 网格点（H1 修复）。
|
||||
let wf = crate::db::normalize_workflow_name(task.workflow_name.as_deref());
|
||||
match state.db.mark_grid_point_running(&task.point_name, &wf).await {
|
||||
match state
|
||||
.db
|
||||
.mark_grid_point_running(&task.point_name, &wf)
|
||||
.await
|
||||
{
|
||||
Ok(false) => {
|
||||
info!(
|
||||
"领用任务 {}（网格点 {}）时点已非 pending/queued 态，跳过 running 标记（迟到/重复领用）",
|
||||
@@ -269,7 +273,7 @@ pub async fn report_task(
|
||||
}
|
||||
|
||||
// Save seed file .7 using atomic temporary writing strategy
|
||||
if report.converged && !report.atmosphere_has_nan {
|
||||
if report.result_valid && !report.atmosphere_has_nan {
|
||||
if let Some(bytes) = seed_file_data {
|
||||
let seed_tmp =
|
||||
model_dir.join(format!("{}.7.{}.tmp", name, uuid::Uuid::new_v4().simple()));
|
||||
@@ -298,7 +302,7 @@ pub async fn report_task(
|
||||
}
|
||||
|
||||
if state_changed
|
||||
&& (!report.converged
|
||||
&& (!report.result_valid
|
||||
|| report.atmosphere_has_nan
|
||||
|| report.status == TaskStatus::Failed
|
||||
|| report.status == TaskStatus::Timeout)
|
||||
@@ -369,8 +373,9 @@ pub async fn import_seed(
|
||||
) -> Result<impl IntoResponse, crate::api::AppError> {
|
||||
let mut summary_json: Option<String> = None;
|
||||
let mut seed_file_data: Option<Vec<u8>> = None;
|
||||
// 收敛途径（cold_run/seed_step）：由 import_results 工具依据旧 conv.json 的 stages 是否
|
||||
// 大气收敛途径（cold_run/seed_step）：由 import_results 工具依据旧 conv.json 的 stages 是否
|
||||
// 含 seed_nc 判定后透传。缺失或非法时兜底 cold_run（容错旧版工具 / 防注入）。
|
||||
// 语义为 TLUSTY 阶段策略，写入 grid_points.tlusty_success_method。
|
||||
let mut success_method: Option<String> = None;
|
||||
let mut multipart_error = false;
|
||||
|
||||
@@ -400,13 +405,13 @@ pub async fn import_seed(
|
||||
multipart_error = true;
|
||||
}
|
||||
}
|
||||
} else if field_name == "success_method" {
|
||||
} else if field_name == "tlusty_success_method" {
|
||||
match field.text().await {
|
||||
Ok(text) => {
|
||||
success_method = Some(text);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("历史种子导入：读取 success_method 字段失败: {}", e);
|
||||
warn!("历史种子导入：读取 tlusty_success_method 字段失败: {}", e);
|
||||
multipart_error = true;
|
||||
}
|
||||
}
|
||||
@@ -463,7 +468,7 @@ pub async fn import_seed(
|
||||
|
||||
let workflow_name = query.workflow;
|
||||
let params = summary.params.clone();
|
||||
let converged = summary.converged && !summary.atmosphere_has_nan;
|
||||
let converged = summary.result_valid && !summary.atmosphere_has_nan;
|
||||
let max_relc = summary.final_max_relc;
|
||||
|
||||
// 1. 幂等写入 grid_points（ON CONFLICT DO NOTHING）：无需事先 start 工作流。
|
||||
@@ -524,7 +529,7 @@ pub async fn import_seed(
|
||||
}
|
||||
}
|
||||
|
||||
// 4. 更新 grid_points 状态：收敛→converged(success_method=工具判定的途径)；否则维持 pending
|
||||
// 4. 更新 grid_points 状态：收敛→converged(大气归因 tlusty_success_method=工具判定的途径)；否则维持 pending
|
||||
// 让正常调度处理（导入未收敛点无意义，但记录其尝试）。
|
||||
// 途径缺失或非法时兜底 cold_run（容错旧版工具 / 防注入），由 db 层再次白名单校验。
|
||||
if converged {
|
||||
@@ -556,6 +561,8 @@ pub async fn import_seed(
|
||||
Json(json!({
|
||||
"status": "ok",
|
||||
"point_name": name,
|
||||
// 注：此 "converged" 键是大气收敛标志（由 summary.result_valid 派生），与 grid_points.status
|
||||
// 的 completed 重命名无关，勿改（改键名会破坏历史种子导入客户端）。
|
||||
"converged": converged,
|
||||
"max_relc": max_relc,
|
||||
})),
|
||||
|
||||
@@ -426,8 +426,8 @@ pub async fn get_workflow_progress(
|
||||
}
|
||||
};
|
||||
|
||||
let rate_per_hour = avg_per_hour(|p| p.converged);
|
||||
let done_rate_per_hour = avg_per_hour(|p| p.converged + p.failed);
|
||||
let rate_per_hour = avg_per_hour(|p| p.completed);
|
||||
let done_rate_per_hour = avg_per_hour(|p| p.completed + p.failed);
|
||||
|
||||
// 速率统计的实际时间跨度（近 2h 子窗口首末间隔，≤2h；回退整窗时为整窗跨度）。
|
||||
let rate_span_hours: Option<f64> = {
|
||||
@@ -448,8 +448,8 @@ pub async fn get_workflow_progress(
|
||||
let stalled_minutes: Option<f64> = if series.len() >= 2 {
|
||||
let mut last_progress_idx = None;
|
||||
for i in 1..series.len() {
|
||||
let prev = series[i - 1].converged + series[i - 1].failed;
|
||||
let cur = series[i].converged + series[i].failed;
|
||||
let prev = series[i - 1].completed + series[i - 1].failed;
|
||||
let cur = series[i].completed + series[i].failed;
|
||||
if cur > prev {
|
||||
last_progress_idx = Some(i);
|
||||
}
|
||||
@@ -529,9 +529,11 @@ pub async fn get_workflow_points(
|
||||
}
|
||||
|
||||
if let Some(s) = &pq.status {
|
||||
// "converged" 保留作旧客户端兼容值（7c 由 converged 改名 completed），
|
||||
// 通过后于下方归一化为 "completed" 再绑定 SQL。
|
||||
if !matches!(
|
||||
s.as_str(),
|
||||
"pending" | "queued" | "running" | "converged" | "failed"
|
||||
"pending" | "queued" | "running" | "completed" | "converged" | "failed"
|
||||
) {
|
||||
return Err(crate::api::AppError::BadRequest(format!(
|
||||
"非法的 status 参数: {}",
|
||||
@@ -540,7 +542,10 @@ pub async fn get_workflow_points(
|
||||
}
|
||||
}
|
||||
if let Some(m) = &pq.method {
|
||||
if !matches!(m.as_str(), "cold_run" | "seed_step") {
|
||||
// 值域：TLUSTY 阶段策略名（cold_run/seed_step，映射 tlusty_success_method）∪
|
||||
// sentinel "synspec_only"（光谱专用点，映射 tlusty IS NULL AND synspec IS NOT NULL）。
|
||||
// 前缀化避免与潜在的同名 TLUSTY 策略碰撞（P9/E 语义拆分）。
|
||||
if !matches!(m.as_str(), "cold_run" | "seed_step" | "synspec_only") {
|
||||
return Err(crate::api::AppError::BadRequest(format!(
|
||||
"非法的 method 参数: {}",
|
||||
m
|
||||
@@ -560,21 +565,16 @@ pub async fn get_workflow_points(
|
||||
} else {
|
||||
"ASC"
|
||||
};
|
||||
// P3 窗口重写后，ORDER BY 作用于外层子查询（列名裸露，无 gp./t. 前缀）；
|
||||
// 耗时排序引用内层 COALESCE 的输出别名 eff_elapsed。列名均为编译期白名单。
|
||||
let order_by = match sort {
|
||||
"wave" => format!("gp.wave {dir}, gp.cno_sum ASC, gp.teff ASC"),
|
||||
"teff" => format!("gp.teff {dir}, gp.wave ASC, gp.cno_sum ASC"),
|
||||
"max_relc" => format!("t.max_relc IS NULL ASC, t.max_relc {dir}, gp.wave ASC"),
|
||||
"attempts" => format!("gp.attempt_count {dir}, gp.wave ASC, gp.cno_sum ASC"),
|
||||
// 耗时取最近一次尝试的真实墙钟（与列表展示同口径 COALESCE），NULL（从未派发）靠后。
|
||||
"elapsed" => {
|
||||
format!(
|
||||
"COALESCE(t.elapsed_sec, gp.last_elapsed_sec) IS NULL ASC, \
|
||||
COALESCE(t.elapsed_sec, gp.last_elapsed_sec) {dir}, gp.wave ASC"
|
||||
)
|
||||
}
|
||||
"last_completed_at" => {
|
||||
format!("t.completed_at IS NULL ASC, t.completed_at {dir}, gp.wave ASC")
|
||||
}
|
||||
"wave" => format!("wave {dir}, cno_sum ASC, teff ASC"),
|
||||
"teff" => format!("teff {dir}, wave ASC, cno_sum ASC"),
|
||||
"max_relc" => format!("max_relc IS NULL ASC, max_relc {dir}, wave ASC"),
|
||||
"attempts" => format!("attempt_count {dir}, wave ASC, cno_sum ASC"),
|
||||
// 耗时取最近一次尝试的真实墙钟（与列表展示同口径 eff_elapsed），NULL（从未派发）靠后。
|
||||
"elapsed" => format!("eff_elapsed IS NULL ASC, eff_elapsed {dir}, wave ASC"),
|
||||
"last_completed_at" => format!("completed_at IS NULL ASC, completed_at {dir}, wave ASC"),
|
||||
_ => {
|
||||
return Err(crate::api::AppError::BadRequest(format!(
|
||||
"非法的 sort 参数: {}",
|
||||
@@ -584,7 +584,14 @@ pub async fn get_workflow_points(
|
||||
};
|
||||
|
||||
let filter = crate::db::PointFilter {
|
||||
status: pq.status.clone(),
|
||||
// 7c 改名：旧值 "converged" 归一化为 "completed"（DB 实际存储值）。
|
||||
status: pq.status.as_deref().map(|s| {
|
||||
if s == "converged" {
|
||||
"completed".to_string()
|
||||
} else {
|
||||
s.to_string()
|
||||
}
|
||||
}),
|
||||
method: pq.method.clone(),
|
||||
wave: pq.wave,
|
||||
q: pq.q.clone(),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,937 @@
|
||||
//! 网格点（从原 db.rs 拆分，2026-08-06）。
|
||||
//! `impl Database` 的 网格点 域方法。共享基础设施（Database struct、连接管理、类型、helper）见父模块 `super`（crate::db）。
|
||||
use super::*;
|
||||
|
||||
impl Database {
|
||||
// --- Grid Point & Task operations ---
|
||||
pub async fn upsert_grid_point(
|
||||
&self,
|
||||
params_in: &GridPointParams,
|
||||
wave: i32,
|
||||
workflow_name: &str,
|
||||
) -> Result<()> {
|
||||
let name = params_in.model_name();
|
||||
self.upsert_grid_point_named(&name, params_in, wave, workflow_name)
|
||||
.await
|
||||
}
|
||||
|
||||
/// 与 `upsert_grid_point` 相同，但使用调用方提供的权威 `name`（而非从 params 重推）。
|
||||
///
|
||||
/// 历史种子导入专用：旧版 conv.json 的 `name` 是源精度真名（如 `t20000_g5.0_...`），
|
||||
/// 而导入路径的 params 来自旧 JSON（数值，无源文本，`model_name()` 会失真），故必须
|
||||
/// 显式传入旧名以保证 DB `name` 列与旧数据逐字符一致。
|
||||
pub async fn upsert_grid_point_named(
|
||||
&self,
|
||||
name: &str,
|
||||
params_in: &GridPointParams,
|
||||
wave: i32,
|
||||
workflow_name: &str,
|
||||
) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let p = params_in.clone();
|
||||
let name = name.to_string();
|
||||
let cno_sum = p.cno_sum();
|
||||
let wf = workflow_name.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"INSERT INTO grid_points (name, workflow_name, teff, logg, loghe, logc, logn, logo, cno_sum, wave)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
|
||||
ON CONFLICT(workflow_name, name) DO NOTHING",
|
||||
params![name, wf, p.teff.value(), p.logg.value(), p.loghe.value(), p.logc.value(), p.logn.value(), p.logo.value(), cno_sum, wave],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 计算某网格点的难度波次（wave）：该工作流内所有 `cno_sum` 严格小于本点（1e-5 容差）
|
||||
/// 的去重值个数，即 `cno_sum` 升序中的桶序号——与 `initialize_grid` 的波次分组口径一致
|
||||
/// （`scheduler.rs` 沿排序序扫描，cno_sum 变化 >1e-5 则 wave+1）。
|
||||
///
|
||||
/// 历史种子导入（`import_seed`）用它替代此前硬编码的 `wave=0`：导入点按现有规则归入
|
||||
/// 正确波次，前端难度波次推进不再把全部导入点挤在第一波。对已完全填充的工作流（所有
|
||||
/// cno_sum 等级都在库内），任意导入顺序的 rank 都正确；对全新工作流，配合 import_results
|
||||
/// 工具按 cno_sum 升序导入即可保证最终波次一致。
|
||||
pub async fn compute_wave_for_cno_sum(&self, workflow_name: &str, cno_sum: f64) -> Result<i32> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = workflow_name.to_string();
|
||||
let cno = cno_sum;
|
||||
let wave = tokio::task::spawn_blocking(move || -> Result<i32> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let n: i64 = conn.query_row(
|
||||
"SELECT COUNT(DISTINCT cno_sum) FROM grid_points WHERE workflow_name = ?1 AND cno_sum < ?2 - 1e-5",
|
||||
params![wf, cno],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
Ok(n as i32)
|
||||
})
|
||||
.await??;
|
||||
Ok(wave)
|
||||
}
|
||||
|
||||
/// 仅用于测试/诊断：取指定工作流的全部 pending 点（无 LIMIT）。生产调度走 _limit 版本。
|
||||
pub async fn get_pending_grid_points(
|
||||
&self,
|
||||
workflow_name: &str,
|
||||
) -> Result<Vec<(String, GridPointParams, i32)>> {
|
||||
self.get_pending_grid_points_limit(usize::MAX, workflow_name)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn get_pending_grid_points_limit(
|
||||
&self,
|
||||
limit: usize,
|
||||
workflow_name: &str,
|
||||
) -> Result<Vec<(String, GridPointParams, i32)>> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = workflow_name.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<(String, GridPointParams, i32)>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT name, teff, logg, loghe, logc, logn, logo, wave FROM grid_points
|
||||
WHERE status = 'pending' AND workflow_name = ?1
|
||||
ORDER BY wave ASC, cno_sum ASC, teff ASC LIMIT ?2",
|
||||
)?;
|
||||
|
||||
let limit_param = if limit == usize::MAX {
|
||||
-1i64
|
||||
} else {
|
||||
limit as i64
|
||||
};
|
||||
let rows_iter = stmt.query_map(params![wf, limit_param], |r| {
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
GridPointParams {
|
||||
teff: GridAxisValue::from_value(r.get::<_, f64>(1)?),
|
||||
logg: GridAxisValue::from_value(r.get::<_, f64>(2)?),
|
||||
loghe: GridAxisValue::from_value(r.get::<_, f64>(3)?),
|
||||
logc: GridAxisValue::from_value(r.get::<_, f64>(4)?),
|
||||
logn: GridAxisValue::from_value(r.get::<_, f64>(5)?),
|
||||
logo: GridAxisValue::from_value(r.get::<_, f64>(6)?),
|
||||
},
|
||||
r.get::<_, i32>(7)?,
|
||||
))
|
||||
})?;
|
||||
|
||||
let mut list = Vec::new();
|
||||
for r in rows_iter {
|
||||
list.push(r?);
|
||||
}
|
||||
Ok(list)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 原子选点：在 IMMEDIATE 事务内将 pending 点标记为 queued 并返回。
|
||||
///
|
||||
/// 解决 `get_pending_grid_points_limit`（SELECT）与 `update_grid_status`（UPDATE）
|
||||
/// 分离导致的 TOCTOU 竞态：两个并发调度调用可能 SELECT 到同一批 pending 点，
|
||||
/// 各自创建任务，产生重复派发（#5 修复）。
|
||||
///
|
||||
/// 与 `pop_task`（sqlite_queue.rs）和 `take_pending_node_token` 同口径：
|
||||
/// IMMEDIATE 事务在 BEGIN 时即获取写锁，SELECT 与 UPDATE 之间不会被其它
|
||||
/// 调用方插入，从而只有一个调用方能 claiming 到某批点。
|
||||
pub async fn claim_pending_grid_points(
|
||||
&self,
|
||||
limit: usize,
|
||||
workflow_name: &str,
|
||||
) -> Result<Vec<(String, GridPointParams, i32)>> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = workflow_name.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<(String, GridPointParams, i32)>> {
|
||||
let mut conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
|
||||
|
||||
let limit_param = if limit == usize::MAX {
|
||||
-1i64
|
||||
} else {
|
||||
limit as i64
|
||||
};
|
||||
|
||||
// P3 附带优化（§5.3）：先按调度优先级 SELECT 有序取行（ORDER BY wave/cno_sum/teff），
|
||||
// 再在同一 IMMEDIATE 事务内按 rowid 原子标记 queued——UPDATE...RETURNING 不保序，
|
||||
// 旧实现依赖 Rust 侧 sort_by 重排；改为 SELECT 排序贯穿后删除 Rust 排序。
|
||||
let mut select_stmt = tx.prepare(
|
||||
"SELECT rowid, name, teff, logg, loghe, logc, logn, logo, wave
|
||||
FROM grid_points
|
||||
WHERE status = 'pending' AND workflow_name = ?1
|
||||
ORDER BY wave ASC, cno_sum ASC, teff ASC
|
||||
LIMIT ?2",
|
||||
)?;
|
||||
|
||||
let mut list = Vec::new();
|
||||
let mut ids = Vec::new();
|
||||
{
|
||||
let rows = select_stmt.query_map(params![wf, limit_param], |r| {
|
||||
Ok((
|
||||
r.get::<_, i64>(0)?,
|
||||
(
|
||||
r.get::<_, String>(1)?,
|
||||
GridPointParams {
|
||||
teff: GridAxisValue::from_value(r.get::<_, f64>(2)?),
|
||||
logg: GridAxisValue::from_value(r.get::<_, f64>(3)?),
|
||||
loghe: GridAxisValue::from_value(r.get::<_, f64>(4)?),
|
||||
logc: GridAxisValue::from_value(r.get::<_, f64>(5)?),
|
||||
logn: GridAxisValue::from_value(r.get::<_, f64>(6)?),
|
||||
logo: GridAxisValue::from_value(r.get::<_, f64>(7)?),
|
||||
},
|
||||
r.get::<_, i32>(8)?,
|
||||
),
|
||||
))
|
||||
})?;
|
||||
for r in rows {
|
||||
let (id, item) = r?;
|
||||
ids.push(id);
|
||||
list.push(item);
|
||||
}
|
||||
}
|
||||
drop(select_stmt);
|
||||
|
||||
// 同事务原子标记 queued（IMMEDIATE 已持有写锁，SELECT→UPDATE 间无竞态窗口）。
|
||||
// 阶段列（5b）同步：tlusty_status 守卫保留既有终态（半失败重试时 tlusty_status 已
|
||||
// converged 不被覆盖）；synspec_status 自由流转为 queued（重试进行中可见——设计
|
||||
// §7.3 打开项 #2 "仅 synspec 侧流转"，审查修正：原实现连 synspec 'failed' 也保留，
|
||||
// 与设计意图不符）。
|
||||
if !ids.is_empty() {
|
||||
// 审查修复 #M6：SQLite 单语句参数上限 999，limit=usize::MAX（映射为 LIMIT -1）
|
||||
// 且 pending 点极多时会撑爆 IN(?,?,...)。按 500 一批分次 UPDATE（与
|
||||
// delete_tasks_by_ids 同口径），每批独立语句、同一事务，原子性与原实现等价。
|
||||
for chunk in ids.chunks(500) {
|
||||
let placeholders = vec!["?"; chunk.len()].join(", ");
|
||||
let sql = format!(
|
||||
"UPDATE grid_points SET status = 'queued',
|
||||
tlusty_status = CASE WHEN tlusty_status NOT IN ('converged','failed') THEN 'queued' ELSE tlusty_status END,
|
||||
synspec_status = 'queued'
|
||||
WHERE rowid IN ({placeholders})"
|
||||
);
|
||||
tx.execute(&sql, rusqlite::params_from_iter(chunk.iter()))?;
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
|
||||
Ok(list)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 重置指定工作流的 queued 点为 pending（系统重启/工作流启动时使用）。
|
||||
/// 按 workflow 隔离，避免误伤其他工作流（多工作流分区修复点）。
|
||||
pub async fn reset_queued_grid_points_to_pending(&self, workflow_name: &str) -> Result<usize> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<usize> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 只重置 queued 状态的任务为 pending。对于 running (正由 Worker 处理的项目)，不可在系统重启或初始化时粗暴清零，让 Worker 正常完成汇报或触发心跳/超时自动逐回
|
||||
let rows = conn.execute(
|
||||
"UPDATE grid_points SET status = 'pending' WHERE status = 'queued' AND workflow_name = ?1",
|
||||
params![wf],
|
||||
)?;
|
||||
Ok(rows)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// ForceRecompute 策略的终态重置（见 docs/task_engine_decoupling_design.md §2.1）：
|
||||
/// 把工作流内**已收敛 / 已失败**的终态点全部打回 pending，使调度器无视历史状态与
|
||||
/// 产物强制重算。在 workflow start 时由 initialize_grid 据策略调用。
|
||||
///
|
||||
/// 不触碰 queued/running（在途任务正常结算）与 pending（本就待派发）。
|
||||
/// 按 workflow 隔离，避免跨工作流误改同名点（多工作流分区修复点）。
|
||||
pub async fn reset_terminal_points_for_recompute(&self, workflow_name: &str) -> Result<usize> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<usize> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let rows = conn.execute(
|
||||
"UPDATE grid_points SET status = 'pending'
|
||||
WHERE workflow_name = ?1 AND status IN ('completed', 'failed')",
|
||||
params![wf],
|
||||
)?;
|
||||
Ok(rows)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// SkipConverged 策略的启动时重置（2026-08-04 语义修正）：仅把**已失败**的点打回
|
||||
/// pending 重试，**已收敛**点保留（增量语义）。与 ForceRecompute（收敛+失败全量
|
||||
/// 重置）和 SkipFailed（收敛/失败都保留）区别开——"跳过收敛、重试失败"是默认策略
|
||||
/// 应有的行为，此前 SkipConverged 与 SkipFailed 在启动时行为相同，无法表达它。
|
||||
///
|
||||
/// 不触碰 queued/running（在途任务正常结算）与 pending（本就待派发）。
|
||||
/// 按 workflow 隔离，避免跨工作流误改同名点。
|
||||
pub async fn reset_failed_points_for_retry(&self, workflow_name: &str) -> Result<usize> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<usize> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let rows = conn.execute(
|
||||
"UPDATE grid_points SET status = 'pending'
|
||||
WHERE workflow_name = ?1 AND status = 'failed'",
|
||||
params![wf],
|
||||
)?;
|
||||
Ok(rows)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 重置指定工作流内一批点（按 name）为 pending。
|
||||
/// 按 workflow 隔离，避免跨工作流误改同名点（多工作流分区修复点）。
|
||||
pub async fn reset_specific_grid_points_to_pending(
|
||||
&self,
|
||||
names: &[String],
|
||||
workflow_name: &str,
|
||||
) -> Result<usize> {
|
||||
if names.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let pool = self.pool.clone();
|
||||
let names_owned = names.to_vec();
|
||||
let wf = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<usize> {
|
||||
let mut conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let tx = conn.transaction()?;
|
||||
let mut count = 0;
|
||||
for name in &names_owned {
|
||||
count += tx.execute(
|
||||
"UPDATE grid_points SET status = 'pending' WHERE name = ?1 AND workflow_name = ?2 AND status IN ('queued', 'running')",
|
||||
params![name, wf],
|
||||
)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(count)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 找出疑似孤儿的网格点及其 stale pending 任务行（#6 修复兜底，2026-08-02 重构）。
|
||||
///
|
||||
/// 候选条件：网格点处于 `running`/`queued` 态，且 `tasks` 表中存在该点创建时间
|
||||
/// 老于 stale_sec 的 `pending` 行。候选**不等于**孤儿——老 pending 行可能是正常
|
||||
/// 在途任务（requeue_stale_tasks 按同 task_id 重投不更新 tasks.created_at，长任务
|
||||
/// 会天然变"老"）。调用方（GridScheduler::reclaim_orphaned_points）必须对每个
|
||||
/// task_id 做 MQ 活性交叉校验（task_row_exists）：队列行仍 pending/claimed 即真
|
||||
/// 在途，放行；全部无队列行才是凭证丢失的真孤儿。
|
||||
///
|
||||
/// 历史教训（2026-08-02 涡旋事故）：旧实现 reset_orphaned_running_points 仅凭
|
||||
/// "存在老 pending 行"即重置点，而 stop/重启遗留的僵尸 pending 行使该判据对每个
|
||||
/// 在跑的点恒真 → 每 30s 巡检重置 + 同轮再派发 → 单点被重复计算上百次。
|
||||
///
|
||||
/// 返回 (name, workflow_name, task_id) 行列表，由调用方按点分组。
|
||||
pub async fn find_stale_pending_points(
|
||||
&self,
|
||||
stale_sec: u64,
|
||||
) -> Result<Vec<(String, String, String)>> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<(String, String, String)>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let stale_offset = format!("-{} seconds", stale_sec);
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT gp.name, gp.workflow_name, t.task_id
|
||||
FROM grid_points gp
|
||||
JOIN tasks t
|
||||
ON t.point_name = gp.name
|
||||
AND t.workflow_name = gp.workflow_name
|
||||
WHERE gp.status IN ('running', 'queued')
|
||||
AND t.status = 'pending'
|
||||
AND t.created_at < datetime('now', ?1)",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![stale_offset], |r| {
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
r.get::<_, String>(1)?,
|
||||
r.get::<_, String>(2)?,
|
||||
))
|
||||
})?;
|
||||
let mut out = Vec::new();
|
||||
for r in rows {
|
||||
out.push(r?);
|
||||
}
|
||||
Ok(out)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 把孤儿网格点救回 `pending`（仅当点仍处于 running/queued 时生效）。
|
||||
///
|
||||
/// 与 `find_stale_pending_points` + MQ 活性校验 + `delete_tasks_by_ids` 组合使用：
|
||||
/// 调用方确认该点所有 stale pending 任务行均无队列凭证（真孤儿）并清除这些行后，
|
||||
/// 调本方法让调度器重新派发。WHERE 的 status 条件是并发防护：若校验期间恰好有
|
||||
/// 迟到上报把点置为终态（converged/failed），本 UPDATE 命中 0 行，不覆盖终态。
|
||||
///
|
||||
/// 返回是否实际重置（false = 点已不在 running/queued，无需处理）。
|
||||
pub async fn rescue_orphaned_point(&self, name: &str, workflow_name: &str) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
let changed = tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let n = conn.execute(
|
||||
"UPDATE grid_points SET status = 'pending'
|
||||
WHERE name = ?1 AND workflow_name = ?2 AND status IN ('running', 'queued')",
|
||||
params![name_owned, wf],
|
||||
)?;
|
||||
Ok(n > 0)
|
||||
})
|
||||
.await??;
|
||||
Ok(changed)
|
||||
}
|
||||
|
||||
/// 更新指定工作流内某点的状态。按 workflow 隔离，防跨工作流误改同名点。
|
||||
pub async fn update_grid_status(
|
||||
&self,
|
||||
name: &str,
|
||||
status: GridPointStatus,
|
||||
workflow_name: &str,
|
||||
) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let status_str = status.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"UPDATE grid_points SET status = ?1 WHERE name = ?2 AND workflow_name = ?3",
|
||||
params![status_str, name_owned, wf],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// H1 活锁修复辅助：记录网格点当前「剩余策略链」（JSON 数组）。运行时回退
|
||||
/// （trigger_strategy_fallback）把点打回 pending 等种子时调用，标记该点已失败过部分
|
||||
/// 策略（如 cold_run），供调度路径重派时用剩余链而非完整 YAML 链，避免重跑已失败策略。
|
||||
pub async fn set_pending_strategies(
|
||||
&self,
|
||||
name: &str,
|
||||
workflow_name: &str,
|
||||
strategies_json: &str,
|
||||
) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
let s = strategies_json.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"UPDATE grid_points SET pending_strategies = ?1 WHERE name = ?2 AND workflow_name = ?3",
|
||||
params![s, name_owned, wf],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 读取网格点的剩余策略链标记（JSON 数组字符串）。无标记返回 None。
|
||||
pub async fn get_pending_strategies(
|
||||
&self,
|
||||
name: &str,
|
||||
workflow_name: &str,
|
||||
) -> Result<Option<String>> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
let v = tokio::task::spawn_blocking(move || -> Result<Option<String>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT pending_strategies FROM grid_points WHERE name = ?1 AND workflow_name = ?2",
|
||||
)?;
|
||||
let v = stmt
|
||||
.query_row(params![name_owned, wf], |r| r.get::<_, Option<String>>(0))
|
||||
.ok()
|
||||
.flatten();
|
||||
Ok(v)
|
||||
})
|
||||
.await??;
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// 清除网格点的剩余策略链标记（调度路径消费后调用）。
|
||||
pub async fn clear_pending_strategies(&self, name: &str, workflow_name: &str) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"UPDATE grid_points SET pending_strategies = NULL WHERE name = ?1 AND workflow_name = ?2",
|
||||
params![name_owned, wf],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 领用任务时把网格点标记为 running（2026-08-02 涡旋事故修复：自带终态守卫）。
|
||||
///
|
||||
/// 仅允许 `pending`/`queued` → `running`：迟到/重复领用（对应点已被上报置为
|
||||
/// converged/failed）不得复活终态点。源态集含 `pending` 是为兼容 requeue 路径
|
||||
/// （requeue_stale_tasks 把点重置为 pending 后，节点可能先于调度器领用重投行）。
|
||||
///
|
||||
/// 不委托 `update_grid_status`：其本体须保持无守卫——种子回退的 failed→Queued
|
||||
/// 复活（scheduler.rs）依赖它覆盖终态。
|
||||
///
|
||||
/// 返回是否实际变更（false = 点已在 running 或终态，本次领用属迟到/重复）。
|
||||
pub async fn mark_grid_point_running(&self, name: &str, workflow_name: &str) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
let changed = tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 阶段列（5b）同步置 running：tlusty_status 守卫保留终态（同 claim），synspec 自由流转。
|
||||
let n = conn.execute(
|
||||
"UPDATE grid_points SET status = 'running',
|
||||
tlusty_status = CASE WHEN tlusty_status NOT IN ('converged','failed') THEN 'running' ELSE tlusty_status END,
|
||||
synspec_status = 'running'
|
||||
WHERE name = ?1 AND workflow_name = ?2 AND status IN ('pending', 'queued')",
|
||||
params![name_owned, wf],
|
||||
)?;
|
||||
Ok(n > 0)
|
||||
})
|
||||
.await??;
|
||||
Ok(changed)
|
||||
}
|
||||
|
||||
/// 历史种子导入专用：把网格点标记为 converged 并记录大气收敛途径 `tlusty_success_method`。
|
||||
///
|
||||
/// 与正常 `record_task_report` 路径的区别：导入不走 task 队列，无 task_type 可取，
|
||||
/// 故由导入工具（import_results）依据旧 conv.json 的 stages 是否含 seed_nc 判定该点
|
||||
/// 当初是冷启动收敛（cold_run）还是种子步进收敛（seed_step），经 multipart 字段透传至此。
|
||||
/// 导入点因此融入冷启动/种子步进统计，而非独立为 imported 分类。
|
||||
///
|
||||
/// - `success_method`：须为 "cold_run" 或 "seed_step"，非法值兜底为 "cold_run"（防注入）。
|
||||
/// 语义为 TLUSTY 阶段策略，写入 `tlusty_success_method` 列。
|
||||
/// - `elapsed_sec`：旧版 conv.json 的单点墙钟耗时（`summary.elapsed_sec`），落入
|
||||
/// `last_elapsed_sec` 列使迁移点在详情页/点表保留真实耗时；无此数据传 None。
|
||||
pub async fn mark_grid_point_imported(
|
||||
&self,
|
||||
name: &str,
|
||||
workflow_name: &str,
|
||||
elapsed_sec: Option<f64>,
|
||||
success_method: &str,
|
||||
) -> Result<()> {
|
||||
// 白名单校验：仅接受两种合法途径，非法值兜底 cold_run（避免拼接 SQL 注入风险）。
|
||||
let method = match success_method {
|
||||
"seed_step" => "seed_step",
|
||||
_ => "cold_run",
|
||||
};
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 阶段列（5b）：导入点是历史 TLUSTY 大气收敛 → tlusty_status='converged'，
|
||||
// synspec_status 保持 NULL（历史结果仅大气，光谱未运行/未记录）。与正常结算路径的
|
||||
// 状态一致性（审查 #2 修正：原实现不设阶段列，导入点与正常点状态口径不一致）。
|
||||
conn.execute(
|
||||
"UPDATE grid_points SET status = 'completed', tlusty_success_method = ?1, last_elapsed_sec = ?2, tlusty_status = 'converged' \
|
||||
WHERE name = ?3 AND workflow_name = ?4",
|
||||
params![method, elapsed_sec, name_owned, wf],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_grid_point_status(
|
||||
&self,
|
||||
name: &str,
|
||||
workflow_name: &str,
|
||||
) -> Result<Option<(String, i32)>> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<Option<(String, i32)>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare("SELECT status, attempt_count FROM grid_points WHERE name = ?1 AND workflow_name = ?2")?;
|
||||
let res = stmt.query_row(params![name_owned, wf], |r| Ok((r.get(0)?, r.get(1)?)));
|
||||
match res {
|
||||
Ok(tuple) => Ok(Some(tuple)),
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 网格汇总统计。
|
||||
///
|
||||
/// `workflow_filter`：
|
||||
/// - `None`：聚合全部工作流的 grid_points（dashboard 全局概览用）。
|
||||
/// - `Some(wf)`：仅聚合指定工作流（按工作流隔离的进度统计）。
|
||||
///
|
||||
/// 口径说明：`pending` 与 `queued` **分开**计数（详情页需要区分"未入队"与"排队中"）；
|
||||
/// 旧版前端若需合并口径，自行相加（见 dashboard state.js）。导入的历史点按其实际
|
||||
/// 收敛途径（cold_run/seed_step）归类，与正常计算点一并统计——不再有独立 imported 分类。
|
||||
pub async fn get_grid_summary_stats(
|
||||
&self,
|
||||
workflow_filter: Option<&str>,
|
||||
) -> Result<serde_json::Value> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = workflow_filter.map(|s| s.to_string());
|
||||
tokio::task::spawn_blocking(move || -> Result<serde_json::Value> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 合并原先 7 条独立 COUNT 查询为单次扫描，用 SUM(CASE WHEN ...) 一次性聚合所有口径，
|
||||
// 显著降低 status API 的数据库往返与锁竞争开销。
|
||||
let row = match &wf {
|
||||
Some(name) => conn.query_row(
|
||||
"SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) AS pending,
|
||||
SUM(CASE WHEN status = 'queued' THEN 1 ELSE 0 END) AS queued,
|
||||
SUM(CASE WHEN status = 'running' THEN 1 ELSE 0 END) AS running,
|
||||
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) AS completed,
|
||||
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed,
|
||||
SUM(CASE WHEN status = 'completed' AND tlusty_success_method = 'cold_run' THEN 1 ELSE 0 END) AS cold_run_converged,
|
||||
SUM(CASE WHEN status = 'completed' AND tlusty_success_method = 'seed_step' THEN 1 ELSE 0 END) AS seed_step_converged,
|
||||
SUM(CASE WHEN status = 'completed' AND synspec_success_method IS NOT NULL THEN 1 ELSE 0 END) AS synspec_converged
|
||||
FROM grid_points WHERE workflow_name = ?1",
|
||||
params![name],
|
||||
|r| {
|
||||
let n = |i: usize| -> i64 { r.get::<_, Option<i64>>(i).unwrap_or(None).unwrap_or(0) };
|
||||
Ok((n(0), n(1), n(2), n(3), n(4), n(5), n(6), n(7), n(8)))
|
||||
},
|
||||
),
|
||||
None => conn.query_row(
|
||||
"SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) AS pending,
|
||||
SUM(CASE WHEN status = 'queued' THEN 1 ELSE 0 END) AS queued,
|
||||
SUM(CASE WHEN status = 'running' THEN 1 ELSE 0 END) AS running,
|
||||
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) AS completed,
|
||||
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed,
|
||||
SUM(CASE WHEN status = 'completed' AND tlusty_success_method = 'cold_run' THEN 1 ELSE 0 END) AS cold_run_converged,
|
||||
SUM(CASE WHEN status = 'completed' AND tlusty_success_method = 'seed_step' THEN 1 ELSE 0 END) AS seed_step_converged,
|
||||
SUM(CASE WHEN status = 'completed' AND synspec_success_method IS NOT NULL THEN 1 ELSE 0 END) AS synspec_converged
|
||||
FROM grid_points",
|
||||
[],
|
||||
|r| {
|
||||
let n = |i: usize| -> i64 { r.get::<_, Option<i64>>(i).unwrap_or(None).unwrap_or(0) };
|
||||
Ok((n(0), n(1), n(2), n(3), n(4), n(5), n(6), n(7), n(8)))
|
||||
},
|
||||
),
|
||||
}?;
|
||||
let (
|
||||
total,
|
||||
pending,
|
||||
queued,
|
||||
running,
|
||||
completed,
|
||||
failed,
|
||||
cold_run_converged,
|
||||
seed_step_converged,
|
||||
synspec_converged,
|
||||
) = row;
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"total": total,
|
||||
"pending": pending,
|
||||
"queued": queued,
|
||||
"running": running,
|
||||
"completed": completed,
|
||||
"failed": failed,
|
||||
"cold_run_converged": cold_run_converged,
|
||||
"seed_step_converged": seed_step_converged,
|
||||
"synspec_converged": synspec_converged,
|
||||
}))
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 单工作流执行统计（详情页数据源）。
|
||||
///
|
||||
/// 在 `get_grid_summary_stats(Some(name))` 之上追加：
|
||||
/// - `waves`：难度波次分布（调度时按 cno_sum 分组的批次进度）；
|
||||
/// - `avg_point_sec`：单点平均真实耗时 = `AVG(COALESCE(tasks.elapsed_sec, 时间戳差))`——
|
||||
/// 优先用 Worker 回报的精确墙钟（P3 落库），历史无 elapsed_sec 的行回退时间戳差近似；
|
||||
/// - `eta_sec`：`avg_point_sec × 剩余点数 ÷ total_slots`（并发感知；slots 为在线节点
|
||||
/// 总槽位，由 handler 传入，≤0 时按串行兜底），无历史数据为 None。
|
||||
///
|
||||
/// `status` 由调用方传入（工作流当前状态），避免重复查询。
|
||||
pub async fn get_workflow_detail_stats(
|
||||
&self,
|
||||
name: &str,
|
||||
status: &str,
|
||||
total_slots: i64,
|
||||
) -> Result<WorkflowStats> {
|
||||
let base = self.get_grid_summary_stats(Some(name)).await?;
|
||||
let g = |k: &str| base.get(k).and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
let total = g("total");
|
||||
let completed = g("completed");
|
||||
let failed = g("failed");
|
||||
|
||||
let pool = self.pool.clone();
|
||||
let wf = name.to_string();
|
||||
let (waves, avg_point_sec) =
|
||||
tokio::task::spawn_blocking(move || -> Result<(Vec<WaveStats>, Option<f64>)> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT wave,
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) AS completed,
|
||||
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed
|
||||
FROM grid_points WHERE workflow_name = ?1
|
||||
GROUP BY wave ORDER BY wave ASC",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![wf], |r| {
|
||||
Ok(WaveStats {
|
||||
wave: r.get(0)?,
|
||||
total: r.get(1)?,
|
||||
completed: r.get::<_, Option<i64>>(2)?.unwrap_or(0),
|
||||
failed: r.get::<_, Option<i64>>(3)?.unwrap_or(0),
|
||||
})
|
||||
})?;
|
||||
let mut waves = Vec::new();
|
||||
for r in rows {
|
||||
waves.push(r?);
|
||||
}
|
||||
|
||||
// AVG 在无匹配行时返回 NULL；过滤掉非正/非有限值，保持 ETA 估算合理。
|
||||
// COALESCE：精确耗时（elapsed_sec）优先，旧数据回退 created→completed 时间戳差。
|
||||
let avg: Option<f64> = conn
|
||||
.query_row(
|
||||
"SELECT AVG(COALESCE(elapsed_sec, (julianday(completed_at) - julianday(created_at)) * 86400.0))
|
||||
FROM tasks
|
||||
WHERE workflow_name = ?1
|
||||
AND completed_at IS NOT NULL
|
||||
AND status IN ('completed', 'failed', 'timeout')",
|
||||
params![wf],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|v: &f64| v.is_finite() && *v > 0.0);
|
||||
|
||||
Ok((waves, avg))
|
||||
})
|
||||
.await??;
|
||||
|
||||
let remaining = total - completed - failed;
|
||||
let slots = (total_slots.max(1)) as f64;
|
||||
let eta_sec = match avg_point_sec {
|
||||
Some(avg) if remaining > 0 => Some(avg * remaining as f64 / slots),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
Ok(WorkflowStats {
|
||||
name: name.to_string(),
|
||||
status: status.to_string(),
|
||||
total,
|
||||
pending: g("pending"),
|
||||
queued: g("queued"),
|
||||
running: g("running"),
|
||||
completed,
|
||||
failed,
|
||||
cold_run_converged: g("cold_run_converged"),
|
||||
seed_step_converged: g("seed_step_converged"),
|
||||
waves,
|
||||
avg_point_sec,
|
||||
eta_sec,
|
||||
})
|
||||
}
|
||||
|
||||
/// 工作流逐点列表（带最近一次尝试信息），分页返回 `(符合过滤的总数, 当前页行)`。
|
||||
///
|
||||
/// 最近尝试用关联子查询取 tasks 最新行（走 idx_tasks_point_wf_time）；
|
||||
/// 所有过滤值参数化绑定，`q` 的 LIKE 通配符先转义；ORDER BY 片段来自白名单。
|
||||
pub async fn list_workflow_points(
|
||||
&self,
|
||||
wf: &str,
|
||||
f: &PointFilter,
|
||||
) -> Result<(i64, Vec<PointRow>)> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = wf.to_string();
|
||||
let f = f.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<(i64, Vec<PointRow>)> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
|
||||
// 动态 WHERE：子句与占位符同步增长，值全部走绑定参数（SQLite 动态类型，
|
||||
// wave 以字符串绑定由列亲和性转回 INTEGER 比较）。
|
||||
// 列名**不带 gp. 前缀**：窗口重写后筛选作用于外层子查询（列名裸露）。
|
||||
let mut clauses: Vec<String> = vec!["workflow_name = ?1".to_string()];
|
||||
let mut binds: Vec<Box<dyn rusqlite::types::ToSql>> = vec![Box::new(wf.clone())];
|
||||
if let Some(s) = &f.status {
|
||||
binds.push(Box::new(s.clone()));
|
||||
clauses.push(format!("status = ?{}", binds.len()));
|
||||
}
|
||||
if let Some(m) = &f.method {
|
||||
if m == "synspec_only" {
|
||||
// 光谱专用收敛点（tlusty 禁用）：仅命中 synspec-only 点（tlusty 归因 NULL），
|
||||
// 与徽章/parSets 的 SYNSPEC 档同口径。双阶段点归因在大气侧，不含在这里。
|
||||
clauses.push(
|
||||
"tlusty_success_method IS NULL AND synspec_success_method IS NOT NULL"
|
||||
.to_string(),
|
||||
);
|
||||
} else {
|
||||
// cold_run/seed_step 等 TLUSTY 阶段策略 → 过滤 tlusty_success_method。
|
||||
binds.push(Box::new(m.clone()));
|
||||
clauses.push(format!("tlusty_success_method = ?{}", binds.len()));
|
||||
}
|
||||
}
|
||||
if let Some(w) = f.wave {
|
||||
binds.push(Box::new(w.to_string()));
|
||||
clauses.push(format!("wave = ?{}", binds.len()));
|
||||
}
|
||||
if let Some(q) = &f.q {
|
||||
// 转义 LIKE 通配符（\ % _），仅影响匹配语义，不构成注入面（值仍绑定）。
|
||||
let escaped = q
|
||||
.replace('\\', "\\\\")
|
||||
.replace('%', "\\%")
|
||||
.replace('_', "\\_");
|
||||
binds.push(Box::new(format!("%{}%", escaped)));
|
||||
clauses.push(format!("name LIKE ?{} ESCAPE '\\'", binds.len()));
|
||||
}
|
||||
let where_sql = clauses.join(" AND ");
|
||||
let bind_refs = || binds.iter().map(|b| b.as_ref());
|
||||
|
||||
// 总数（同过滤条件，作用于 grid_points，无需 JOIN）
|
||||
let total: i64 = conn.query_row(
|
||||
&format!("SELECT COUNT(*) FROM grid_points WHERE {}", where_sql),
|
||||
rusqlite::params_from_iter(bind_refs()),
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
|
||||
// 数据行：ROW_NUMBER() 窗口单遍取每点最新任务，替代逐行相关子查询
|
||||
// （P3，数千点 → 数千次子查询消除）。窗口 PARTITION BY (name, workflow_name)
|
||||
// 内按「未完成靠后 → completed_at DESC → created_at DESC」选最新行，与旧
|
||||
// 相关子查询 ORDER BY 语义逐行等价；外层 rn=1 过滤 + 用户排序/分页。
|
||||
let mut all_binds = binds;
|
||||
// limit=None 时不拼 LIMIT 子句（联合分析需全量，截断会让分析失真）。
|
||||
let limit_clause = match f.limit {
|
||||
Some(lim) => {
|
||||
let limit_idx = all_binds.len() + 1;
|
||||
all_binds.push(Box::new(lim));
|
||||
format!(" LIMIT ?{}", limit_idx)
|
||||
}
|
||||
None => String::new(),
|
||||
};
|
||||
// offset 仅在有 limit 或非零时才有意义；None-limit 全量场景强制忽略 offset。
|
||||
let offset_clause = if f.limit.is_some() {
|
||||
let offset_idx = all_binds.len() + 1;
|
||||
all_binds.push(Box::new(f.offset));
|
||||
format!(" OFFSET ?{}", offset_idx)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
// 排序：order_by 由 API 层编译期白名单拼出；空（如测试用 default filter）则省略子句。
|
||||
let order_clause = if f.order_by.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(" ORDER BY {}", f.order_by)
|
||||
};
|
||||
let sql = format!(
|
||||
"SELECT * FROM (
|
||||
SELECT gp.name, gp.teff, gp.logg, gp.loghe, gp.logc, gp.logn, gp.logo,
|
||||
gp.cno_sum, gp.wave, gp.status, gp.tlusty_success_method, gp.attempt_count,
|
||||
t.max_relc,
|
||||
t.seed_point_name, t.node_id,
|
||||
t.completed_at, t.error_message,
|
||||
COALESCE(t.elapsed_sec, gp.last_elapsed_sec) AS eff_elapsed,
|
||||
gp.workflow_name,
|
||||
gp.synspec_success_method,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY gp.name, gp.workflow_name
|
||||
ORDER BY t.completed_at IS NULL, t.completed_at DESC, t.created_at DESC
|
||||
) AS rn
|
||||
FROM grid_points gp
|
||||
LEFT JOIN tasks t
|
||||
ON t.point_name = gp.name AND t.workflow_name = gp.workflow_name
|
||||
) WHERE rn = 1 AND {}{}{}{}",
|
||||
where_sql, order_clause, limit_clause, offset_clause
|
||||
);
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
let rows = stmt.query_map(
|
||||
rusqlite::params_from_iter(all_binds.iter().map(|b| b.as_ref())),
|
||||
point_row_from_query,
|
||||
)?;
|
||||
let mut points = Vec::new();
|
||||
for r in rows {
|
||||
points.push(r?);
|
||||
}
|
||||
Ok((total, points))
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 单点行（带最近尝试信息），点不存在返回 None。与列表端点同一 SELECT 列序。
|
||||
pub async fn get_workflow_point_row(&self, wf: &str, point: &str) -> Result<Option<PointRow>> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = wf.to_string();
|
||||
let point = point.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<Option<PointRow>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
// 与 list_workflow_points 同构的 ROW_NUMBER() 窗口取最近任务（P3），外层 rn=1。
|
||||
"SELECT * FROM (
|
||||
SELECT gp.name, gp.teff, gp.logg, gp.loghe, gp.logc, gp.logn, gp.logo,
|
||||
gp.cno_sum, gp.wave, gp.status, gp.tlusty_success_method, gp.attempt_count,
|
||||
t.max_relc,
|
||||
t.seed_point_name, t.node_id,
|
||||
t.completed_at, t.error_message,
|
||||
COALESCE(t.elapsed_sec, gp.last_elapsed_sec) AS eff_elapsed,
|
||||
gp.workflow_name,
|
||||
gp.synspec_success_method,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY gp.name, gp.workflow_name
|
||||
ORDER BY t.completed_at IS NULL, t.completed_at DESC, t.created_at DESC
|
||||
) AS rn
|
||||
FROM grid_points gp
|
||||
LEFT JOIN tasks t
|
||||
ON t.point_name = gp.name AND t.workflow_name = gp.workflow_name
|
||||
) WHERE rn = 1 AND workflow_name = ?1 AND name = ?2",
|
||||
)?;
|
||||
let res = stmt.query_row(params![wf, point], point_row_from_query);
|
||||
match res {
|
||||
Ok(row) => Ok(Some(row)),
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
})
|
||||
.await?
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,577 @@
|
||||
//! Node/凭据（从原 db.rs 拆分，2026-08-06）。
|
||||
//! `impl Database` 的 Node/凭据 域方法。共享基础设施（Database struct、连接管理、类型、helper）见父模块 `super`（crate::db）。
|
||||
use super::*;
|
||||
|
||||
impl Database {
|
||||
// --- Node operations ---
|
||||
/// 注册/刷新节点。返回 (is_new, existing_status)：
|
||||
/// - 新申请：`(true, None)`
|
||||
/// - 已存在（含 online 等已审批态）：`(false, Some(<旧状态>))`，仅更新配置保持既有状态。
|
||||
///
|
||||
/// 返回旧状态供 API 层区分响应：已审批（online）的节点免凭据重新注册时，
|
||||
/// 不应回 "pending_approval"（误导运维以为还需审批），而应如实告知其已是已授权节点。
|
||||
pub async fn register_node(
|
||||
&self,
|
||||
req: &NodeRegisterRequest,
|
||||
) -> Result<(bool, Option<String>, Option<String>)> {
|
||||
let pool = self.pool.clone();
|
||||
let req_cloned = req.clone();
|
||||
|
||||
let (is_new, existing_status, registration_secret) =
|
||||
tokio::task::spawn_blocking(move || -> Result<(bool, Option<String>, Option<String>)> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare("SELECT status FROM nodes WHERE node_id = ?1")?;
|
||||
let existing_status: Option<String> =
|
||||
stmt.query_row(params![req_cloned.node_id], |r| r.get(0)).ok();
|
||||
|
||||
match &existing_status {
|
||||
Some(_st) => {
|
||||
// 已存在的节点：更新配置，保持既有状态
|
||||
conn.execute(
|
||||
"UPDATE nodes SET max_slots = ?1, last_heartbeat = datetime('now') WHERE node_id = ?2",
|
||||
params![req_cloned.max_slots, req_cloned.node_id],
|
||||
)?;
|
||||
Ok((false, existing_status, None))
|
||||
}
|
||||
None => {
|
||||
// 新申请节点：生成一次性 registration_secret（H8）并插入待审批状态。
|
||||
// registration_secret 用于 /node/check_status 取走专属 token 的二次凭据，
|
||||
// 防止知道 node_id（常源自主机名，可猜测）的攻击者抢先取走待发 token。
|
||||
let secret = format!(
|
||||
"{}{}",
|
||||
uuid::Uuid::new_v4().simple(),
|
||||
uuid::Uuid::new_v4().simple()
|
||||
);
|
||||
conn.execute(
|
||||
"INSERT INTO nodes (node_id, max_slots, status, last_heartbeat, registration_secret)
|
||||
VALUES (?1, ?2, 'pending_approval', datetime('now'), ?3)",
|
||||
params![req_cloned.node_id, req_cloned.max_slots, secret],
|
||||
)?;
|
||||
Ok((true, None, Some(secret)))
|
||||
}
|
||||
}
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok((is_new, existing_status, registration_secret))
|
||||
}
|
||||
|
||||
/// 管理员审批同意节点接入：将节点状态切为 online 并生成专属 node_token（返回明文 token）。
|
||||
pub async fn approve_node(&self, node_id: &str) -> Result<String> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"UPDATE nodes SET status = 'online', last_heartbeat = datetime('now') WHERE node_id = ?1",
|
||||
params![node_id_owned],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
// 颁发专属 node_token
|
||||
let new_token = self.issue_node_token(node_id).await?;
|
||||
Ok(new_token)
|
||||
}
|
||||
|
||||
/// 管理员拒绝节点接入：彻底清理该节点的注册申请记录。
|
||||
pub async fn reject_node(&self, node_id: &str) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"DELETE FROM nodes WHERE node_id = ?1",
|
||||
params![node_id_owned],
|
||||
)?;
|
||||
conn.execute(
|
||||
"DELETE FROM node_credentials WHERE node_id = ?1",
|
||||
params![node_id_owned],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 记录心跳并回读管理员配额。
|
||||
///
|
||||
/// 返回 `Option<i32>`：当前节点的 `admin_max_slots`（None = 无限制）。供 API 层
|
||||
/// 在心跳响应体透传给 Worker（见 docs/dynamic_cpu_slots_design.md §3.1）。在 UPDATE
|
||||
/// 之后立即 SELECT，保证管理员刚设置的配额在本次心跳即下发（心跳间隔内的延迟可接受）。
|
||||
pub async fn heartbeat_node(&self, req: &NodeHeartbeatRequest) -> Result<Option<i32>> {
|
||||
let pool = self.pool.clone();
|
||||
let req_cloned = req.clone();
|
||||
|
||||
let admin_max_slots = tokio::task::spawn_blocking(move || -> Result<Option<i32>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 心跳刷新指标与时间戳，status 由 CASE 决定：
|
||||
// - online/offline → online（offline 节点复活）
|
||||
// - disabled → disabled（管理员手动停用，永不被心跳复活；
|
||||
// 仅刷新 cpu/mem/心跳，让 Dashboard 仍能看到该节点存活待命）
|
||||
// - pending_approval 不在 IN 列表，心跳对其无效（维持审批语义）。
|
||||
conn.execute(
|
||||
"UPDATE nodes SET active_slots = ?1, cpu_usage = ?2, memory_usage = ?3,
|
||||
last_heartbeat = datetime('now'),
|
||||
status = CASE WHEN status = 'disabled' THEN 'disabled' ELSE 'online' END
|
||||
WHERE node_id = ?4 AND status IN ('online', 'offline', 'disabled')",
|
||||
params![
|
||||
req_cloned.active_slots,
|
||||
req_cloned.cpu_usage,
|
||||
req_cloned.memory_usage,
|
||||
req_cloned.node_id
|
||||
],
|
||||
)?;
|
||||
// 回读配额。注：即便 UPDATE 命中 0 行（如 pending_approval 节点心跳），
|
||||
// 节点行仍存在，配额回读仍应给出当前值；这里以 node_id 直查即可。
|
||||
// 仅把「无行」视为 None；真实 DB 错误（I/O、列缺失等）须向上传播，
|
||||
// 避免把故障静默降级为「无配额」而让 Worker 以物理上限满负荷运行。
|
||||
// （审查修复 N2：旧实现 .ok().flatten() 会把 SQL 错误一并吞成 None。）
|
||||
let quota: Option<i32> = {
|
||||
let res = conn.query_row(
|
||||
"SELECT admin_max_slots FROM nodes WHERE node_id = ?1",
|
||||
params![req_cloned.node_id],
|
||||
|r| r.get::<_, Option<i32>>(0),
|
||||
);
|
||||
match res {
|
||||
Ok(v) => v,
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => None,
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
};
|
||||
Ok(quota)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(admin_max_slots)
|
||||
}
|
||||
|
||||
/// 设置管理员强制并发槽位上限（动态调整 CPU 核数）。
|
||||
///
|
||||
/// - `Some(n)`（n>=0）：限制节点最多并发 n 个任务（0 = 暂停接新任务，正在跑的不会中断）。
|
||||
/// - `None`：清除限制，恢复节点物理 `max_slots` 上限。
|
||||
///
|
||||
/// 返回是否命中行（节点未注册时返回 false，调用方据此返回 404）。负数由 API 层
|
||||
/// 校验拒绝（不在此处理），避免把非法值写入 DB。
|
||||
pub async fn set_node_admin_max_slots(
|
||||
&self,
|
||||
node_id: &str,
|
||||
admin_max_slots: Option<i32>,
|
||||
) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
|
||||
let count = tokio::task::spawn_blocking(move || -> Result<u64> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let rows = conn.execute(
|
||||
"UPDATE nodes SET admin_max_slots = ?1 WHERE node_id = ?2",
|
||||
params![admin_max_slots, node_id_owned],
|
||||
)?;
|
||||
Ok(rows as u64)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(count > 0)
|
||||
}
|
||||
|
||||
/// 手动停用/启用节点（管理员运维操作）。
|
||||
///
|
||||
/// - `enabled=false`：把 `online`/`offline` 节点切为 `disabled`，claim 将不再向其分发任务。
|
||||
/// `pending_approval` 不在范围（审批流程独立），`disabled` 再点为幂等空操作。
|
||||
/// - `enabled=true`：把 `disabled` 节点切为 `offline`（而非 online），靠节点下一次心跳
|
||||
/// 自然翻成 online——既能自愈，又不会对真实离线的节点虚报在线。
|
||||
///
|
||||
/// 返回 `true` 表示状态确有迁移；`false` 表示当前状态不匹配（调用方据此返回 409）。
|
||||
pub async fn set_node_enabled(&self, node_id: &str, enabled: bool) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
|
||||
let count = tokio::task::spawn_blocking(move || -> Result<u64> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let rows = if enabled {
|
||||
conn.execute(
|
||||
"UPDATE nodes SET status = 'offline' WHERE node_id = ?1 AND status = 'disabled'",
|
||||
params![node_id_owned],
|
||||
)?
|
||||
} else {
|
||||
conn.execute(
|
||||
"UPDATE nodes SET status = 'disabled' WHERE node_id = ?1 AND status IN ('online', 'offline')",
|
||||
params![node_id_owned],
|
||||
)?
|
||||
};
|
||||
Ok(rows as u64)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(count > 0)
|
||||
}
|
||||
|
||||
/// 节点是否处于手动停用态（claim 热路径用，命中主键索引，开销可忽略）。
|
||||
pub async fn is_node_disabled(&self, node_id: &str) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
|
||||
let disabled = tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt =
|
||||
conn.prepare("SELECT 1 FROM nodes WHERE node_id = ?1 AND status = 'disabled'")?;
|
||||
let exists = stmt.exists(params![node_id_owned])?;
|
||||
Ok(exists)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(disabled)
|
||||
}
|
||||
|
||||
/// 为指定 node 颁发专属 token：生成随机明文 token，DB 存其 SHA-256 hash。
|
||||
/// 返回明文 token（仅此一次，由调用方转交 node 持久化）。
|
||||
/// 若该 node 已有凭据则覆盖（重新颁发）。
|
||||
pub async fn issue_node_token(&self, node_id: &str) -> Result<String> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
// 两个 v4 UUID（各 16 字节随机）拼接 → 各 32 hex 字符 = 64 字符 token
|
||||
let token =
|
||||
uuid::Uuid::new_v4().simple().to_string() + &uuid::Uuid::new_v4().simple().to_string();
|
||||
let token_hash = hash_token(&token);
|
||||
let token_for_ret = token.clone();
|
||||
let token_to_db = token_for_ret.clone();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"INSERT INTO node_credentials (node_id, token_hash, issued_at, raw_token_pending)
|
||||
VALUES (?1, ?2, datetime('now'), ?3)
|
||||
ON CONFLICT(node_id) DO UPDATE SET
|
||||
token_hash = excluded.token_hash,
|
||||
issued_at = datetime('now'),
|
||||
raw_token_pending = excluded.raw_token_pending",
|
||||
params![node_id_owned, token_hash, token_to_db],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
// token 轮换：旧 token_hash 已失效，新 token_hash 即将生效。整体清空缓存最稳妥
|
||||
// （issue 是低频运维动作，全清代价可忽略）。
|
||||
self.invalidate_token_cache().await;
|
||||
|
||||
Ok(token_for_ret)
|
||||
}
|
||||
|
||||
/// 一次性拉取并清除暂存的明文 node_token（取走即焚安全策略）。
|
||||
///
|
||||
/// 在单个 IMMEDIATE 事务内：先 SELECT 读出明文，再 UPDATE 置 NULL。IMMEDIATE 事务在
|
||||
/// BEGIN 时即获取写锁，保证 SELECT 与 UPDATE 之间不会被其它调用方插入，从而只有一个
|
||||
/// 调用方能取到 token（原子语义）。
|
||||
///
|
||||
/// 注：SQLite 的 `UPDATE ... RETURNING` 返回的是列的**新值**（SET 之后），故清空后
|
||||
/// RETURNING 该列只会得到 NULL，无法用于读旧值；因此这里用显式 SELECT + UPDATE。
|
||||
pub async fn take_pending_node_token(
|
||||
&self,
|
||||
node_id: &str,
|
||||
registration_secret: Option<&str>,
|
||||
) -> Result<Option<String>> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
let secret_owned = registration_secret.map(|s| s.to_string());
|
||||
|
||||
let token = tokio::task::spawn_blocking(move || -> Result<Option<String>> {
|
||||
let mut conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
|
||||
|
||||
// H8：取走待发 token 前校验 registration_secret（节点注册时下发的一次性凭据）。
|
||||
// 仅当 nodes 表记录的 registration_secret 与请求提供的一致（恒定时间比对），
|
||||
// 才允许取走 token，防止仅知道 node_id（可猜测）的攻击者抢先取走。
|
||||
let stored_secret: Option<String> = {
|
||||
let mut secret_stmt =
|
||||
tx.prepare("SELECT registration_secret FROM nodes WHERE node_id = ?1")?;
|
||||
secret_stmt
|
||||
.query_row(params![node_id_owned], |r| r.get::<_, Option<String>>(0))
|
||||
.ok()
|
||||
.flatten()
|
||||
};
|
||||
let secret_ok = match (&stored_secret, &secret_owned) {
|
||||
(Some(a), Some(b)) => ct_eq_option(a, b),
|
||||
// 旧库节点（无 registration_secret）不强制要求，保持向后兼容；
|
||||
// 新节点（有 secret）必须提供正确 secret。
|
||||
(None, _) => true,
|
||||
(Some(_), None) => false,
|
||||
};
|
||||
if !secret_ok {
|
||||
tx.commit()?;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let raw_token: Option<String> = {
|
||||
let mut select_stmt = tx.prepare(
|
||||
"SELECT raw_token_pending FROM node_credentials
|
||||
WHERE node_id = ?1 AND raw_token_pending IS NOT NULL",
|
||||
)?;
|
||||
select_stmt
|
||||
.query_row(params![node_id_owned], |r| r.get::<_, String>(0))
|
||||
.ok()
|
||||
};
|
||||
if raw_token.is_some() {
|
||||
tx.execute(
|
||||
"UPDATE node_credentials SET raw_token_pending = NULL
|
||||
WHERE node_id = ?1 AND raw_token_pending IS NOT NULL",
|
||||
params![&node_id_owned],
|
||||
)?;
|
||||
// M2 修复：取走后把 registration_secret 轮换为全新随机值（本地一次性凭据）。
|
||||
// 设计文档（node_credentials 注释）明确 registration_secret 是「审批前一次性凭据，
|
||||
// 取走专属 token 前消费」。若取走后旧 secret 仍有效，被攻陷节点/截获注册响应的
|
||||
// 攻击者可凭「可猜测的 node_id + 旧 secret」反复取走每次 reissue 产生的新 token。
|
||||
// 轮换成无人知晓的新值后旧 secret 立即失效，强制 reissue 的 token 走管理员线下
|
||||
// 下发路径（worker.rs 401 提示引导），杜绝凭 secret 复活取 token。
|
||||
// 注意刻意**不清空为 NULL**：下方鉴权把 `(None, _) => true` 当作旧库兼容放行，
|
||||
// 清空会让取走后的节点退化为「免 secret 可取」的旧库语义，反而开新洞。
|
||||
let new_secret = format!(
|
||||
"{}{}",
|
||||
uuid::Uuid::new_v4().simple(),
|
||||
uuid::Uuid::new_v4().simple()
|
||||
);
|
||||
tx.execute(
|
||||
"UPDATE nodes SET registration_secret = ?2 WHERE node_id = ?1",
|
||||
params![&node_id_owned, new_secret],
|
||||
)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(raw_token)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
/// 按 token（明文）反查所属 node_id；仅当 token 有效（未被重发覆盖）时返回 Some。
|
||||
/// 用于中间件：请求带来 node token，由此确定调用方身份。
|
||||
///
|
||||
/// 高频路径（每个 Node 请求一次）：先查内存 token_cache，命中且未过期直接返回；
|
||||
/// miss 才落 DB，并回填缓存。issue 会主动清空整个缓存。
|
||||
///
|
||||
/// 失效语义：重发（issue_node_token）会用 ON CONFLICT 覆盖该 node 的 token_hash，
|
||||
/// 旧 token 明文 hash 不再存在于表 → 查询返回 None → 401。无需独立的 revoked 标记。
|
||||
///
|
||||
/// 撤销竞态修复：历史上存在 TOCTOU 窗口——线程 A 用旧 token miss 落 DB 查到 node_id
|
||||
/// 后准备回填，期间线程 B（管理员 reissue）覆盖 DB 的 token_hash 并 clear() 缓存，
|
||||
/// 随后线程 A 拿到写锁把旧 token_hash 回填进缓存，导致已撤销的旧 token 在 TTL(60s)
|
||||
/// 内仍能鉴权。修复：回填时在同一把写锁内重新校验该 token_hash 是否仍是 DB 当前值
|
||||
/// （未被 reissue 覆盖），是才回填，杜绝旧 token 复活窗口。
|
||||
pub async fn find_node_by_token(&self, token: &str) -> Option<String> {
|
||||
let token_hash = hash_token(token);
|
||||
|
||||
// 1) 先查内存缓存
|
||||
{
|
||||
let cache = self.token_cache.read().await;
|
||||
if let Some((node_id, inserted)) = cache.entries.get(&token_hash) {
|
||||
if inserted.elapsed() < TOKEN_CACHE_TTL {
|
||||
return Some(node_id.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2) 记录 DB 查询前的缓存 generation，miss 落 DB。
|
||||
// generation 用于回填时的 TOCTOU 终极防护：若 DB 查询与回填之间发生过
|
||||
// invalidate（reissue），generation 会变化，本次回填将被丢弃。
|
||||
let gen_before = { self.token_cache.read().await.generation };
|
||||
let pool = self.pool.clone();
|
||||
let hash_for_db = token_hash.clone();
|
||||
let db_hit: Option<String> =
|
||||
tokio::task::spawn_blocking(move || -> Result<Option<String>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT node_id FROM node_credentials WHERE token_hash = ?1 LIMIT 1",
|
||||
)?;
|
||||
let res = stmt.query_row(params![hash_for_db], |r| r.get::<_, String>(0));
|
||||
match res {
|
||||
Ok(id) => Ok(Some(id)),
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|r| match r {
|
||||
Ok(id) => id,
|
||||
// 审查修复 #M4：原 `.and_then(|r| r.ok())` 把 DB 错误静默吞成 None（鉴权 401），
|
||||
// 故障期所有节点请求被判 401 且无任何日志。现至少记录 warn 便于排障。
|
||||
Err(e) => {
|
||||
tracing::warn!("find_node_by_token DB 查询失败，降级为鉴权未命中: {}", e);
|
||||
None
|
||||
}
|
||||
});
|
||||
|
||||
// 3) 命中则回填缓存；回填前校验 generation 未变化（期间无 invalidate），
|
||||
// 彻底消除"旧 token_hash 复活"窗口。generation 变化则视为已撤销，不缓存、不返回。
|
||||
if let Some(id) = db_hit {
|
||||
let mut cache = self.token_cache.write().await;
|
||||
if cache.generation == gen_before {
|
||||
cache
|
||||
.entries
|
||||
.insert(token_hash, (id.clone(), std::time::Instant::now()));
|
||||
Some(id)
|
||||
} else {
|
||||
// 期间发生过 reissue 导致的 invalidate：旧 token_hash 已不应复活。
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// 清空全部 token 反查缓存并自增 generation。在 issue（token 轮换使旧 token 失效）时调用。
|
||||
/// 自增 generation 使所有在途的 find_node_by_token 回填（gen_before 已过期）被丢弃，
|
||||
/// 彻底消除"DB 读取旧 hash → reissue clear → 回填旧 hash"的 TOCTOU 复活窗口。
|
||||
async fn invalidate_token_cache(&self) {
|
||||
let mut cache = self.token_cache.write().await;
|
||||
cache.entries.clear();
|
||||
cache.generation = cache.generation.wrapping_add(1);
|
||||
}
|
||||
|
||||
/// 判断指定 node_id 是否已存在于 nodes 表（重发 token 前置校验，防幽灵 node_id）。
|
||||
pub async fn get_node_exists(&self, node_id: &str) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
let exists = tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare("SELECT 1 FROM nodes WHERE node_id = ?1 LIMIT 1")?;
|
||||
Ok(stmt.exists(params![node_id_owned])?)
|
||||
})
|
||||
.await??;
|
||||
Ok(exists)
|
||||
}
|
||||
|
||||
/// 统计已颁发 node 凭据数量（用于启动期半配置告警判断）。
|
||||
pub async fn node_credentials_count(&self) -> Result<i64> {
|
||||
let pool = self.pool.clone();
|
||||
let count = tokio::task::spawn_blocking(move || -> Result<i64> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let n: i64 =
|
||||
conn.query_row("SELECT COUNT(*) FROM node_credentials", [], |r| r.get(0))?;
|
||||
Ok(n)
|
||||
})
|
||||
.await??;
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
/// 列出全部节点及其凭据状态（LEFT JOIN node_credentials）。
|
||||
/// 用于管理 API：admin 可查看每个节点的在线状态、是否已颁发 token、颁发时间。
|
||||
/// 尚未注册凭据的节点（如旧数据迁移）token_status 为 "none"、token_issued_at 为 None。
|
||||
pub async fn list_nodes_with_credentials(&self) -> Result<Vec<NodeCredentialView>> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<NodeCredentialView>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT n.node_id, n.max_slots, n.active_slots, n.status,
|
||||
n.cpu_usage, n.memory_usage,
|
||||
strftime('%Y-%m-%dT%H:%M:%SZ', n.last_heartbeat),
|
||||
c.token_hash, strftime('%Y-%m-%dT%H:%M:%SZ', c.issued_at),
|
||||
n.admin_max_slots
|
||||
FROM nodes n
|
||||
LEFT JOIN node_credentials c ON c.node_id = n.node_id
|
||||
ORDER BY n.status ASC, n.node_id ASC",
|
||||
)?;
|
||||
let rows = stmt.query_map([], |r| {
|
||||
let hb_str: String = r.get::<_, String>(6)?;
|
||||
Ok(NodeCredentialView {
|
||||
node_id: r.get(0)?,
|
||||
max_slots: r.get(1)?,
|
||||
active_slots: r.get(2)?,
|
||||
status: r.get(3)?,
|
||||
cpu_usage: r.get(4)?,
|
||||
memory_usage: r.get(5)?,
|
||||
last_heartbeat: chrono::DateTime::parse_from_rfc3339(&hb_str)
|
||||
.map(|d| d.with_timezone(&chrono::Utc))
|
||||
.unwrap_or_else(|_| chrono::DateTime::UNIX_EPOCH),
|
||||
// c.token_hash 为 NULL 表示该节点无凭据记录；非空即为有效 token
|
||||
// （token 失效靠重发覆盖 hash 实现，不存在「已吊销」中间态）
|
||||
token_status: match r.get::<_, Option<String>>(7)? {
|
||||
None => "none".to_string(),
|
||||
Some(_) => "active".to_string(),
|
||||
},
|
||||
token_issued_at: r.get::<_, Option<String>>(8)?,
|
||||
admin_max_slots: r.get::<_, Option<i32>>(9)?,
|
||||
})
|
||||
})?;
|
||||
let mut list = Vec::new();
|
||||
for row in rows {
|
||||
list.push(row?);
|
||||
}
|
||||
Ok(list)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
pub async fn get_active_nodes(&self) -> Result<Vec<NodeInfo>> {
|
||||
let pool = self.pool.clone();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<NodeInfo>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT node_id, max_slots, active_slots, status, cpu_usage, memory_usage, strftime('%Y-%m-%dT%H:%M:%SZ', last_heartbeat), admin_max_slots FROM nodes WHERE status = 'online'"
|
||||
)?;
|
||||
|
||||
let node_iter = stmt.query_map([], |r| {
|
||||
let hb_str: String = r.get(6)?;
|
||||
Ok(NodeInfo {
|
||||
node_id: r.get(0)?,
|
||||
max_slots: r.get(1)?,
|
||||
active_slots: r.get(2)?,
|
||||
status: r.get(3)?,
|
||||
cpu_usage: r.get(4)?,
|
||||
memory_usage: r.get(5)?,
|
||||
last_heartbeat: chrono::DateTime::parse_from_rfc3339(&hb_str)
|
||||
.map(|d| d.with_timezone(&chrono::Utc))
|
||||
.unwrap_or_else(|_| chrono::Utc::now()),
|
||||
admin_max_slots: r.get::<_, Option<i32>>(7)?,
|
||||
})
|
||||
})?;
|
||||
|
||||
let mut nodes = Vec::new();
|
||||
for n in node_iter {
|
||||
nodes.push(n?);
|
||||
}
|
||||
Ok(nodes)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
pub async fn mark_stale_nodes_offline(&self, stale_sec: u64) -> Result<u64> {
|
||||
let pool = self.pool.clone();
|
||||
let count = tokio::task::spawn_blocking(move || -> Result<u64> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let rows = conn.execute(
|
||||
"UPDATE nodes SET status = 'offline' WHERE status = 'online' AND strftime('%s', 'now') - strftime('%s', last_heartbeat) > ?1",
|
||||
params![stale_sec as i64],
|
||||
)?;
|
||||
Ok(rows as u64)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
//! 种子（从原 db.rs 拆分，2026-08-06）。
|
||||
//! `impl Database` 的 种子 域方法。共享基础设施（Database struct、连接管理、类型、helper）见父模块 `super`（crate::db）。
|
||||
use super::*;
|
||||
|
||||
impl Database {
|
||||
pub async fn reload_seed_cache(&self) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let items = tokio::task::spawn_blocking(move || -> Result<Vec<SeedCacheItem>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT point_name, teff, logg, loghe, logc, logn, logo, file_path FROM seeds WHERE is_clean = 1"
|
||||
)?;
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok(SeedCacheItem {
|
||||
point_name: row.get(0)?,
|
||||
params: GridPointParams {
|
||||
teff: GridAxisValue::from_value(row.get::<_, f64>(1)?),
|
||||
logg: GridAxisValue::from_value(row.get::<_, f64>(2)?),
|
||||
loghe: GridAxisValue::from_value(row.get::<_, f64>(3)?),
|
||||
logc: GridAxisValue::from_value(row.get::<_, f64>(4)?),
|
||||
logn: GridAxisValue::from_value(row.get::<_, f64>(5)?),
|
||||
logo: GridAxisValue::from_value(row.get::<_, f64>(6)?),
|
||||
},
|
||||
file_path: row.get(7)?,
|
||||
})
|
||||
})?;
|
||||
let mut list = Vec::new();
|
||||
for r in rows {
|
||||
list.push(r?);
|
||||
}
|
||||
Ok(list)
|
||||
})
|
||||
.await??;
|
||||
|
||||
// 同步重建 exact_family 索引（每个种子写入其 floor/floor+1 两个桶）。
|
||||
let mut index: std::collections::HashMap<SeedBucketKey, Vec<SeedCacheItem>> =
|
||||
std::collections::HashMap::new();
|
||||
for item in &items {
|
||||
for key in SeedBucketKey::from_params(&item.params) {
|
||||
index.entry(key).or_default().push(item.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let mut lock = self.seed_cache.write().await;
|
||||
*lock = items;
|
||||
drop(lock);
|
||||
let mut idx_lock = self.seed_index.write().await;
|
||||
*idx_lock = index;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn insert_seed(&self, params_in: &GridPointParams, file_path: &str) -> Result<()> {
|
||||
let name = params_in.model_name();
|
||||
self.insert_seed_named(&name, params_in, file_path).await
|
||||
}
|
||||
|
||||
/// 与 `insert_seed` 相同，但用调用方提供的权威 `name`（而非从 params 重推）。
|
||||
///
|
||||
/// 历史种子导入专用：保证 `seeds.point_name` 与旧版 conv.json 的源精度真名一致。
|
||||
pub async fn insert_seed_named(
|
||||
&self,
|
||||
name: &str,
|
||||
params_in: &GridPointParams,
|
||||
file_path: &str,
|
||||
) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let p = params_in.clone();
|
||||
let name = name.to_string();
|
||||
let path_owned = file_path.to_string();
|
||||
|
||||
let name_db = name.clone();
|
||||
let path_db = path_owned.clone();
|
||||
let p_db = p.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"INSERT INTO seeds (point_name, teff, logg, loghe, logc, logn, logo, file_path)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
|
||||
ON CONFLICT(point_name) DO UPDATE SET file_path = excluded.file_path",
|
||||
params![
|
||||
name_db,
|
||||
p_db.teff.value(),
|
||||
p_db.logg.value(),
|
||||
p_db.loghe.value(),
|
||||
p_db.logc.value(),
|
||||
p_db.logn.value(),
|
||||
p_db.logo.value(),
|
||||
path_db
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
let item = SeedCacheItem {
|
||||
point_name: name.clone(),
|
||||
params: p.clone(),
|
||||
file_path: path_owned,
|
||||
};
|
||||
let mut lock = self.seed_cache.write().await;
|
||||
let is_new;
|
||||
if let Some(pos) = lock.iter().position(|x| x.point_name == item.point_name) {
|
||||
// 已存在：seeds 表 ON CONFLICT 只更新 file_path，point_name/物理参数不变，
|
||||
// 故 exact_family 桶键不变，索引无需重写，仅同步 Vec 里的 file_path。
|
||||
lock[pos].file_path = item.file_path.clone();
|
||||
is_new = false;
|
||||
} else {
|
||||
lock.push(item.clone());
|
||||
is_new = true;
|
||||
}
|
||||
drop(lock);
|
||||
|
||||
// 新种子才需写入索引（已存在的种子 params 不变，桶键未变）。
|
||||
if is_new {
|
||||
let mut idx_lock = self.seed_index.write().await;
|
||||
for key in SeedBucketKey::from_params(&p) {
|
||||
idx_lock.entry(key).or_default().push(item.clone());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn find_best_seed_from_db(
|
||||
&self,
|
||||
target: &GridPointParams,
|
||||
) -> Result<Option<common::seed_finder::SeedMatch>> {
|
||||
// 优先走 exact_family 索引（O(1)~O(小)）：取出 target 的两个候选桶的全部种子快照后
|
||||
// 立即释放读锁，避免阻塞 insert_seed 写。exact_family 是绝大多数命中的路径。
|
||||
let exact_candidates: Vec<SeedCacheItem> = {
|
||||
let idx_lock = self.seed_index.read().await;
|
||||
let keys = SeedBucketKey::from_params(target);
|
||||
let mut out = Vec::new();
|
||||
for key in keys {
|
||||
if let Some(bucket) = idx_lock.get(&key) {
|
||||
out.extend(bucket.iter().cloned());
|
||||
}
|
||||
}
|
||||
out
|
||||
};
|
||||
|
||||
let mut exact_family: Option<(String, std::path::PathBuf, f64)> = None;
|
||||
for item in &exact_candidates {
|
||||
let (is_exact, d) = common::seed_finder::calculate_seed_distance(&item.params, target);
|
||||
if is_exact {
|
||||
let path = std::path::PathBuf::from(&item.file_path);
|
||||
if exact_family.is_none() || d < exact_family.as_ref().unwrap().2 {
|
||||
exact_family = Some((item.point_name.clone(), path, d));
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some((name, path, d)) = exact_family {
|
||||
return Ok(Some(common::seed_finder::SeedMatch {
|
||||
name,
|
||||
path,
|
||||
distance: d,
|
||||
}));
|
||||
}
|
||||
|
||||
// exact_family 未命中：退化到全量 global 扫描。克隆参数缩小读锁持有范围。
|
||||
let snapshot: Vec<_> = {
|
||||
let lock = self.seed_cache.read().await;
|
||||
lock.iter()
|
||||
.map(|item| {
|
||||
(
|
||||
item.point_name.clone(),
|
||||
item.params.clone(),
|
||||
item.file_path.clone(),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
|
||||
let mut global_closest: Option<(String, std::path::PathBuf, f64)> = None;
|
||||
for (point_name, params, file_path) in snapshot {
|
||||
let (is_exact, d) = common::seed_finder::calculate_seed_distance(¶ms, target);
|
||||
// exact_family 路径已在上面处理过（索引已覆盖），这里只关心 global 候选。
|
||||
if !is_exact
|
||||
&& d <= common::seed_finder::MAX_GLOBAL_SEED_DISTANCE
|
||||
&& (global_closest.is_none() || d < global_closest.as_ref().unwrap().2)
|
||||
{
|
||||
let path = std::path::PathBuf::from(&file_path);
|
||||
global_closest = Some((point_name, path, d));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some((name, path, d)) = global_closest {
|
||||
Ok(Some(common::seed_finder::SeedMatch {
|
||||
name,
|
||||
path,
|
||||
distance: d,
|
||||
}))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
//! 进度快照（从原 db.rs 拆分，2026-08-06）。
|
||||
//! `impl Database` 的 进度快照 域方法。共享基础设施（Database struct、连接管理、类型、helper）见父模块 `super`（crate::db）。
|
||||
use super::*;
|
||||
|
||||
impl Database {
|
||||
/// 记录一条进度快照：与上一条快照计数完全一致时不写（去重防膨胀）。
|
||||
/// 返回是否实际写入。由后台循环对每个运行中工作流调用。
|
||||
pub async fn record_progress_snapshot(&self, wf: &str) -> Result<bool> {
|
||||
let base = self.get_grid_summary_stats(Some(wf)).await?;
|
||||
let g = |k: &str| base.get(k).and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
let cur = (
|
||||
g("total"),
|
||||
g("pending"),
|
||||
g("queued"),
|
||||
g("running"),
|
||||
g("completed"),
|
||||
g("failed"),
|
||||
);
|
||||
let pool = self.pool.clone();
|
||||
let wf = wf.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let mut conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 审查修复 #M1：原 SELECT last + INSERT 非原子，两个并发后台循环会各自读到
|
||||
// last != cur 各自 INSERT 相同快照，去重失效导致快照表膨胀。现用单条
|
||||
// INSERT ... WHERE NOT EXISTS 原子去重（单语句天然原子，无需 IMMEDIATE——
|
||||
// 写语句在提交时持有写锁，WHERE NOT EXISTS 的读与写同锁内一致）。
|
||||
let tx = conn.transaction()?;
|
||||
let inserted = tx.execute(
|
||||
"INSERT INTO workflow_progress_snapshots
|
||||
(workflow_name, total, pending, queued, running, completed, failed)
|
||||
SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM workflow_progress_snapshots
|
||||
WHERE workflow_name = ?1
|
||||
AND total = ?2 AND pending = ?3 AND queued = ?4
|
||||
AND running = ?5 AND completed = ?6 AND failed = ?7
|
||||
)",
|
||||
params![wf, cur.0, cur.1, cur.2, cur.3, cur.4, cur.5],
|
||||
)?;
|
||||
tx.commit()?;
|
||||
Ok(inserted > 0)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 清理超过保留期（天）的进度快照。返回无意义计数以外的错误。
|
||||
pub async fn purge_progress_snapshots(&self, keep_days: u64) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"DELETE FROM workflow_progress_snapshots WHERE ts < datetime('now', ?1)",
|
||||
params![format!("-{} days", keep_days)],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 最近 `hours` 小时的进度时间序列（ts 升序）。
|
||||
pub async fn get_progress_series(&self, wf: &str, hours: i64) -> Result<Vec<ProgressPoint>> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = wf.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<ProgressPoint>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT ts, total, pending, queued, running, completed, failed
|
||||
FROM workflow_progress_snapshots
|
||||
WHERE workflow_name = ?1 AND ts >= datetime('now', ?2)
|
||||
ORDER BY ts ASC, id ASC",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![wf, format!("-{} hours", hours)], |r| {
|
||||
Ok(ProgressPoint {
|
||||
ts: r.get(0)?,
|
||||
total: r.get(1)?,
|
||||
pending: r.get(2)?,
|
||||
queued: r.get(3)?,
|
||||
running: r.get(4)?,
|
||||
completed: r.get(5)?,
|
||||
failed: r.get(6)?,
|
||||
})
|
||||
})?;
|
||||
let mut series = Vec::new();
|
||||
for r in rows {
|
||||
series.push(r?);
|
||||
}
|
||||
Ok(series)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,603 @@
|
||||
//! 任务/结算（从原 db.rs 拆分，2026-08-06）。
|
||||
//! `impl Database` 的 任务/结算 域方法。共享基础设施（Database struct、连接管理、类型、helper）见父模块 `super`（crate::db）。
|
||||
use super::*;
|
||||
|
||||
impl Database {
|
||||
/// 判断某个网格点在特定工作流内是否已经派发过种子步进 (seed_step) 任务。
|
||||
///
|
||||
/// 注意：**生产回退逻辑已改用策略链弹栈**（`pop_stage_strategy_for_fallback` +
|
||||
/// `trigger_strategy_fallback`，"种子回退仅一次"守卫由「链耗尽 → 保持 failed」取代）。
|
||||
/// 本方法仅保留作**测试断言辅助**（api_tests / db.rs 单测核验"全程未产生 seed_step 行"）。
|
||||
///
|
||||
/// 判定口径（Phase 6 起派生）：`json_extract(tlusty_strategies, '$[0]') = 'seed_step'`——
|
||||
/// task_type 列已删除，策略链首项即"当前执行策略"的权威快照。
|
||||
///
|
||||
/// 为何计数 pending（2026-08-02 涡旋事故定稿，回退 2026-08-01 的 pending 排除）：
|
||||
/// pending 的 seed_step 行要么是正在排队/在途的真任务（计数它正是对在途回退的
|
||||
/// 去重——否则救援途中收到的迟到失败报告会触发第二份 seed_step 重复派发）；
|
||||
/// 要么是"无队列行"的僵尸行（insert_task 后 push 前崩溃等）。2026-08-01 事故中
|
||||
/// 僵尸行曾永久堵死 21 个点的正当回退，当时的 pending 排除是应急解法；本修复后
|
||||
/// 僵尸行被结构性清除——stop/重启卫生（clear_queue_by_workflow 返回 ids 同步删
|
||||
/// tasks 行）、派发去重（schedule 前 MQ 活性校验删死行）、回退内僵尸卫生
|
||||
/// （trigger_strategy_fallback 先删死行再弹链）——pending 行不再可能是僵尸，
|
||||
/// 计数它既安全又必要。
|
||||
pub async fn has_seed_step_attempt(&self, name: &str, workflow_name: &str) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf_owned = workflow_name.to_string();
|
||||
let exists = tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT 1 FROM tasks WHERE point_name = ?1 AND json_extract(tlusty_strategies, '$[0]') = 'seed_step' AND workflow_name = ?2 LIMIT 1",
|
||||
)?;
|
||||
let res = stmt.exists(params![name_owned, wf_owned])?;
|
||||
Ok(res)
|
||||
})
|
||||
.await??;
|
||||
Ok(exists)
|
||||
}
|
||||
|
||||
/// 幂等上报吸收（见 api/task.rs report_task）：查 tasks 表中某任务是否**已由本节点结算**
|
||||
/// （node_id 归属匹配 + status 为终态）。首轮上报成功后 `remove_task` 清掉了 MQ 领用行，
|
||||
/// 若响应在链路上丢失，节点重试上报会经 `verify_task_claim` 落空——此时凭本方法判定为
|
||||
/// 已结算的幂等重放，返回 200 并补写种子，而非 403 误诊「token 失效」。
|
||||
///
|
||||
/// 返回 `(point_name, workflow_name)` 供调用方复用定向校验；未命中返回 None。
|
||||
pub async fn find_settled_task_claim(
|
||||
&self,
|
||||
task_id: &str,
|
||||
node_id: &str,
|
||||
) -> Result<Option<(String, Option<String>)>> {
|
||||
let pool = self.pool.clone();
|
||||
let tid = task_id.to_string();
|
||||
let nid = node_id.to_string();
|
||||
let row = tokio::task::spawn_blocking(move || -> Result<Option<(String, Option<String>)>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT point_name, workflow_name FROM tasks
|
||||
WHERE task_id = ?1 AND node_id = ?2 AND status IN ('completed', 'failed', 'timeout')
|
||||
LIMIT 1",
|
||||
)?;
|
||||
Ok(stmt
|
||||
.query_row(params![tid, nid], |r| {
|
||||
Ok((r.get::<_, String>(0)?, r.get::<_, Option<String>>(1)?))
|
||||
})
|
||||
.ok())
|
||||
})
|
||||
.await??;
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
/// 策略链自动回退（见 docs/task_engine_decoupling_design.md §4.2）：
|
||||
/// 读取指定网格点最近一条**已上报**（非 pending）tasks 行的 `tlusty_strategies`，
|
||||
/// 弹出首项，返回 `(剩余链, 被弹出的策略)`。
|
||||
///
|
||||
/// 兼容别名：`trigger_strategy_fallback` 的 TLUSTY 分支使用（failed_stage 为
|
||||
/// "tlusty" 或缺省时）。SYNSPEC 链的弹栈走 `pop_stage_strategy_for_fallback(_, _, "synspec")`。
|
||||
pub async fn pop_tlusty_strategy_for_fallback(
|
||||
&self,
|
||||
name: &str,
|
||||
workflow_name: &str,
|
||||
) -> Result<Option<FallbackSnapshot>> {
|
||||
self.pop_stage_strategy_for_fallback(name, workflow_name, "tlusty")
|
||||
.await
|
||||
}
|
||||
|
||||
/// 策略链自动回退（阶段参数化版，见 docs/task_engine_decoupling_design.md §4.2 注）：
|
||||
/// 与 `pop_tlusty_strategy_for_fallback` 语义一致，但按 `stage` 选择弹哪条链——
|
||||
/// `"synspec"` → `synspec_strategies` 列，其余 → `tlusty_strategies` 列。
|
||||
///
|
||||
/// 设计 §4.2 注明确要求 SYNSPEC 策略链的自动弹栈与 TLUSTY 完全一致（若配置了多策略链），
|
||||
/// 由失败阶段归因（TaskReport.failed_stage）决定弹哪条链。
|
||||
///
|
||||
/// **只读不改写**（见审查修复）：旧实现把剩余链写回旧行，随后调度器 insert_task
|
||||
/// 新建一行携带相同链 → 两行共享同一链、旧行的 strategies 变陈旧。现改为纯读取：
|
||||
/// 调度器拿到剩余链后构造新 TaskSpec（insert_task 写入新行携带剩余链），旧行保持
|
||||
/// 原状（记录该任务实际执行的策略，审计正确）。
|
||||
///
|
||||
/// **并发安全**（审查修复 #C2）：本方法本身非原子（内存 remove(0) 不落库），并发调用
|
||||
/// 会各自读到完整链、各自派发回退任务。现由调用方 `trigger_strategy_fallback` 持有
|
||||
/// `schedule_lock` 串行化，且 fallback 的「status == failed」状态守卫保证第二次调用
|
||||
/// 进入时点已被首次 fallback 改为 queued 而被拦截。故「只读 + 锁 + 状态守卫」组合
|
||||
/// 在保证审计正确性的同时消除了并发重复派发，无需改写旧行。
|
||||
///
|
||||
/// **返回派发时快照（[`FallbackSnapshot`]）**：策略链 + 被弹策略 + 该行落库的 policy。
|
||||
/// policy 与策略链同源（同一行、同一时刻派发），供回退决策（SkipFailed 门控）与重试
|
||||
/// 任务构造使用——回退行为由派发时配置决定，不随运行期 YAML 编辑漂移（对齐 §4.2
|
||||
/// 「不修改原有 policy，保持用户初始配置」）。
|
||||
///
|
||||
/// 过滤 `status IN ('failed','timeout')`：只有**失败**任务才触发回退、其策略链才应被弹。
|
||||
/// L4 修复：原 `status != 'pending'` 也会命中已完成的 `completed` 行——在孤儿回收重派
|
||||
/// 场景下，「最新已上报行」可能是一条 stale 的 completed 行，其链与刚失败触发回退的任务
|
||||
/// 不符，弹出会移除错误链的顺位。收紧到失败/超时行后，弹栈始终对准触发回退的失败任务。
|
||||
/// pending 行（未上报僵尸/在途）与 completed 行（已成功、链已消费完）均不参与弹栈。
|
||||
///
|
||||
/// 仅匹配精确 workflow_name（移除旧 `IS NULL / = ''` 兜底子句——多工作流分区迁移
|
||||
/// 已把历史行回填为 `__legacy__`，NULL 兜底反而让同名点跨工作流命中 legacy 行）。
|
||||
pub async fn pop_stage_strategy_for_fallback(
|
||||
&self,
|
||||
name: &str,
|
||||
workflow_name: &str,
|
||||
stage: &str,
|
||||
) -> Result<Option<FallbackSnapshot>> {
|
||||
// 阶段 → 策略链列 / policy 列的白名单映射（防注入：非法 stage 兜底为 tlusty 列）。
|
||||
let (col, policy_col) = match stage {
|
||||
"synspec" => ("synspec_strategies", "synspec_policy"),
|
||||
_ => ("tlusty_strategies", "tlusty_policy"),
|
||||
};
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
let sql = format!(
|
||||
"SELECT {col}, {policy_col} FROM tasks
|
||||
WHERE point_name = ?1 AND workflow_name = ?2 AND status IN ('failed', 'timeout')
|
||||
ORDER BY created_at DESC, rowid DESC LIMIT 1"
|
||||
);
|
||||
tokio::task::spawn_blocking(move || -> Result<Option<FallbackSnapshot>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 只读最新已上报行（非 pending），单条查询无需事务。
|
||||
let row: Option<(String, String)> = {
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
stmt.query_row(params![name_owned, wf], |r| {
|
||||
Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))
|
||||
})
|
||||
.ok()
|
||||
};
|
||||
let Some((json_str, policy_str)) = row else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut strategies: Vec<String> = serde_json::from_str(&json_str).unwrap_or_default();
|
||||
if strategies.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let popped = strategies.remove(0);
|
||||
Ok(Some(FallbackSnapshot {
|
||||
rest_strategies: strategies,
|
||||
popped,
|
||||
policy: ResumePolicy::from_str_lossy(&policy_str),
|
||||
}))
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 读取指定网格点最近一条 tasks 行的 TLUSTY 策略链（不修改）。供回退守卫判断
|
||||
/// 「策略链是否已含 seed_step 且为当前执行策略」等。无行返回空 Vec。
|
||||
pub async fn get_latest_tlusty_strategies(
|
||||
&self,
|
||||
name: &str,
|
||||
workflow_name: &str,
|
||||
) -> Result<Vec<String>> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
let strategies = tokio::task::spawn_blocking(move || -> Result<Vec<String>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let json_str: Option<String> = {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT tlusty_strategies FROM tasks
|
||||
WHERE point_name = ?1 AND workflow_name = ?2
|
||||
ORDER BY created_at DESC, rowid DESC LIMIT 1",
|
||||
)?;
|
||||
stmt.query_row(params![name_owned, wf], |r| r.get::<_, String>(0))
|
||||
.ok()
|
||||
};
|
||||
let json_str = match json_str {
|
||||
Some(s) => s,
|
||||
None => return Ok(Vec::new()),
|
||||
};
|
||||
Ok(serde_json::from_str(&json_str).unwrap_or_default())
|
||||
})
|
||||
.await??;
|
||||
Ok(strategies)
|
||||
}
|
||||
|
||||
pub async fn insert_task(&self, spec: &common::models::TaskSpec) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let spec = spec.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 阶段配置打平入库（见 docs/task_engine_decoupling_design.md §4.1）。
|
||||
// strategies 序列化为 JSON 数组文本列；policy 取 as_str。
|
||||
// Phase 6 起无 task_type 列，当前执行策略由 tlusty_strategies[0] 派生。
|
||||
let tlusty_enabled = spec.tlusty_config.enabled;
|
||||
let tlusty_policy = spec.tlusty_config.policy.as_str();
|
||||
let tlusty_strategies = serde_json::to_string(&spec.tlusty_config.strategies)?;
|
||||
let synspec_enabled = spec.synspec_config.enabled;
|
||||
let synspec_policy = spec.synspec_config.policy.as_str();
|
||||
let synspec_strategies = serde_json::to_string(&spec.synspec_config.strategies)?;
|
||||
conn.execute(
|
||||
"INSERT INTO tasks (task_id, point_name, seed_point_name, status, created_at, workflow_name,
|
||||
tlusty_enabled, tlusty_policy, tlusty_strategies,
|
||||
synspec_enabled, synspec_policy, synspec_strategies, atmosphere_ref)
|
||||
VALUES (?1, ?2, ?3, 'pending', datetime('now'), ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)
|
||||
ON CONFLICT(task_id) DO NOTHING",
|
||||
params![
|
||||
spec.task_id.to_string(),
|
||||
spec.point_name,
|
||||
spec.seed_point_name,
|
||||
spec.workflow_name,
|
||||
tlusty_enabled,
|
||||
tlusty_policy,
|
||||
tlusty_strategies,
|
||||
synspec_enabled,
|
||||
synspec_policy,
|
||||
synspec_strategies,
|
||||
spec.atmosphere_ref,
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 删除 tasks 历史表中指定 task_id 的行。
|
||||
///
|
||||
/// 用于调度回滚：当 push_task 失败时，grid_points 已回滚、queue 已清理，
|
||||
/// 但先于 push 插入的 tasks 历史行（status='pending'）会遗留，污染每点尝试计数统计。
|
||||
/// 此方法在回滚路径中调用以保持三者一致。
|
||||
pub async fn delete_task(&self, task_id: &uuid::Uuid) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let id = task_id.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute("DELETE FROM tasks WHERE task_id = ?1", params![id])?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 列出指定网格点在当前工作流内仍处于 `pending` 的 tasks 行 task_id。
|
||||
///
|
||||
/// tasks 行的 pending 意为"已创建但从未上报"，可能是真在途（队列行仍 pending/
|
||||
/// claimed）或僵尸（队列行已不存在）。调用方须配合 SqliteTaskQueue::task_row_exists
|
||||
/// 做 MQ 活性交叉校验区分二者（2026-08-02 涡旋事故修复引入）：
|
||||
/// - 派发去重：任一活 → 跳过派发；全死 → 清僵尸后正常派发。
|
||||
/// - 孤儿回收 / 回退种子卫生：同上分流。
|
||||
///
|
||||
/// `strategy` 传 Some("seed_step")/Some("cold_run") 时按当前策略过滤（Phase 6 起派生：
|
||||
/// 比较 `json_extract(tlusty_strategies, '$[0]')`，不再依赖已删除的 task_type 列），
|
||||
/// None 不过滤。生产调用仅传 None，Some 分支供测试断言用。
|
||||
pub async fn has_pending_tasks_for_point(
|
||||
&self,
|
||||
point_name: &str,
|
||||
workflow_name: &str,
|
||||
strategy: Option<&str>,
|
||||
) -> Result<Vec<String>> {
|
||||
let pool = self.pool.clone();
|
||||
let point = point_name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
let tt = strategy.map(|s| s.to_string());
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<String>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut ids = Vec::new();
|
||||
match tt {
|
||||
Some(tt) => {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT task_id FROM tasks WHERE point_name = ?1 AND workflow_name = ?2 AND status = 'pending' AND json_extract(tlusty_strategies, '$[0]') = ?3",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![point, wf, tt], |r| r.get::<_, String>(0))?;
|
||||
for r in rows {
|
||||
ids.push(r?);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT task_id FROM tasks WHERE point_name = ?1 AND workflow_name = ?2 AND status = 'pending'",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![point, wf], |r| r.get::<_, String>(0))?;
|
||||
for r in rows {
|
||||
ids.push(r?);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(ids)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 批量删除 tasks 表中指定 task_id 且仍为 `pending` 的行，返回删除行数。
|
||||
///
|
||||
/// 用于僵尸行清理（派发去重 / 孤儿回收 / stop·重启卫生）。`AND status = 'pending'`
|
||||
/// 是 TOCTOU 防护：校验活性与删除之间若有上报事务恰好提交（行变 completed/failed），
|
||||
/// 本删除不会误删已完成的审计历史（attempt_count/阶段归因列均不受影响）。
|
||||
/// 空切片短路返回 0；按 500 个一批分块（SQLite 默认宿主参数上限 999）。
|
||||
pub async fn delete_tasks_by_ids(&self, task_ids: &[String]) -> Result<usize> {
|
||||
if task_ids.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let pool = self.pool.clone();
|
||||
let ids = task_ids.to_vec();
|
||||
tokio::task::spawn_blocking(move || -> Result<usize> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut total = 0usize;
|
||||
for chunk in ids.chunks(500) {
|
||||
let placeholders = vec!["?"; chunk.len()].join(", ");
|
||||
let sql = format!(
|
||||
"DELETE FROM tasks WHERE task_id IN ({}) AND status = 'pending'",
|
||||
placeholders
|
||||
);
|
||||
let params: Vec<&dyn rusqlite::ToSql> =
|
||||
chunk.iter().map(|s| s as &dyn rusqlite::ToSql).collect();
|
||||
total += conn.execute(&sql, params.as_slice())?;
|
||||
}
|
||||
Ok(total)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 记录节点上报的任务结果。返回网格点状态是否发生**实际迁移**（bool）：
|
||||
/// 上层 report_task 据此决定失败报告是否触发种子回退——被终态守卫吸收的
|
||||
/// 重复报告返回 false，不再触发回退（2026-08-02 涡旋事故修复）。
|
||||
pub async fn record_task_report(
|
||||
&self,
|
||||
report: &TaskReport,
|
||||
workflow_name: &str,
|
||||
) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let report_cloned = report.clone();
|
||||
let point_name = report.point_name.clone();
|
||||
let wf = workflow_name.to_string();
|
||||
let converged = report.result_valid;
|
||||
let atmo_has_nan = report.atmosphere_has_nan;
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let mut conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let tx = conn.transaction()?;
|
||||
|
||||
let status_str = match report_cloned.status {
|
||||
TaskStatus::Completed => "completed",
|
||||
TaskStatus::Failed => "failed",
|
||||
TaskStatus::Timeout => "timeout",
|
||||
_ => "pending",
|
||||
};
|
||||
|
||||
// P1（Phase 1）：结算补落阶段信息。failed_stage 仅对失败/超时任务落库（缺省兜底
|
||||
// 'tlusty'，旧节点不携带该字段 → 按 TLUSTY 链回退，与调度端 infer_failed_stage
|
||||
// 默认一致）；成功任务显式置 NULL，避免污染按 failed_stage 过滤的审计/统计
|
||||
// （审查修复：旧实现 COALESCE 无条件填充，成功任务也被写入 'tlusty'）。
|
||||
// summary_json 透传 ModelSummary 序列化（错误路径为 {"error": ...}，服务端只透传不解析）。
|
||||
let is_failure = matches!(
|
||||
report_cloned.status,
|
||||
TaskStatus::Failed | TaskStatus::Timeout
|
||||
);
|
||||
let failed_stage_value: Option<&str> = if is_failure {
|
||||
report_cloned.failed_stage.as_deref().or(Some("tlusty"))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// H2 修复：任务结算去重守卫。并发重复上报同一 task 时，仅首个报告能把 tasks 行
|
||||
// 从非终态置为终态（WHERE status NOT IN 终态）。若 UPDATE 命中 0 行且任务行仍存在，
|
||||
// 说明该 task 已被先前报告结算为终态 → 本报告为重复结算 → 直接返回 changed=false、
|
||||
// 跳过网格点翻转。这消除了「report2 把 report1 回退后已置为 queued 的网格点再翻回
|
||||
// failed、重复触发 seed_step 回退」的 TOCTOU（2026-08-02 涡旋事故的并发残余）：
|
||||
// 两个报告即便都通过 API 层的 verify_task_claim（纯 SELECT 不消费），也只有一个
|
||||
// 会在 DB 层把任务置为终态并结算网格点。
|
||||
let task_updated = tx.execute(
|
||||
"UPDATE tasks SET status = ?1, node_id = ?2, max_relc = ?3, atmosphere_has_nan = ?4, completed_at = datetime('now'), error_message = ?5, elapsed_sec = ?6, failed_stage = ?8, summary_json = ?9 WHERE task_id = ?7 AND status NOT IN ('completed', 'failed', 'timeout')",
|
||||
params![
|
||||
status_str,
|
||||
report_cloned.node_id,
|
||||
report_cloned.max_relc,
|
||||
report_cloned.atmosphere_has_nan,
|
||||
report_cloned.error_message,
|
||||
report_cloned.elapsed_sec,
|
||||
report_cloned.task_id.to_string(),
|
||||
failed_stage_value,
|
||||
report_cloned.summary_json,
|
||||
],
|
||||
)?;
|
||||
|
||||
if task_updated == 0 {
|
||||
// 0 行更新：要么任务行已终态（重复结算），要么任务行不存在（被清理）。
|
||||
// 区分二者：任务行仍存在则为重复结算，跳过网格点翻转（首次结算的审计信息
|
||||
// 已保留在任务行）；任务行不存在则继续走原逻辑（策略读取回落到安全默认）。
|
||||
let task_exists = match tx.query_row(
|
||||
"SELECT 1 FROM tasks WHERE task_id = ?1",
|
||||
params![report_cloned.task_id.to_string()],
|
||||
|_| Ok(true),
|
||||
) {
|
||||
Ok(_) => true,
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => false,
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
if task_exists {
|
||||
tx.rollback()?;
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
|
||||
// 失败次数计数自增（attempt_count 仅作观测/统计用途，保留原子 UPDATE 避免并发竞态）。
|
||||
// 注意：状态迁移不再依赖该计数值（失败统一置 failed，是否回退复活由
|
||||
// trigger_strategy_fallback 按策略链弹栈 + policy 快照决定，见 db.rs FallbackSnapshot）。
|
||||
// 三条 UPDATE grid_points 均带 workflow_name 过滤，避免跨工作流误改同名点。
|
||||
//
|
||||
// 审查修复 #S2：原 `unwrap_or(0)` 把任何 rusqlite::Error（锁超时、磁盘 I/O）吞成 0。
|
||||
// 现区分 QueryReturnedNoRows（grid_points 无该点行，返回 0 合理）与其余错误（上抛）。
|
||||
match tx.query_row(
|
||||
"UPDATE grid_points SET attempt_count = attempt_count + 1 WHERE name = ?1 AND workflow_name = ?2 RETURNING attempt_count",
|
||||
params![point_name, wf],
|
||||
|r| r.get::<_, i32>(0),
|
||||
) {
|
||||
Ok(_) => {}
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
|
||||
// 成功归因（修复审查 #4 + Phase 6 派生口径 + Phase 5a + P9 命名拆分）：
|
||||
// - tlusty_success_method：TLUSTY 阶段以何策略收敛。tlusty_enabled=1 时取
|
||||
// `tlusty_strategies[0]`（cold_run/seed_step）；TLUSTY 禁用（synspec-only）为 NULL。
|
||||
// - synspec_success_method（5a）：光谱阶段以何策略收敛。synspec_enabled=1 时取
|
||||
// `synspec_strategies[0]`（如 "standard"）；TLUSTY-only 成功保持 NULL。
|
||||
// 成功分支（status=Completed）已由 derive_report_status 保证「synspec 启用则无错」，
|
||||
// 故"synspec 成功" ⟺ synspec_enabled=1，无需再查 synspec_rc。
|
||||
// 整体归因（原 success_method 值域混用）改为消费方派生：tlusty ?? synspec。
|
||||
// 注：依赖 SQLite 内建 JSON1 的 json_extract（rusqlite bundled SQLite 默认启用）。
|
||||
//
|
||||
// 审查修复 #S2（严重）：原 `unwrap_or((None, None, true, true))` 把任何 DB 错误
|
||||
// （锁超时、I/O）当成「行不存在」，用 tlusty_enabled=true/synspec_enabled=true 默认值
|
||||
// 继续写阶段状态，故障期会静默写入错误归因数据。现区分 QueryReturnedNoRows（行不存在，
|
||||
// 用安全默认）与其余错误（上抛）。
|
||||
let (tlusty_success_method, synspec_success_method, tlusty_enabled, synspec_enabled): (
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
bool,
|
||||
bool,
|
||||
) = match tx.query_row(
|
||||
"SELECT
|
||||
CASE WHEN tlusty_enabled = 1
|
||||
THEN json_extract(tlusty_strategies, '$[0]')
|
||||
END,
|
||||
CASE WHEN synspec_enabled = 1
|
||||
THEN json_extract(synspec_strategies, '$[0]')
|
||||
END,
|
||||
tlusty_enabled,
|
||||
synspec_enabled
|
||||
FROM tasks WHERE task_id = ?1",
|
||||
params![report_cloned.task_id.to_string()],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
||||
) {
|
||||
Ok(v) => v,
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => (None, None, true, true),
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
|
||||
// Phase 5b：阶段状态（§7.2 语义表）。NULL = 阶段不适用（enabled=0）。
|
||||
// 整体 grid_points.status 仍是权威状态，阶段列是补充可查信息。
|
||||
// **阶段列值固定用 'converged'**（阶段收敛语义，与 StageSummary.converged 一致）——
|
||||
// claim/running 守卫按 'converged' 保留终态，二者必须匹配（审查 CRITICAL#1：7c 曾误
|
||||
// 把这里改成 'completed' 导致半失败守卫失效，已回退）。
|
||||
let pipe_ok = report_cloned.status == TaskStatus::Completed && converged && !atmo_has_nan;
|
||||
let (tlusty_status, synspec_status): (Option<&str>, Option<&str>) = if pipe_ok {
|
||||
// 整管线成功：启用阶段均 converged。
|
||||
(
|
||||
tlusty_enabled.then_some("converged"),
|
||||
synspec_enabled.then_some("converged"),
|
||||
)
|
||||
} else if converged {
|
||||
// 半失败：大气收敛 + 光谱失败。tlusty 阶段确已收敛（tlusty_enabled 防御性守卫——
|
||||
// 该分支在 synspec-only 下不可达，但避免未来代码路径把禁用阶段误标为 converged）。
|
||||
(tlusty_enabled.then_some("converged"), Some("failed"))
|
||||
} else if tlusty_enabled {
|
||||
// 大气未收敛（TLUSTY 启用）→ TLUSTY 失败、SYNSPEC 未运行。
|
||||
(Some("failed"), Some("pending"))
|
||||
} else {
|
||||
// synspec-only 失败（converged 由 synspec_rc 决定）。
|
||||
(None, Some("failed"))
|
||||
};
|
||||
|
||||
// 终态守卫（2026-08-02 涡旋事故修复）：
|
||||
// - 成功分支：converged 为吸收态，迟到的重复成功报告不覆盖阶段归因列；
|
||||
// 允许 failed/queued/pending/running → converged（迟到的真收敛结果获胜，
|
||||
// 含种子救援路径 failed → converged）。
|
||||
// - 失败分支：NOT IN ('completed','failed')——converged 不可被迟到失败报告翻黑
|
||||
// （事故直接症状）；failed 吸收重复失败报告（changed=0 → 不触发重复回退）。
|
||||
// queued/pending → failed 是正确语义：能通过 verify_task_claim 的报告必为
|
||||
// 真实领用并计算过的任务（如回收器重置后在途任务才上报、领用后 mark 失败等）。
|
||||
// - 阶段列（5b）：tlusty_status 用 CASE 守卫保留既有终态——半失败重试的 synspec-only
|
||||
// 任务（tlusty 关闭）不覆盖已收敛的 tlusty_status（设计 §7.3 打开项 #2：仅 synspec
|
||||
// 侧流转）；synspec_status 自由流转。
|
||||
let grid_changed = if pipe_ok {
|
||||
tx.execute(
|
||||
"UPDATE grid_points SET status = 'completed', last_elapsed_sec = ?1, tlusty_success_method = ?2, synspec_success_method = ?3,
|
||||
tlusty_status = CASE WHEN ?6 IS NOT NULL THEN ?6 ELSE tlusty_status END,
|
||||
synspec_status = ?7
|
||||
WHERE name = ?4 AND workflow_name = ?5 AND status != 'completed'",
|
||||
params![
|
||||
report_cloned.elapsed_sec,
|
||||
tlusty_success_method,
|
||||
synspec_success_method,
|
||||
point_name,
|
||||
wf,
|
||||
tlusty_status,
|
||||
synspec_status
|
||||
],
|
||||
)?
|
||||
} else {
|
||||
tx.execute(
|
||||
"UPDATE grid_points SET status = 'failed', last_elapsed_sec = ?1,
|
||||
tlusty_status = CASE WHEN ?4 IS NOT NULL THEN ?4 ELSE tlusty_status END,
|
||||
synspec_status = ?5
|
||||
WHERE name = ?2 AND workflow_name = ?3 AND status NOT IN ('completed', 'failed')",
|
||||
params![report_cloned.elapsed_sec, point_name, wf, tlusty_status, synspec_status],
|
||||
)?
|
||||
};
|
||||
|
||||
// 审查修复 #S1：workflow 完成 flip 并入同一事务（原实现在 tx.commit() 后单独 UPDATE，
|
||||
// 崩溃窗口期 task 已终态但 workflow 卡 running；且 `let _ =` 丢弃 I/O 错误）。
|
||||
// 现在事务内更新，与 task/grid_points 结算原子提交，错误正常传播。
|
||||
tx.execute(
|
||||
"UPDATE workflows
|
||||
SET status = 'completed', updated_at = datetime('now')
|
||||
WHERE name = ?1
|
||||
AND status = 'running'
|
||||
AND EXISTS (SELECT 1 FROM grid_points WHERE workflow_name = workflows.name)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM grid_points
|
||||
WHERE workflow_name = workflows.name
|
||||
AND status IN ('pending', 'queued', 'running')
|
||||
)",
|
||||
params![wf],
|
||||
)?;
|
||||
|
||||
tx.commit()?;
|
||||
|
||||
Ok(grid_changed > 0)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 单点的全部任务尝试（created_at 升序）。
|
||||
///
|
||||
/// 严格匹配 `workflow_name = ?`：分区迁移前的历史任务（NULL/''）不会出现，
|
||||
/// 这是有意的多工作流隔离语义。
|
||||
pub async fn list_point_attempts(&self, wf: &str, point: &str) -> Result<Vec<AttemptRow>> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = wf.to_string();
|
||||
let point = point.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<AttemptRow>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT task_id, seed_point_name, status, max_relc,
|
||||
atmosphere_has_nan, node_id, error_message, created_at, completed_at,
|
||||
elapsed_sec, failed_stage, summary_json
|
||||
FROM tasks
|
||||
WHERE point_name = ?1 AND workflow_name = ?2
|
||||
ORDER BY created_at ASC, completed_at ASC",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![point, wf], |r| {
|
||||
Ok(AttemptRow {
|
||||
task_id: r.get(0)?,
|
||||
seed_point_name: r.get(1)?,
|
||||
status: r.get(2)?,
|
||||
max_relc: r.get(3)?,
|
||||
atmosphere_has_nan: r.get(4)?,
|
||||
node_id: r.get(5)?,
|
||||
error_message: r.get(6)?,
|
||||
created_at: r.get(7)?,
|
||||
completed_at: r.get(8)?,
|
||||
elapsed_sec: r.get(9)?,
|
||||
failed_stage: r.get(10)?,
|
||||
summary_json: r.get(11)?,
|
||||
})
|
||||
})?;
|
||||
let mut attempts = Vec::new();
|
||||
for r in rows {
|
||||
attempts.push(r?);
|
||||
}
|
||||
Ok(attempts)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
//! 工作流（从原 db.rs 拆分，2026-08-06）。
|
||||
//! `impl Database` 的 工作流 域方法。共享基础设施（Database struct、连接管理、类型、helper）见父模块 `super`（crate::db）。
|
||||
use super::*;
|
||||
|
||||
impl Database {
|
||||
pub async fn upsert_workflow(
|
||||
&self,
|
||||
name: &str,
|
||||
description: Option<&str>,
|
||||
config_yaml: &str,
|
||||
status: &str,
|
||||
) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let desc_owned = description.map(|s| s.to_string());
|
||||
let yaml_owned = config_yaml.to_string();
|
||||
let status_owned = status.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"INSERT INTO workflows (name, description, config_yaml, status, created_at, updated_at)
|
||||
VALUES (?1, ?2, ?3, ?4, datetime('now'), datetime('now'))
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
description = excluded.description,
|
||||
config_yaml = excluded.config_yaml,
|
||||
status = CASE WHEN workflows.status = 'running' THEN workflows.status ELSE excluded.status END,
|
||||
updated_at = datetime('now')",
|
||||
params![name_owned, desc_owned, yaml_owned, status_owned],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 自动巡检所有处于 running 状态的工作流：
|
||||
/// 若某个工作流下的所有网格点均已到达终态（无 pending/queued/running 点），
|
||||
/// 则自动将该工作流的数据库 status 翻转为 'completed'。
|
||||
pub async fn sync_all_running_workflows_completion(&self) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// L6 修复：原 `let _ =` 丢弃 r2d2/rusqlite 错误——若该后台对账 UPDATE 失败，
|
||||
// 工作流可持续卡在 running 无任何提示。现记录错误（仍返回 Ok 不中断主流程，
|
||||
// 因为每份上报内的 workflow-completion flip 才是主路径，见 record_task_report）。
|
||||
if let Err(e) = conn.execute(
|
||||
"UPDATE workflows
|
||||
SET status = 'completed', updated_at = datetime('now')
|
||||
WHERE status = 'running'
|
||||
AND EXISTS (SELECT 1 FROM grid_points WHERE workflow_name = workflows.name)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM grid_points
|
||||
WHERE workflow_name = workflows.name
|
||||
AND status IN ('pending', 'queued', 'running')
|
||||
)",
|
||||
[],
|
||||
) {
|
||||
tracing::error!("后台对账：同步 running 工作流完成态失败: {}", e);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
pub async fn list_workflows(&self) -> Result<Vec<WorkflowSummary>> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<WorkflowSummary>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT name, description, status, created_at, updated_at FROM workflows ORDER BY updated_at DESC"
|
||||
)?;
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok(WorkflowSummary {
|
||||
name: row.get(0)?,
|
||||
description: row.get(1)?,
|
||||
status: row.get(2)?,
|
||||
created_at: row.get(3)?,
|
||||
updated_at: row.get(4)?,
|
||||
stats: None,
|
||||
})
|
||||
})?;
|
||||
|
||||
let mut list = Vec::new();
|
||||
for r in rows {
|
||||
list.push(r?);
|
||||
}
|
||||
|
||||
// 一次 GROUP BY 聚合全部工作流的网格计数并回填（不做逐工作流查询，无 N+1）。
|
||||
let mut agg_stmt = conn.prepare(
|
||||
"SELECT workflow_name,
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) AS completed,
|
||||
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed,
|
||||
SUM(CASE WHEN status = 'running' THEN 1 ELSE 0 END) AS running,
|
||||
SUM(CASE WHEN status = 'completed' AND tlusty_success_method = 'cold_run' THEN 1 ELSE 0 END) AS cold,
|
||||
SUM(CASE WHEN status = 'completed' AND tlusty_success_method = 'seed_step' THEN 1 ELSE 0 END) AS seed,
|
||||
SUM(CASE WHEN status = 'completed' AND synspec_success_method IS NOT NULL THEN 1 ELSE 0 END) AS synspec
|
||||
FROM grid_points GROUP BY workflow_name",
|
||||
)?;
|
||||
let agg_rows = agg_stmt.query_map([], |r| {
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
WorkflowListStats {
|
||||
total: r.get(1)?,
|
||||
completed: r.get::<_, Option<i64>>(2)?.unwrap_or(0),
|
||||
failed: r.get::<_, Option<i64>>(3)?.unwrap_or(0),
|
||||
running: r.get::<_, Option<i64>>(4)?.unwrap_or(0),
|
||||
cold_run_converged: r.get::<_, Option<i64>>(5)?.unwrap_or(0),
|
||||
seed_step_converged: r.get::<_, Option<i64>>(6)?.unwrap_or(0),
|
||||
synspec_converged: r.get::<_, Option<i64>>(7)?.unwrap_or(0),
|
||||
},
|
||||
))
|
||||
})?;
|
||||
let mut stats_map = std::collections::HashMap::new();
|
||||
for r in agg_rows {
|
||||
let (wf, s) = r?;
|
||||
stats_map.insert(wf, s);
|
||||
}
|
||||
for wf in &mut list {
|
||||
wf.stats = stats_map.remove(&wf.name);
|
||||
}
|
||||
|
||||
Ok(list)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
pub async fn get_workflow(&self, name: &str) -> Result<Option<WorkflowItem>> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<Option<WorkflowItem>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT name, description, config_yaml, status, created_at, updated_at FROM workflows WHERE name = ?1"
|
||||
)?;
|
||||
let row = stmt.query_row(params![name_owned], |row| {
|
||||
Ok(WorkflowItem {
|
||||
name: row.get(0)?,
|
||||
description: row.get(1)?,
|
||||
config_yaml: row.get(2)?,
|
||||
status: row.get(3)?,
|
||||
created_at: row.get(4)?,
|
||||
updated_at: row.get(5)?,
|
||||
})
|
||||
});
|
||||
|
||||
match row {
|
||||
Ok(item) => Ok(Some(item)),
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
pub async fn update_workflow_status(&self, name: &str, status: &str) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let status_owned = status.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"UPDATE workflows SET status = ?1, updated_at = datetime('now') WHERE name = ?2",
|
||||
params![status_owned, name_owned],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete_workflow(&self, name: &str) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let mut conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let tx = conn.transaction()?;
|
||||
tx.execute("DELETE FROM workflows WHERE name = ?1", params![name_owned])?;
|
||||
tx.execute(
|
||||
"DELETE FROM grid_points WHERE workflow_name = ?1",
|
||||
params![name_owned],
|
||||
)?;
|
||||
tx.execute(
|
||||
"DELETE FROM tasks WHERE workflow_name = ?1",
|
||||
params![name_owned],
|
||||
)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn has_running_workflow(&self) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let count: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM workflows WHERE status = 'running'",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
Ok(count > 0)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 返回当前处于 running / initializing 状态的**全部**工作流名称。
|
||||
///
|
||||
/// 多工作流并发分区：后台调度需对每个 running 工作流分别派发任务，
|
||||
/// 替代原来「全局只有一个 running workflow」的 LIMIT 1 假设。
|
||||
pub async fn get_running_workflow_names(&self) -> Result<Vec<String>> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<String>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT name FROM workflows WHERE status IN ('running', 'initializing') ORDER BY updated_at ASC",
|
||||
)?;
|
||||
let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;
|
||||
let mut list = Vec::new();
|
||||
for r in rows {
|
||||
list.push(r?);
|
||||
}
|
||||
Ok(list)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 原子切转工作流至 initializing 预占启动状态，杜绝高并发 POST /start 触发双重全量排队与重置网格竞态
|
||||
pub async fn transition_workflow_to_initializing(&self, name: &str) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let affected = tokio::task::spawn_blocking(move || -> Result<usize> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let count = conn.execute(
|
||||
"UPDATE workflows SET status = 'initializing', updated_at = datetime('now') WHERE name = ?1 AND status NOT IN ('running', 'initializing')",
|
||||
params![name_owned],
|
||||
)?;
|
||||
Ok(count)
|
||||
})
|
||||
.await??;
|
||||
Ok(affected > 0)
|
||||
}
|
||||
|
||||
/// 获取运行或启动态中的所有工作流 YAML 配置（替代原来低效 N 次循环与嵌套查询）
|
||||
pub async fn get_running_workflow_config_yamls(&self) -> Result<Vec<String>> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<String>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT config_yaml FROM workflows WHERE status IN ('running', 'initializing')",
|
||||
)?;
|
||||
let rows = stmt.query_map([], |row| row.get(0))?;
|
||||
let mut list = Vec::new();
|
||||
for r in rows {
|
||||
list.push(r?);
|
||||
}
|
||||
Ok(list)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 获取所有处于 `initializing` 态的工作流 (name, config_yaml)。
|
||||
///
|
||||
/// 用于服务端启动恢复：`start_workflow` 把状态切到 `initializing` 后在后台 spawn
|
||||
/// `initialize_grid`。若进程在初始化中途崩溃/重启，工作流会永久卡在 `initializing`
|
||||
/// （`get_running_workflow_names` 仍把它算作可调度，但无人完成网格展开）。
|
||||
/// 启动时检测到这些半初始化工作流后重新跑 `initialize_grid`（幂等，ON CONFLICT DO NOTHING）
|
||||
/// 把状态推进到 `running`，避免半初始化网格被调度。
|
||||
pub async fn get_initializing_workflows(&self) -> Result<Vec<(String, String)>> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<(String, String)>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT name, config_yaml FROM workflows WHERE status = 'initializing'")?;
|
||||
let rows = stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?;
|
||||
let mut list = Vec::new();
|
||||
for r in rows {
|
||||
list.push(r?);
|
||||
}
|
||||
Ok(list)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod api;
|
||||
pub mod cors;
|
||||
pub mod db;
|
||||
pub mod migrations;
|
||||
pub mod scheduler;
|
||||
|
||||
@@ -228,7 +228,17 @@ async fn main() -> Result<()> {
|
||||
// 校验确认真孤儿后重置为 pending 让调度器重新派发，并清除作为判据的
|
||||
// 僵尸 tasks 行（旧实现仅凭 tasks 表 stale pending 行判定，僵尸行使
|
||||
// 判据恒真 → 重复派发涡旋，已废弃）。
|
||||
match bg_scheduler_clone.reclaim_orphaned_points(stale_sec).await {
|
||||
//
|
||||
// 审查修复 #M4：reclaim 用独立且更大的阈值（2 * stale_sec）。reclaim 的语义是
|
||||
// 「孤儿回收」（队列凭证完全丢失），时间尺度应比 requeue 的「claim 超时重投」
|
||||
// 更宽松：刚被 claim 的任务 tasks 行仍 pending，过小阈值会把它误判孤儿候选、
|
||||
// 在 requeue 把队列行打回 pending 到节点重新 claim 的窗口内增加抖动。2x 给
|
||||
// 正常长任务足够缓冲。
|
||||
let reclaim_threshold = stale_sec.saturating_mul(2);
|
||||
match bg_scheduler_clone
|
||||
.reclaim_orphaned_points(reclaim_threshold)
|
||||
.await
|
||||
{
|
||||
Ok(reset) => {
|
||||
if reset > 0 {
|
||||
info!(
|
||||
@@ -494,14 +504,15 @@ async fn security_headers_middleware(
|
||||
|
||||
let headers = resp.headers_mut();
|
||||
// CSP：default-src 'self'；放行 Google Fonts（index.html 引用）；允许 data: 图片。
|
||||
// 已移除 'unsafe-eval'：dashboard 构建产物不使用 eval/new Function（已核实），保留它会
|
||||
// 显著削弱 CSP 的脚本注入防护。'unsafe-inline' 暂留（静态 SPA 内联脚本/handler 需要），
|
||||
// 彻底方案需前端改造为外链 + per-request nonce 注入，见 docs TODO。
|
||||
// 已移除 'unsafe-eval'（dashboard 不用 eval/new Function）与 script-src 'unsafe-inline'
|
||||
// （主题初始化脚本已移为外链 public/theme-init.js，见 dashboard/index.html）。无内联
|
||||
// 脚本/内联事件处理器，外链脚本走 'self' 即可，脚本注入防御最大化。
|
||||
// style-src 保留 'unsafe-inline'：index.html 含内联 style 属性（SVG flex 布局等），移除会破坏渲染。
|
||||
headers
|
||||
.entry(axum::http::header::CONTENT_SECURITY_POLICY)
|
||||
.or_insert_with(|| {
|
||||
HeaderValue::from_static(
|
||||
"default-src 'self'; script-src 'self' 'unsafe-inline'; \
|
||||
"default-src 'self'; script-src 'self'; \
|
||||
style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; \
|
||||
font-src 'self' data: https://fonts.gstatic.com; \
|
||||
connect-src 'self'; img-src 'self' data: blob:; \
|
||||
|
||||
@@ -0,0 +1,565 @@
|
||||
//! 主库版本化迁移基础设施（Phase 0，见 docs/database_refactor_design.md §2）。
|
||||
//!
|
||||
//! 背景：init_tables 内持续堆积手写幂等 ALTER 块，无版本追踪，风险随 schema 演进累积。
|
||||
//! 本模块引入 `PRAGMA user_version` 驱动的版本化迁移，为后续各 Phase 的结构变更
|
||||
//! （P1/P2/P4/P5a/P6）提供统一、可检测、事务化、可中断恢复的迁移通道。
|
||||
//!
|
||||
//! 约定：
|
||||
//! - **V0 = 0**（`PRAGMA user_version` 对全新库的默认值）；后续编号迁移从 1 开始。
|
||||
//! - **新库**：bootstrap 的 CREATE TABLE 始终是最新形态（含各 Phase 新增列）→ 置 V0 →
|
||||
//! 顺序应用 V0+1..N。每个迁移自带 detect 守卫，已存在的列/索引直接跳过 → 新库上所有迁移为 no-op。
|
||||
//! - **旧库**（user_version=0 但表已存在）：bootstrap 幂等补全既有列 → 置 V0 → 应用后续迁移，
|
||||
//! detect 守卫保证只补缺的列/索引，数据零搬运。
|
||||
//! - **幂等性关键**：SQLite 无 `ADD COLUMN IF NOT EXISTS`，迁移必须靠 detect 守卫而非裸 SQL
|
||||
//! 数组实现幂等（审查 CRITICAL#3——否则全新库上 bootstrap 已建新列，迁移再 ADD 会报
|
||||
//! duplicate column 崩启动）。
|
||||
//!
|
||||
//! 队列库（dcts_queue.db）无版本迁移，维持现状；本模块**仅主库**引入版本号。
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use rusqlite::{params, Connection, TransactionBehavior};
|
||||
|
||||
/// 一个版本化迁移。
|
||||
///
|
||||
/// - `version`：> V0(=0) 的顺序号（1..N），`PRAGMA user_version = version` 即代表已应用。
|
||||
/// - `name`：便于日志与审计。
|
||||
/// - `detect`：该迁移是否已应用（列/索引存在性检测）。为 true 时跳过 `up`，仅推进版本号。
|
||||
/// - `up`：未应用时才执行，同一事务内顺序执行。
|
||||
///
|
||||
/// `Clone + Copy`：fn 指针与 `&'static str` 均 Copy，迁移定义可原地复用（测试重跑场景）。
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct Migration {
|
||||
pub version: u32,
|
||||
pub name: &'static str,
|
||||
pub detect: fn(&Connection) -> Result<bool>,
|
||||
pub up: &'static [&'static str],
|
||||
}
|
||||
|
||||
/// 全部迁移。随各 Phase 追加（Phase 0 交付基础设施，Phase 1 起逐个加入）。
|
||||
pub const MIGRATIONS: &[Migration] = &[
|
||||
// M1（Phase 1，P1）：tasks 阶段信息补全。failed_stage = 失败阶段归因，
|
||||
// summary_json = ModelSummary 全保真 JSON。两列均在线 ADD COLUMN，旧节点上报不破坏结算。
|
||||
Migration {
|
||||
version: 1,
|
||||
name: "tasks-stage-info",
|
||||
detect: |c| {
|
||||
Ok(has_column(c, "tasks", "failed_stage")?
|
||||
&& has_column(c, "tasks", "summary_json")?)
|
||||
},
|
||||
up: &[
|
||||
"ALTER TABLE tasks ADD COLUMN failed_stage TEXT",
|
||||
"ALTER TABLE tasks ADD COLUMN summary_json TEXT",
|
||||
],
|
||||
},
|
||||
// M2（Phase 2，P2）：tasks 单列 workflow_name 查询的覆盖索引。
|
||||
// 覆盖 `COUNT(*) WHERE workflow_name=?` 及详情页按工作流统计；用户决策不做 tasks 清理。
|
||||
// CREATE INDEX IF NOT EXISTS 天然幂等，新库/旧库统一由此迁移建立（无需进 init_tables）。
|
||||
Migration {
|
||||
version: 2,
|
||||
name: "tasks-wf-status-created-index",
|
||||
detect: |c| has_index(c, "idx_tasks_wf_status_created"),
|
||||
up: &["CREATE INDEX IF NOT EXISTS idx_tasks_wf_status_created ON tasks(workflow_name, status, created_at)"],
|
||||
},
|
||||
// M4（Phase 4，P5）：清除 node_credentials 死列 revoked。
|
||||
// 新代码不读写它；registration_secret 保留在 nodes（审查 CRITICAL#1/#2：pending 节点无
|
||||
// node_credentials 行，迁移会静默丢凭据；token_hash NOT NULL + 唯一索引塞不下空占位）。
|
||||
// DROP COLUMN 涉及表重建（bundled SQLite 3.45+），部署走低峰窗口 + 手动备份（§11）。
|
||||
Migration {
|
||||
version: 4,
|
||||
name: "drop-revoked-dead-column",
|
||||
detect: |c| Ok(!has_column(c, "node_credentials", "revoked")?),
|
||||
up: &["ALTER TABLE node_credentials DROP COLUMN revoked"],
|
||||
},
|
||||
// M6（Phase 6，P8）：删除 tasks 冗余列 task_type。
|
||||
// 该列与 tlusty_strategies[0] 恒等、synspec-only 场景为"假值"，执行链已改由 strategies[0]
|
||||
// 推导（executor.rs），归因/过滤全部改派生口径。前提：集群无历史节点（用户决策）。
|
||||
// DROP COLUMN 涉及表重建，部署走低峰窗口 + 手动备份（§11）；升级前确认队列为空（§8.8）。
|
||||
Migration {
|
||||
version: 6,
|
||||
name: "drop-task-type-column",
|
||||
detect: |c| Ok(!has_column(c, "tasks", "task_type")?),
|
||||
up: &["ALTER TABLE tasks DROP COLUMN task_type"],
|
||||
},
|
||||
// M7（Phase 5a，P6）：grid_points 补 synspec 收敛归因列。
|
||||
// 与 success_method 镜像的 synspec 分支（光谱以什么策略收敛），解锁「光谱以 standard 等
|
||||
// 策略收敛了多少点」的 SQL 统计；TLUSTY-only 成功保持 NULL。在线 ADD COLUMN，无停写窗口。
|
||||
//
|
||||
// 版本号 = 7（而非 5）：**迁移版本必须与部署顺序单调一致**——§11 部署顺序是 6 → 5a，
|
||||
// 若 5a 编号为 5，则已升到 v6 的库会因 `version <= current` 跳过它，synspec 列永不创建。
|
||||
//（设计 §2.2 的 "M5" 标签是早期命名，此处按部署序改号 M7。）
|
||||
Migration {
|
||||
version: 7,
|
||||
name: "synspec-success-method",
|
||||
detect: |c| has_column(c, "grid_points", "synspec_success_method"),
|
||||
up: &["ALTER TABLE grid_points ADD COLUMN synspec_success_method TEXT"],
|
||||
},
|
||||
// M8（Phase 5b，P6）：grid_points 阶段状态列。
|
||||
// 解除点级单值 status 掩盖两阶段管线：半失败点（大气收敛+光谱失败）可查
|
||||
// tlusty_status='converged' + synspec_status='failed'。NULL = 阶段不适用（tlusty_enabled=0
|
||||
// 或 synspec_enabled=0）。整体 grid_points.status 仍是权威状态，阶段列是补充可查信息。
|
||||
// 在线 ADD COLUMN；同步触点见 docs/database_refactor_design.md §7.3 5b。
|
||||
Migration {
|
||||
version: 8,
|
||||
name: "grid-point-stage-status",
|
||||
detect: |c| {
|
||||
Ok(has_column(c, "grid_points", "tlusty_status")?
|
||||
&& has_column(c, "grid_points", "synspec_status")?)
|
||||
},
|
||||
up: &[
|
||||
"ALTER TABLE grid_points ADD COLUMN tlusty_status TEXT",
|
||||
"ALTER TABLE grid_points ADD COLUMN synspec_status TEXT",
|
||||
],
|
||||
},
|
||||
// M9（Phase 7c）：grid_points.status 值 'converged' → 'completed'。
|
||||
// TLUSTY-first 残留：'converged' 暗示"大气收敛"，实为"管线完成"（大气+光谱）。
|
||||
// 数据迁移 + 全链 SQL 字面量同步（见 db.rs/scheduler.rs，值全部改 'completed'）。
|
||||
Migration {
|
||||
version: 9,
|
||||
name: "grid-status-converged-to-completed",
|
||||
detect: |c| {
|
||||
// 已迁移 = 不再存在旧值 'converged'（detect 检查数据而非列）。
|
||||
// 安全性：apply_migrations_with 以 user_version 闸控——M9 只在 version<9 时评估，
|
||||
// 一旦版本推进到 9 即永不再走此 detect，故即便后续代码意外再写入 'converged'
|
||||
// 也不会触发本迁移重放（user_version 不会回退）。
|
||||
let mut stmt = c.prepare("SELECT 1 FROM grid_points WHERE status = 'converged' LIMIT 1")?;
|
||||
Ok(!stmt.exists([])?)
|
||||
},
|
||||
up: &["UPDATE grid_points SET status = 'completed' WHERE status = 'converged'"],
|
||||
},
|
||||
// M10（Phase 7c）：workflow_progress_snapshots 列名 converged → completed。
|
||||
// 该列存"管线完成点数"，列名随状态值改名保持一致（§9.5 耦合项）。RENAME COLUMN 在线。
|
||||
Migration {
|
||||
version: 10,
|
||||
name: "snapshots-converged-column-rename",
|
||||
detect: |c| has_column(c, "workflow_progress_snapshots", "completed"),
|
||||
up: &["ALTER TABLE workflow_progress_snapshots RENAME COLUMN converged TO completed"],
|
||||
},
|
||||
// M11（H1 活锁修复）：grid_points 补 pending_strategies 列。
|
||||
// 运行时回退（trigger_strategy_fallback）把点打回 pending 等种子时，记录「剩余策略链」
|
||||
// （JSON 数组），调度路径据此用剩余链重派、避免重跑已失败策略导致的无界失败重试活锁。
|
||||
// 在线 ADD COLUMN，无停写窗口。detect 幂等（全新库 bootstrap 已含该列 → 跳过）。
|
||||
Migration {
|
||||
version: 11,
|
||||
name: "grid-point-pending-strategies",
|
||||
detect: |c| has_column(c, "grid_points", "pending_strategies"),
|
||||
up: &["ALTER TABLE grid_points ADD COLUMN pending_strategies TEXT"],
|
||||
},
|
||||
// M12（P9 命名拆分）：rid_points.success_method 值域混用列拆为阶段列 tlusty_success_method。
|
||||
// success_method 原是 TLUSTY-first 整体归因：TLUSTY 任务存 tlusty_strategies[0]
|
||||
// （cold_run/seed_step），synspec-only 任务却存 synspec_strategies[0]（standard）——
|
||||
// 同一列两个值域，前端需猜策略名区分。拆后：
|
||||
// - tlusty_success_method：TLUSTY 阶段策略（tlusty 禁用为 NULL）
|
||||
// - synspec_success_method（既有）：光谱阶段策略（synspec 禁用为 NULL）
|
||||
// 整体归因改由消费方派生（前端 tlusty ?? synspec）。
|
||||
//
|
||||
// 回填判别：旧数据里 synspec-only 点 success_method 与 synspec_success_method 同值
|
||||
// （正是被清理的冗余）；故命中该等式的点不写 tlusty（保持 NULL），其余（正常双阶段
|
||||
// cold_run/seed_step）→ tlusty = 原 success_method。局限：极端情形下 TLUSTY 点
|
||||
// tlusty_strategies[0]==synspec_strategies[0]（如都叫 standard）会被误判为 NULL；
|
||||
// 实际 sdB_cno 中大气策略与光谱策略不冲突，可接受。在线 ADD + UPDATE，无停写窗口。
|
||||
Migration {
|
||||
version: 12,
|
||||
name: "tlusty-success-method",
|
||||
detect: |c| has_column(c, "grid_points", "tlusty_success_method"),
|
||||
up: &[
|
||||
"ALTER TABLE grid_points ADD COLUMN tlusty_success_method TEXT",
|
||||
"UPDATE grid_points SET tlusty_success_method = success_method \
|
||||
WHERE success_method IS NOT NULL \
|
||||
AND NOT (success_method = synspec_success_method AND synspec_success_method IS NOT NULL)",
|
||||
],
|
||||
},
|
||||
// M13（P9 命名拆分）：删除值域混用列 success_method（M12 已回填 tlusty_success_method）。
|
||||
// DROP COLUMN 涉及表重建（bundled SQLite 3.45+），部署走低峰窗口 + 手动备份（§11）。
|
||||
Migration {
|
||||
version: 13,
|
||||
name: "drop-success-method-column",
|
||||
detect: |c| Ok(!has_column(c, "grid_points", "success_method")?),
|
||||
up: &["ALTER TABLE grid_points DROP COLUMN success_method"],
|
||||
},
|
||||
];
|
||||
|
||||
/// 当前 schema 版本（`PRAGMA user_version`）。
|
||||
pub fn current_version(conn: &Connection) -> Result<u32> {
|
||||
Ok(conn.pragma_query_value(None, "user_version", |r| r.get(0))?)
|
||||
}
|
||||
|
||||
/// 从当前版本顺序应用常量表 `MIGRATIONS` 中尚未执行的迁移。
|
||||
pub fn apply_migrations(conn: &mut Connection) -> Result<()> {
|
||||
apply_migrations_with(conn, MIGRATIONS)
|
||||
}
|
||||
|
||||
/// 应用给定迁移列表中尚未执行的部分，每个迁移独立事务（供测试传入自定义列表）。
|
||||
///
|
||||
/// 对每个 `version > current` 的迁移：
|
||||
/// - `detect = true`（已应用，如新库 bootstrap 已建列）→ 仅推进 user_version，不执行 `up`；
|
||||
/// - `detect = false` → `BEGIN IMMEDIATE` → 执行 `up` SQL → `PRAGMA user_version = V` → `COMMIT`。
|
||||
///
|
||||
/// 中途失败不推进版本（进程启动时重试）：当前迁移所在事务回滚，之前迁移的版本号已持久化。
|
||||
fn apply_migrations_with(conn: &mut Connection, migrations: &[Migration]) -> Result<()> {
|
||||
let mut current = current_version(conn)?;
|
||||
for m in migrations {
|
||||
if m.version <= current {
|
||||
continue;
|
||||
}
|
||||
if (m.detect)(conn)? {
|
||||
// 已应用（detect 命中，如全新库 bootstrap 已建列）→ 仅推进版本号，不执行 up。
|
||||
conn.pragma_update(None, "user_version", m.version)?;
|
||||
tracing::info!(
|
||||
version = m.version,
|
||||
name = m.name,
|
||||
"迁移已应用（detect 跳过）"
|
||||
);
|
||||
current = m.version;
|
||||
continue;
|
||||
}
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
for sql in m.up {
|
||||
tx.execute_batch(sql)
|
||||
.with_context(|| format!("迁移 M{} ({}) 失败：{}", m.version, m.name, sql))?;
|
||||
}
|
||||
tx.pragma_update(None, "user_version", m.version)?;
|
||||
tx.commit()?;
|
||||
tracing::info!(version = m.version, name = m.name, "迁移已应用");
|
||||
current = m.version;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 检测表是否已含指定列（PRAGMA table_info）。`table` 必须是 MIGRATIONS 内写死的常量表名，
|
||||
/// 绝不来自外部输入（表名不参与任何用户数据路径）。
|
||||
fn has_column(conn: &Connection, table: &str, column: &str) -> Result<bool> {
|
||||
let mut stmt = conn.prepare(&format!("PRAGMA table_info({table})"))?;
|
||||
let rows = stmt.query_map([], |r| r.get::<_, String>(1))?;
|
||||
for r in rows {
|
||||
if r.map(|name| name == column).unwrap_or(false) {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// 检测索引是否已存在（sqlite_master）。
|
||||
fn has_index(conn: &Connection, index: &str) -> Result<bool> {
|
||||
let mut stmt = conn.prepare("SELECT 1 FROM sqlite_master WHERE type='index' AND name=?1")?;
|
||||
Ok(stmt.exists(params![index])?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn mem_conn() -> Connection {
|
||||
Connection::open_in_memory().unwrap()
|
||||
}
|
||||
|
||||
/// 新库全流程（对齐 Database::new）：bootstrap 已建出最新形态 schema（含 M1 新增列）
|
||||
/// → 版本 0 → apply_migrations 应**无报错**地把版本推进到最新（已存在的列/索引经 detect
|
||||
/// 跳过，不因 duplicate column 崩溃——审查 CRITICAL#3 回归；索引类迁移在最新列上正常建立）。
|
||||
#[test]
|
||||
fn fresh_db_bootstrap_then_migrations_advance_version() {
|
||||
let mut conn = mem_conn();
|
||||
// 模拟 init_tables bootstrap：tasks 是含全部迁移引用列的最新形态（M1 列已存在、
|
||||
// M2 索引目标列已存在但索引本身未建）。
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE tasks (
|
||||
task_id TEXT PRIMARY KEY,
|
||||
point_name TEXT NOT NULL,
|
||||
node_id TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
created_at DATETIME NOT NULL,
|
||||
completed_at DATETIME,
|
||||
workflow_name TEXT,
|
||||
failed_stage TEXT,
|
||||
summary_json TEXT
|
||||
);
|
||||
CREATE TABLE grid_points (
|
||||
name TEXT NOT NULL,
|
||||
workflow_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
tlusty_success_method TEXT,
|
||||
synspec_success_method TEXT,
|
||||
tlusty_status TEXT,
|
||||
synspec_status TEXT,
|
||||
pending_strategies TEXT
|
||||
);
|
||||
CREATE TABLE workflow_progress_snapshots (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
workflow_name TEXT NOT NULL,
|
||||
ts DATETIME NOT NULL,
|
||||
total INTEGER NOT NULL,
|
||||
pending INTEGER NOT NULL,
|
||||
queued INTEGER NOT NULL,
|
||||
running INTEGER NOT NULL,
|
||||
completed INTEGER NOT NULL,
|
||||
failed INTEGER NOT NULL
|
||||
)",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(current_version(&conn).unwrap(), 0);
|
||||
apply_migrations(&mut conn).unwrap();
|
||||
let latest = MIGRATIONS.last().map(|m| m.version).unwrap_or(0);
|
||||
assert_eq!(current_version(&conn).unwrap(), latest);
|
||||
}
|
||||
|
||||
/// 正常迁移：up 顺序执行，版本推进。
|
||||
#[test]
|
||||
fn migration_applies_and_advances_version() {
|
||||
let mut conn = mem_conn();
|
||||
let m = Migration {
|
||||
version: 1,
|
||||
name: "test-add-col",
|
||||
detect: |c| has_column(c, "t", "c"),
|
||||
up: &[
|
||||
"CREATE TABLE t(id INTEGER PRIMARY KEY)",
|
||||
"ALTER TABLE t ADD COLUMN c TEXT",
|
||||
],
|
||||
};
|
||||
apply_migrations_with(&mut conn, &[m]).unwrap();
|
||||
assert_eq!(current_version(&conn).unwrap(), 1);
|
||||
assert!(has_column(&conn, "t", "c").unwrap());
|
||||
}
|
||||
|
||||
/// detect=true（已应用）→ 跳过 up（若执行会 duplicate column 崩），仅推进版本。
|
||||
#[test]
|
||||
fn detect_skip_advances_version_without_running_up() {
|
||||
let mut conn = mem_conn();
|
||||
conn.execute_batch("CREATE TABLE t(id INTEGER PRIMARY KEY, c TEXT)")
|
||||
.unwrap();
|
||||
let m = Migration {
|
||||
version: 1,
|
||||
name: "test-add-col",
|
||||
detect: |c| has_column(c, "t", "c"),
|
||||
up: &["ALTER TABLE t ADD COLUMN c TEXT"],
|
||||
};
|
||||
apply_migrations_with(&mut conn, &[m]).unwrap();
|
||||
assert_eq!(current_version(&conn).unwrap(), 1);
|
||||
}
|
||||
|
||||
/// M4 专项：旧库 node_credentials 含 revoked 死列 → apply_migrations 后列消失、版本推进。
|
||||
/// 同时验证 M1（列已存在跳过）+ M2（索引缺失建立）+ M4（DROP COLUMN）在同库顺序生效。
|
||||
#[test]
|
||||
fn m4_drops_revoked_dead_column_on_old_db() {
|
||||
let mut conn = mem_conn();
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE tasks (
|
||||
task_id TEXT PRIMARY KEY,
|
||||
point_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
created_at DATETIME NOT NULL,
|
||||
workflow_name TEXT,
|
||||
failed_stage TEXT,
|
||||
summary_json TEXT
|
||||
);
|
||||
CREATE TABLE grid_points (
|
||||
name TEXT NOT NULL,
|
||||
workflow_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
success_method TEXT,
|
||||
synspec_success_method TEXT
|
||||
);
|
||||
CREATE TABLE workflow_progress_snapshots (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
workflow_name TEXT NOT NULL,
|
||||
ts DATETIME NOT NULL,
|
||||
total INTEGER NOT NULL,
|
||||
pending INTEGER NOT NULL,
|
||||
queued INTEGER NOT NULL,
|
||||
running INTEGER NOT NULL,
|
||||
converged INTEGER NOT NULL,
|
||||
failed INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE node_credentials (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
token_hash TEXT NOT NULL,
|
||||
issued_at DATETIME NOT NULL,
|
||||
revoked INTEGER NOT NULL DEFAULT 0,
|
||||
raw_token_pending TEXT
|
||||
);",
|
||||
)
|
||||
.unwrap();
|
||||
apply_migrations(&mut conn).unwrap();
|
||||
assert!(
|
||||
!has_column(&conn, "node_credentials", "revoked").unwrap(),
|
||||
"revoked 死列应被 M4 清除"
|
||||
);
|
||||
assert!(
|
||||
has_index(&conn, "idx_tasks_wf_status_created").unwrap(),
|
||||
"M2 索引应建立"
|
||||
);
|
||||
// M10：旧 snapshots 的 converged 列应被重命名为 completed。
|
||||
assert!(has_column(&conn, "workflow_progress_snapshots", "completed").unwrap());
|
||||
assert!(!has_column(&conn, "workflow_progress_snapshots", "converged").unwrap());
|
||||
// M11：grid_points 的 pending_strategies 列应被补齐（H1 活锁修复标记）。
|
||||
assert!(has_column(&conn, "grid_points", "pending_strategies").unwrap());
|
||||
let latest = MIGRATIONS.last().map(|m| m.version).unwrap_or(0);
|
||||
assert_eq!(current_version(&conn).unwrap(), latest);
|
||||
}
|
||||
|
||||
/// M9（Phase 7c）：grid_points.status 值 'converged' → 'completed' 数据迁移。
|
||||
/// 旧库残留 'converged' 值 → apply_migrations 后全部转为 'completed'。
|
||||
#[test]
|
||||
fn m9_converged_status_value_migrated() {
|
||||
let mut conn = mem_conn();
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE tasks (
|
||||
task_id TEXT PRIMARY KEY,
|
||||
point_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
created_at DATETIME NOT NULL,
|
||||
workflow_name TEXT
|
||||
);
|
||||
CREATE TABLE grid_points (
|
||||
name TEXT NOT NULL,
|
||||
workflow_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
success_method TEXT,
|
||||
synspec_success_method TEXT
|
||||
);
|
||||
INSERT INTO grid_points (name, workflow_name, status) VALUES ('p1','wf_a','converged');
|
||||
INSERT INTO grid_points (name, workflow_name, status) VALUES ('p2','wf_a','failed');
|
||||
CREATE TABLE workflow_progress_snapshots (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
workflow_name TEXT NOT NULL,
|
||||
ts DATETIME NOT NULL,
|
||||
total INTEGER NOT NULL,
|
||||
pending INTEGER NOT NULL,
|
||||
queued INTEGER NOT NULL,
|
||||
running INTEGER NOT NULL,
|
||||
converged INTEGER NOT NULL,
|
||||
failed INTEGER NOT NULL
|
||||
);",
|
||||
)
|
||||
.unwrap();
|
||||
apply_migrations(&mut conn).unwrap();
|
||||
// 旧值迁为 'completed'；'failed' 不受影响。
|
||||
let statuses: Vec<String> = {
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT status FROM grid_points ORDER BY name")
|
||||
.unwrap();
|
||||
stmt.query_map([], |r| r.get::<_, String>(0))
|
||||
.unwrap()
|
||||
.filter_map(Result::ok)
|
||||
.collect()
|
||||
};
|
||||
assert_eq!(
|
||||
statuses,
|
||||
vec!["completed".to_string(), "failed".to_string()]
|
||||
);
|
||||
// M10：snapshots 列改名。
|
||||
assert!(has_column(&conn, "workflow_progress_snapshots", "completed").unwrap());
|
||||
}
|
||||
|
||||
/// 中断恢复：M1 成功（版本 1），M2 中途失败（事务回滚，版本停在 1）；
|
||||
/// 修正 M2 后重跑，M1 跳过、M2 成功。
|
||||
#[test]
|
||||
fn interrupted_migration_rolls_back_and_retries() {
|
||||
let mut conn = mem_conn();
|
||||
let m1 = Migration {
|
||||
version: 1,
|
||||
name: "m1",
|
||||
detect: |c| has_column(c, "t", "c1"),
|
||||
up: &[
|
||||
"CREATE TABLE t(id INTEGER PRIMARY KEY)",
|
||||
"ALTER TABLE t ADD COLUMN c1 TEXT",
|
||||
],
|
||||
};
|
||||
let m2_bad = Migration {
|
||||
version: 2,
|
||||
name: "m2-bad",
|
||||
detect: |c| has_column(c, "t", "c2"),
|
||||
up: &["ALTER TABLE t ADD COLUMN c2 TEXT", "THIS IS NOT VALID SQL"],
|
||||
};
|
||||
let err = apply_migrations_with(&mut conn, &[m1, m2_bad]).unwrap_err();
|
||||
assert!(err.to_string().contains("M2 (m2-bad) 失败"));
|
||||
// M1 已提交、版本停在 1；M2 回滚（c2 未建）。
|
||||
assert_eq!(current_version(&conn).unwrap(), 1);
|
||||
assert!(!has_column(&conn, "t", "c2").unwrap());
|
||||
// 重跑：M1（version 1）跳过，M2 修正后成功。
|
||||
let m2_good = Migration {
|
||||
version: 2,
|
||||
name: "m2-good",
|
||||
detect: |c| has_column(c, "t", "c2"),
|
||||
up: &["ALTER TABLE t ADD COLUMN c2 TEXT"],
|
||||
};
|
||||
apply_migrations_with(&mut conn, &[m1, m2_good]).unwrap();
|
||||
assert_eq!(current_version(&conn).unwrap(), 2);
|
||||
assert!(has_column(&conn, "t", "c2").unwrap());
|
||||
}
|
||||
|
||||
/// M12/M13 专项（P9 命名拆分）：旧库 success_method 值域混用列 → 拆为
|
||||
/// tlusty_success_method（TLUSTY 阶段策略）+ 既有 synspec_success_method，再删 success_method。
|
||||
/// 回填判别：synspec-only 点 success_method 与 synspec_success_method 同值（值域混用冗余），
|
||||
/// 命中该等式 → tlusty 保持 NULL；其余（正常双阶段 cold_run/seed_step）→ tlusty = 原值。
|
||||
#[test]
|
||||
fn m12_m13_split_success_method_backfills_and_drops() {
|
||||
let mut conn = mem_conn();
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE grid_points (
|
||||
name TEXT NOT NULL,
|
||||
workflow_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
success_method TEXT,
|
||||
synspec_success_method TEXT
|
||||
);
|
||||
-- 正常双阶段点：success_method = 大气策略 cold_run（≠ 光谱归因 standard）
|
||||
INSERT INTO grid_points (name, workflow_name, status, success_method, synspec_success_method)
|
||||
VALUES ('p_tlusty', 'wf', 'completed', 'cold_run', 'standard');
|
||||
-- 种子步进点
|
||||
INSERT INTO grid_points (name, workflow_name, status, success_method, synspec_success_method)
|
||||
VALUES ('p_seed', 'wf', 'completed', 'seed_step', 'standard');
|
||||
-- synspec-only 点：success_method == synspec_success_method（值域混用冗余）
|
||||
INSERT INTO grid_points (name, workflow_name, status, success_method, synspec_success_method)
|
||||
VALUES ('p_syn', 'wf', 'completed', 'standard', 'standard');
|
||||
-- 失败点：归因为 NULL
|
||||
INSERT INTO grid_points (name, workflow_name, status)
|
||||
VALUES ('p_failed', 'wf', 'failed')",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// M12 单跑：ADD tlusty_success_method + 回填。
|
||||
let m12 = MIGRATIONS.iter().find(|m| m.version == 12).unwrap();
|
||||
apply_migrations_with(&mut conn, &[*m12]).unwrap();
|
||||
let read = |conn: &Connection, name: &str, col: &str| -> Option<String> {
|
||||
// `col` 为测试内写死的列名常量（非用户输入）。
|
||||
conn.query_row(
|
||||
&format!("SELECT {col} FROM grid_points WHERE name = ?1"),
|
||||
rusqlite::params![name],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.ok()
|
||||
};
|
||||
assert_eq!(
|
||||
read(&conn, "p_tlusty", "tlusty_success_method").as_deref(),
|
||||
Some("cold_run"),
|
||||
"双阶段点回填大气策略"
|
||||
);
|
||||
assert_eq!(
|
||||
read(&conn, "p_seed", "tlusty_success_method").as_deref(),
|
||||
Some("seed_step"),
|
||||
"种子步进点回填 seed_step"
|
||||
);
|
||||
assert_eq!(
|
||||
read(&conn, "p_syn", "tlusty_success_method"),
|
||||
None,
|
||||
"synspec-only 点（两列同值）tlusty 保持 NULL"
|
||||
);
|
||||
assert_eq!(read(&conn, "p_failed", "tlusty_success_method"), None);
|
||||
assert_eq!(current_version(&conn).unwrap(), 12);
|
||||
|
||||
// M13 单跑：DROP success_method 列。
|
||||
let m13 = MIGRATIONS.iter().find(|m| m.version == 13).unwrap();
|
||||
apply_migrations_with(&mut conn, &[*m13]).unwrap();
|
||||
assert!(
|
||||
!has_column(&conn, "grid_points", "success_method").unwrap(),
|
||||
"success_method 值域混用列应被删除"
|
||||
);
|
||||
assert!(has_column(&conn, "grid_points", "tlusty_success_method").unwrap());
|
||||
assert!(has_column(&conn, "grid_points", "synspec_success_method").unwrap());
|
||||
// 回填数据在 DROP 后仍保留（tlusty_success_method 是独立列）。
|
||||
assert_eq!(
|
||||
read(&conn, "p_tlusty", "tlusty_success_method").as_deref(),
|
||||
Some("cold_run")
|
||||
);
|
||||
assert_eq!(current_version(&conn).unwrap(), 13);
|
||||
}
|
||||
}
|
||||
+361
-157
File diff suppressed because it is too large
Load Diff
@@ -266,6 +266,17 @@ async fn test_l2_node_token_issue_reissue_flow() {
|
||||
);
|
||||
// 旧 token 仍失效（已被覆盖）
|
||||
assert!(db.find_node_by_token(&token).await.is_none());
|
||||
|
||||
// M2：registration_secret 被首次 take 轮换后，旧 secret 无法取走 reissue 产生的新 token。
|
||||
// 首次 take（上文）已把 secret 轮换为无人知晓的新值，旧 secret（注册时下发的）立即失效。
|
||||
let stolen = db
|
||||
.take_pending_node_token("node-l2-test", secret.as_deref())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
stolen.is_none(),
|
||||
"被轮换的旧 registration_secret 不得取走 reissue 后的新 token（M2 一次性凭据）"
|
||||
);
|
||||
}
|
||||
|
||||
/// 验证中间件对 node token 的端到端鉴权：
|
||||
@@ -1323,9 +1334,9 @@ fn make_import_multipart(
|
||||
body.extend_from_slice(b"Content-Type: application/octet-stream\r\n\r\n");
|
||||
body.extend_from_slice(seed_bytes);
|
||||
body.extend_from_slice(b"\r\n");
|
||||
// 收敛途径字段（cold_run/seed_step）：模拟 import_results 工具判定后透传的途径。
|
||||
// 大气收敛途径字段（cold_run/seed_step）：模拟 import_results 工具判定后透传的途径。
|
||||
body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes());
|
||||
body.extend_from_slice(b"Content-Disposition: form-data; name=\"success_method\"\r\n");
|
||||
body.extend_from_slice(b"Content-Disposition: form-data; name=\"tlusty_success_method\"\r\n");
|
||||
body.extend_from_slice(b"Content-Type: text/plain\r\n\r\n");
|
||||
body.extend_from_slice(success_method.as_bytes());
|
||||
body.extend_from_slice(b"\r\n");
|
||||
@@ -1433,7 +1444,7 @@ async fn test_import_seed_admin_endpoint() {
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("grid_points 应存在");
|
||||
assert_eq!(gp.0, "converged", "导入的收敛点应为 converged 状态");
|
||||
assert_eq!(gp.0, "completed", "导入的收敛点应为 converged 状态");
|
||||
|
||||
// 3. 幂等：重复导入同名点不应报错，状态仍 converged。
|
||||
let body_bytes = make_import_multipart(
|
||||
@@ -1462,7 +1473,7 @@ async fn test_import_seed_admin_endpoint() {
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(gp.0, "converged");
|
||||
assert_eq!(gp.0, "completed");
|
||||
|
||||
// 4. 未收敛点 → 200，但不写 .7、grid_points 维持 pending（未建 converged）。
|
||||
let conv_fail = make_legacy_conv_json("t20000_g5.0_he-2_c-4_n-4_o-4_fail", false);
|
||||
@@ -1570,8 +1581,8 @@ async fn test_import_seed_python_legacy_conv_json() {
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("grid_points 应存在");
|
||||
assert_eq!(row.status, "converged");
|
||||
assert_eq!(row.success_method.as_deref(), Some("cold_run"));
|
||||
assert_eq!(row.status, "completed");
|
||||
assert_eq!(row.tlusty_success_method.as_deref(), Some("cold_run"));
|
||||
assert_eq!(
|
||||
row.last_elapsed_sec,
|
||||
Some(715.0),
|
||||
@@ -1585,7 +1596,7 @@ async fn test_import_seed_python_legacy_conv_json() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_node_disable_enable_flow() {
|
||||
use common::models::{GridAxisValue, GridPointParams, TaskSpec, TaskType};
|
||||
use common::models::{GridAxisValue, GridPointParams, TaskSpec};
|
||||
use uuid::Uuid;
|
||||
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
@@ -1668,7 +1679,6 @@ async fn test_node_disable_enable_flow() {
|
||||
logn: GridAxisValue::from_value(-2.0),
|
||||
logo: GridAxisValue::from_value(-2.0),
|
||||
},
|
||||
task_type: TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 60,
|
||||
workflow_name: Some("wf_test".to_string()),
|
||||
@@ -2120,13 +2130,13 @@ async fn test_admin_set_node_quota_and_heartbeat_sync() {
|
||||
|
||||
/// 测试专用：走真实写路径派发并回报一个网格点任务。
|
||||
///
|
||||
/// `insert_task` → `record_task_report` 会回填 `grid_points.success_method`、
|
||||
/// `insert_task` → `record_task_report` 会回填 `grid_points.tlusty_success_method`、
|
||||
/// `attempt_count` 与 `tasks.completed_at`，与生产链路一致（不绕过任何状态机逻辑）。
|
||||
async fn dispatch_and_report(
|
||||
db: &Database,
|
||||
wf: &str,
|
||||
p: &common::models::GridPointParams,
|
||||
task_type: common::models::TaskType,
|
||||
strategy: &str,
|
||||
seed: Option<String>,
|
||||
converged: bool,
|
||||
) {
|
||||
@@ -2135,11 +2145,15 @@ async fn dispatch_and_report(
|
||||
task_id,
|
||||
point_name: p.model_name(),
|
||||
params: p.clone(),
|
||||
task_type,
|
||||
seed_point_name: seed,
|
||||
timeout_sec: 7200,
|
||||
workflow_name: Some(wf.to_string()),
|
||||
wave: 0,
|
||||
// Phase 6 起策略链首项即"当前策略"（归因/过滤全派生自它）。
|
||||
tlusty_config: common::models::PhaseConfig {
|
||||
strategies: vec![strategy.to_string()],
|
||||
..common::models::PhaseConfig::default_tlusty()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
db.insert_task(&spec).await.unwrap();
|
||||
@@ -2153,7 +2167,7 @@ async fn dispatch_and_report(
|
||||
} else {
|
||||
common::models::TaskStatus::Failed
|
||||
},
|
||||
converged,
|
||||
result_valid: converged,
|
||||
max_relc: if converged { Some(0.0005) } else { Some(9.5e5) },
|
||||
atmosphere_has_nan: false,
|
||||
elapsed_sec: 120.0,
|
||||
@@ -2267,33 +2281,17 @@ async fn test_wf_stats_endpoint() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
dispatch_and_report(
|
||||
&db,
|
||||
"wf_stats",
|
||||
&p_cold,
|
||||
common::models::TaskType::ColdRun,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(&db, "wf_stats", &p_cold, "cold_run", None, true).await;
|
||||
dispatch_and_report(
|
||||
&db,
|
||||
"wf_stats",
|
||||
&p_seed,
|
||||
common::models::TaskType::SeedStep,
|
||||
"seed_step",
|
||||
Some(p_cold.model_name()),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(
|
||||
&db,
|
||||
"wf_stats",
|
||||
&p_failed,
|
||||
common::models::TaskType::ColdRun,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(&db, "wf_stats", &p_failed, "cold_run", None, false).await;
|
||||
db.mark_grid_point_imported(&p_imported.model_name(), "wf_stats", None, "cold_run")
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -2347,7 +2345,7 @@ async fn test_wf_stats_endpoint() {
|
||||
assert_eq!(data["pending"], 1, "pending 与 queued 必须分开计数");
|
||||
assert_eq!(data["queued"], 1);
|
||||
assert_eq!(data["running"], 1);
|
||||
assert_eq!(data["converged"], 3);
|
||||
assert_eq!(data["completed"], 3);
|
||||
assert_eq!(data["failed"], 1);
|
||||
assert_eq!(data["cold_run_converged"], 2);
|
||||
assert_eq!(data["seed_step_converged"], 1);
|
||||
@@ -2356,8 +2354,8 @@ async fn test_wf_stats_endpoint() {
|
||||
assert_eq!(waves.len(), 3);
|
||||
assert_eq!(waves[0]["wave"], 0);
|
||||
assert_eq!(waves[0]["total"], 3);
|
||||
assert_eq!(waves[0]["converged"], 0);
|
||||
assert_eq!(waves[1]["converged"], 2);
|
||||
assert_eq!(waves[0]["completed"], 0);
|
||||
assert_eq!(waves[1]["completed"], 2);
|
||||
assert_eq!(waves[2]["failed"], 1);
|
||||
// 有已完成任务且有剩余点 → ETA 可估算。
|
||||
// P3 后 avg 取精确 elapsed_sec（夹具每次回报 120s），无在线节点按串行兜底：
|
||||
@@ -2436,48 +2434,24 @@ async fn seed_obs_fixture(db: &Database, db_path: &std::path::Path, wf: &str) ->
|
||||
}
|
||||
db.upsert_grid_point(&p_imported, 1, wf).await.unwrap();
|
||||
|
||||
dispatch_and_report(
|
||||
db,
|
||||
wf,
|
||||
&p_cold,
|
||||
common::models::TaskType::ColdRun,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(db, wf, &p_cold, "cold_run", None, true).await;
|
||||
dispatch_and_report(
|
||||
db,
|
||||
wf,
|
||||
&p_seed,
|
||||
common::models::TaskType::SeedStep,
|
||||
"seed_step",
|
||||
Some(p_cold.model_name()),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(
|
||||
db,
|
||||
wf,
|
||||
&p_failed,
|
||||
common::models::TaskType::ColdRun,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(db, wf, &p_failed, "cold_run", None, false).await;
|
||||
// rescued：先冷启动失败，再种子步进救回（2 次尝试，最终 converged/seed_step）
|
||||
dispatch_and_report(db, wf, &p_rescued, "cold_run", None, false).await;
|
||||
dispatch_and_report(
|
||||
db,
|
||||
wf,
|
||||
&p_rescued,
|
||||
common::models::TaskType::ColdRun,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(
|
||||
db,
|
||||
wf,
|
||||
&p_rescued,
|
||||
common::models::TaskType::SeedStep,
|
||||
"seed_step",
|
||||
Some(p_cold.model_name()),
|
||||
true,
|
||||
)
|
||||
@@ -2490,7 +2464,7 @@ async fn seed_obs_fixture(db: &Database, db_path: &std::path::Path, wf: &str) ->
|
||||
conn.execute(
|
||||
"UPDATE tasks SET created_at = datetime('now','-60 seconds'),
|
||||
completed_at = datetime('now','-60 seconds')
|
||||
WHERE workflow_name = ?1 AND NOT (point_name = ?2 AND task_type = 'seed_step')",
|
||||
WHERE workflow_name = ?1 AND NOT (point_name = ?2 AND json_extract(tlusty_strategies, '$[0]') = 'seed_step')",
|
||||
rusqlite::params![wf, p_rescued.model_name()],
|
||||
)
|
||||
.unwrap();
|
||||
@@ -2670,19 +2644,14 @@ async fn test_wf_points_endpoint() {
|
||||
// ---- 10. 最近尝试 JOIN：rescued 双尝试取最新（seed_step 救回） ----
|
||||
let (_, data) = get_points(&app, "/api/workflows/wf_pts/points").await;
|
||||
let rescued = find_point(&data, &n.rescued);
|
||||
assert_eq!(rescued["status"], "converged");
|
||||
assert_eq!(rescued["success_method"], "seed_step");
|
||||
assert_eq!(rescued["status"], "completed");
|
||||
assert_eq!(rescued["tlusty_success_method"], "seed_step");
|
||||
assert_eq!(rescued["attempt_count"], 2, "两次尝试都应计数");
|
||||
assert_eq!(
|
||||
rescued["last_task_type"], "seed_step",
|
||||
"最近尝试应为种子步进"
|
||||
);
|
||||
assert_eq!(rescued["seed_point_name"], n.cold, "种子来源应为 cold 点");
|
||||
assert_eq!(rescued["last_max_relc"], 0.0005);
|
||||
assert_eq!(rescued["last_elapsed_sec"], 120.0, "真实墙钟耗时应落库");
|
||||
// pending 点无任何尝试 → last_* 全 null
|
||||
let pending = find_point(&data, &n.pending);
|
||||
assert!(pending["last_task_type"].is_null());
|
||||
assert!(pending["last_completed_at"].is_null());
|
||||
assert!(pending["last_elapsed_sec"].is_null());
|
||||
assert_eq!(pending["attempt_count"], 0);
|
||||
@@ -2782,13 +2751,13 @@ async fn test_point_detail_endpoint() {
|
||||
let (st, data) = get_detail(&app, &uri).await;
|
||||
assert_eq!(st, StatusCode::OK);
|
||||
assert_eq!(data["point"]["name"], n.cold);
|
||||
assert_eq!(data["point"]["status"], "converged");
|
||||
assert_eq!(data["point"]["success_method"], "cold_run");
|
||||
assert_eq!(data["point"]["status"], "completed");
|
||||
assert_eq!(data["point"]["tlusty_success_method"], "cold_run");
|
||||
let attempts = data["attempts"].as_array().unwrap();
|
||||
assert_eq!(attempts.len(), 1);
|
||||
assert_eq!(attempts[0]["task_type"], "cold_run");
|
||||
assert!(attempts[0]["seed_point_name"].is_null(), "冷启动无种子来源");
|
||||
assert_eq!(attempts[0]["status"], "completed");
|
||||
assert_eq!(data["conv"]["converged"], true, "conv.json 应被解析");
|
||||
assert_eq!(data["conv"]["result_valid"], true, "conv.json 应被解析");
|
||||
assert_eq!(data["conv"]["final_max_relc"], 0.000321);
|
||||
|
||||
// ---- 3. rescued 点：2 次尝试按时间升序（冷启失败 → 种子步进救回），conv 为 null ----
|
||||
@@ -2798,10 +2767,9 @@ async fn test_point_detail_endpoint() {
|
||||
assert_eq!(data["point"]["attempt_count"], 2);
|
||||
let attempts = data["attempts"].as_array().unwrap();
|
||||
assert_eq!(attempts.len(), 2);
|
||||
assert_eq!(attempts[0]["task_type"], "cold_run", "首次应为冷启动");
|
||||
assert!(attempts[0]["seed_point_name"].is_null(), "首次冷启动无种子");
|
||||
assert_eq!(attempts[0]["status"], "failed");
|
||||
assert_eq!(attempts[0]["elapsed_sec"], 120.0, "每次尝试耗时应落库");
|
||||
assert_eq!(attempts[1]["task_type"], "seed_step", "第二次应为种子步进");
|
||||
assert_eq!(attempts[1]["status"], "completed");
|
||||
assert_eq!(
|
||||
attempts[1]["seed_point_name"], n.cold,
|
||||
@@ -2878,15 +2846,7 @@ async fn test_list_workflows_inline_stats() {
|
||||
let p2 = wf_stats_test_params(25000.0, -4.0);
|
||||
db.upsert_grid_point(&p1, 0, "wf_list_a").await.unwrap();
|
||||
db.upsert_grid_point(&p2, 0, "wf_list_a").await.unwrap();
|
||||
dispatch_and_report(
|
||||
&db,
|
||||
"wf_list_a",
|
||||
&p1,
|
||||
common::models::TaskType::ColdRun,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(&db, "wf_list_a", &p1, "cold_run", None, true).await;
|
||||
|
||||
// 无 token → 401
|
||||
let res = app
|
||||
@@ -2923,7 +2883,7 @@ async fn test_list_workflows_inline_stats() {
|
||||
let wf_b = list.iter().find(|w| w["name"] == "wf_list_b").unwrap();
|
||||
|
||||
assert_eq!(wf_a["stats"]["total"], 2);
|
||||
assert_eq!(wf_a["stats"]["converged"], 1);
|
||||
assert_eq!(wf_a["stats"]["completed"], 1);
|
||||
assert_eq!(wf_a["stats"]["cold_run_converged"], 1);
|
||||
assert_eq!(wf_a["stats"]["failed"], 0);
|
||||
assert_eq!(wf_a["stats"]["running"], 0);
|
||||
@@ -2985,15 +2945,7 @@ async fn test_wf_progress_endpoint() {
|
||||
db.record_progress_snapshot("wf_prog").await.unwrap(),
|
||||
"首次记录应写入"
|
||||
);
|
||||
dispatch_and_report(
|
||||
&db,
|
||||
"wf_prog",
|
||||
&p1,
|
||||
common::models::TaskType::ColdRun,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(&db, "wf_prog", &p1, "cold_run", None, true).await;
|
||||
assert!(
|
||||
db.record_progress_snapshot("wf_prog").await.unwrap(),
|
||||
"计数变化应写入"
|
||||
@@ -3008,7 +2960,7 @@ async fn test_wf_progress_endpoint() {
|
||||
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||
conn.execute(
|
||||
"UPDATE workflow_progress_snapshots SET ts = datetime('now', '-2 hours') \
|
||||
WHERE workflow_name = 'wf_prog' AND converged = 0",
|
||||
WHERE workflow_name = 'wf_prog' AND completed = 0",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
@@ -3057,7 +3009,7 @@ async fn test_wf_progress_endpoint() {
|
||||
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||
conn.execute(
|
||||
"UPDATE workflow_progress_snapshots SET ts = datetime('now', '-10 days') \
|
||||
WHERE workflow_name = 'wf_prog' AND converged = 0",
|
||||
WHERE workflow_name = 'wf_prog' AND completed = 0",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
@@ -3171,7 +3123,6 @@ async fn test_duplicate_failure_report_cannot_flip_converged() {
|
||||
task_id: uuid::Uuid::new_v4(),
|
||||
point_name: name.clone(),
|
||||
params: params.clone(),
|
||||
task_type: common::models::TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 7200,
|
||||
workflow_name: Some("wf_flip".to_string()),
|
||||
@@ -3201,7 +3152,7 @@ async fn test_duplicate_failure_report_cannot_flip_converged() {
|
||||
params: Some(params.clone()),
|
||||
node_id: "node-flip".to_string(),
|
||||
status: common::models::TaskStatus::Completed,
|
||||
converged: true,
|
||||
result_valid: true,
|
||||
max_relc: Some(0.0005),
|
||||
atmosphere_has_nan: false,
|
||||
elapsed_sec: 120.0,
|
||||
@@ -3231,7 +3182,7 @@ async fn test_duplicate_failure_report_cannot_flip_converged() {
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.0,
|
||||
"converged"
|
||||
"completed"
|
||||
);
|
||||
|
||||
// ---- 任务 B：迟到的重复失败报告（涡旋残留任务的典型行为）----
|
||||
@@ -3239,7 +3190,6 @@ async fn test_duplicate_failure_report_cannot_flip_converged() {
|
||||
task_id: uuid::Uuid::new_v4(),
|
||||
point_name: name.clone(),
|
||||
params: params.clone(),
|
||||
task_type: common::models::TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 7200,
|
||||
workflow_name: Some("wf_flip".to_string()),
|
||||
@@ -3269,7 +3219,7 @@ async fn test_duplicate_failure_report_cannot_flip_converged() {
|
||||
params: Some(params.clone()),
|
||||
node_id: "node-flip".to_string(),
|
||||
status: common::models::TaskStatus::Failed,
|
||||
converged: false,
|
||||
result_valid: false,
|
||||
max_relc: Some(9.5e5),
|
||||
atmosphere_has_nan: false,
|
||||
elapsed_sec: 130.0,
|
||||
@@ -3305,7 +3255,7 @@ async fn test_duplicate_failure_report_cannot_flip_converged() {
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.0,
|
||||
"converged",
|
||||
"completed",
|
||||
"迟到失败报告不得翻黑 converged 点"
|
||||
);
|
||||
assert!(
|
||||
@@ -3384,7 +3334,7 @@ async fn test_save_workflow_validates_stage_configs() {
|
||||
|
||||
// 1. 双阶段全关 → 400
|
||||
let both_off = format!(
|
||||
"{}\ntlusty:\n enabled: false\n policy: skip_converged\n strategies: [cold_run, seed_step]\n\
|
||||
"{}\ntlusty_stage:\n enabled: false\n policy: skip_converged\n strategies: [cold_run, seed_step]\n\
|
||||
synspec_stage:\n enabled: false\n policy: skip_converged\n strategies: [standard]\n",
|
||||
base_yaml
|
||||
);
|
||||
@@ -3393,7 +3343,7 @@ async fn test_save_workflow_validates_stage_configs() {
|
||||
|
||||
// 2. 启用阶段空策略链 → 400
|
||||
let empty_chain = format!(
|
||||
"{}\ntlusty:\n enabled: true\n policy: skip_converged\n strategies: []\n",
|
||||
"{}\ntlusty_stage:\n enabled: true\n policy: skip_converged\n strategies: []\n",
|
||||
base_yaml
|
||||
);
|
||||
let res = post_save("wf_val_b", &empty_chain).await;
|
||||
@@ -3405,7 +3355,7 @@ async fn test_save_workflow_validates_stage_configs() {
|
||||
|
||||
// 3. 合法：TLUSTY 关 + SYNSPEC 启（设计 §2.2 场景 B：仅更新光谱）→ 200
|
||||
let syn_only = format!(
|
||||
"{}\ntlusty:\n enabled: false\n policy: skip_converged\n strategies: [cold_run, seed_step]\n\
|
||||
"{}\ntlusty_stage:\n enabled: false\n policy: skip_converged\n strategies: [cold_run, seed_step]\n\
|
||||
synspec_stage:\n enabled: true\n policy: force_recompute\n strategies: [standard]\n",
|
||||
base_yaml
|
||||
);
|
||||
|
||||
@@ -66,7 +66,7 @@ async fn test_same_workflow_name_preserves_converged() {
|
||||
let st = db.get_grid_point_status(name, "sdB_cno").await.unwrap();
|
||||
assert_eq!(
|
||||
st.unwrap().0,
|
||||
"converged",
|
||||
"completed",
|
||||
"同名工作流：导入的 converged 应被保留，避免重算"
|
||||
);
|
||||
println!("✓ 场景1（同名）：status=converged，旧结果被保留，不会重算");
|
||||
@@ -96,7 +96,7 @@ async fn test_different_workflow_name_causes_recompute() {
|
||||
|
||||
// imported 分区：converged（种子库有，但不会被 sdB_cno 调度看到）
|
||||
let st_imp = db.get_grid_point_status(name, "imported").await.unwrap();
|
||||
assert_eq!(st_imp.unwrap().0, "converged");
|
||||
assert_eq!(st_imp.unwrap().0, "completed");
|
||||
|
||||
// sdB_cno 分区：pending（重新算！看不到 imported 的 converged）
|
||||
let st_real = db.get_grid_point_status(name, "sdB_cno").await.unwrap();
|
||||
@@ -138,7 +138,7 @@ async fn test_mixed_grid_import_then_init_avoids_recompute() {
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
st_old.unwrap().0,
|
||||
"converged",
|
||||
"completed",
|
||||
"旧点应保持 converged 不重算"
|
||||
);
|
||||
|
||||
@@ -192,7 +192,7 @@ async fn test_precision_diff_import_then_init_preserves_converged() {
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
st.unwrap().0,
|
||||
"converged",
|
||||
"completed",
|
||||
"精度一致(g5.0=g5.0)时导入的 converged 必须保留，不重算"
|
||||
);
|
||||
println!("✓ 场景4（精度差异命门）：g5.0 入库 + initialize_grid → converged 保留，避免重算");
|
||||
|
||||
Reference in New Issue
Block a user