feat(all): 任务引擎双阶段解耦、僵尸涡旋修复、动态 CPU 配额与前端详情页重构
将 TLUSTY/SYNSPEC 拆为各自独立的 enabled/policy/strategies 阶段,
以策略链自动弹栈取代单级 seed_step 布尔回退;定向修复 2026-08-02
僵尸任务涡旋事故;新增节点并发配额热调;前端详情页从 1412 行巨型
视图拆为薄控制器 + detail 子模块,并补齐工具层与单测。
引擎与调度(task_engine_decoupling_design.md)
- models.rs: 新增 StagePolicy / EngineStageConfig / TaskSpec 阶段字段、
normalize_compat() 校正旧版在途消息策略链、failed_stage 归因
- scheduler.rs: resolve_dispatchable_chain 派发门控、
trigger_strategy_fallback 按 failed_stage 精确弹栈;启动期
force_recompute/skip_converged(默认)/skip_failed 三策略
- db.rs: tasks 表 +7 列持久化阶段配置;终态守卫
(mark_grid_point_running 仅 pending/queued→running;
record_task_report 拒绝迟到失败翻黑 converged);策略弹栈快照
僵尸涡旋修复(runbook-20260802-zombie-vortex-fix.md)
- 全链路跨库活性交叉校验:派发/claim/孤儿回收/回退统一查 MQ 队列活性,
活则放行、死则清僵尸,结构性消除"每点重复派发"
- stop/重启卫生:清队列同步 delete_tasks_by_ids,杜绝遗留 pending 行
- report_task: 幂等吸收 + 409 区分迟到冗余结果,仅 state_changed 时回退
- MQ: NULL workflow_name 回填 __legacy__、requeue 后迟到上报被 403 竞态修复
动态 CPU 配额(dynamic_cpu_slots_design.md)
- admin.rs: POST /admin/nodes/:id/quota(Option<Option<i32>> 区分
缺字段/显式 null);nodes 表 +admin_max_slots
- worker.rs: effective_max_slots = min(admin, physical),心跳下发原子生效
科学产物保全(tlusty_result_artifacts.md)
- runner.rs: SYNSPEC 启动前快照 fort.12/fort.14 → .bfac/.emflux 防覆盖
- 半失败点(大气收敛+光谱失败)改判 Failed 并写入 note;仅 SYNSPEC
场景不再恒判失败;撤销归档 LRU 200 上限改为永久保留
- executor.rs: 透传 synspec_params 数值参数(此前固定 None)
前端(dashboard/)
- workflowDetail.js 1412→328 行,拆出 views/detail/{ctx,overview,
pointsTable,parSets,pointPanel}.js,AbortController 治理监听/请求生命周期
- 删除 wfActions.js,新增 wfEnginePanel.js(双阶段三维配置编辑面板)
- 新增 utils/{errors,format,icons,polling,yamlStage}.js 纯函数模块
- 路由级动态 import 代码分割;节点配额三点菜单 + Modal 管理
- 首次引入 node:test 单测(format/polling/yamlStage/psCache,644 行)
- 系统性补齐 a11y:skip-link、ARIA、Tab 键盘漫游、toast 关闭、退出动画
文档与工具
- 新增 6 篇设计/调研:引擎解耦、动态配额、涡旋 runbook、
光谱正确性分析、收敛判断、产物归档
- PIPELINE/design/api/database 等协同重写为分布式 C/S 架构口径
- scripts/fetch_results.sh 跨节点产物备份;import_results 按 cno 升序导入
- workflows/sdB_cno.yaml: 新增 tlusty/synspec_stage 配置块,修正 wstart 笔误
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
use crate::models::GridAxisValue;
|
||||
use crate::models::{EngineStageConfig, GridAxisValue};
|
||||
use anyhow::{Context, Result};
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -138,6 +138,43 @@ impl GridConfig {
|
||||
}
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
/// 解析 TLUSTY 阶段配置。
|
||||
///
|
||||
/// 优先级(见 docs/task_engine_decoupling_design.md §3):
|
||||
/// 1. 新版顶层 `tlusty:` 块(EngineStageConfig)—— 显式覆盖;
|
||||
/// 2. 旧版 `seed_step_fallback: bool` —— true → `[cold_run, seed_step]`,
|
||||
/// false → `[cold_run]`(不回退种子步进);
|
||||
/// 3. 兜底 `default_tlusty()`。
|
||||
///
|
||||
/// 注:旧版只控制是否回退种子步进,无 enabled/policy 维度,故回退路径固定
|
||||
/// enabled=true / policy=SkipConverged(与新默认一致)。
|
||||
pub fn resolve_tlusty_config(&self) -> EngineStageConfig {
|
||||
if let Some(cfg) = &self.tlusty {
|
||||
return cfg.clone();
|
||||
}
|
||||
let mut cfg = EngineStageConfig::default_tlusty();
|
||||
if !self.seed_step_fallback {
|
||||
cfg.strategies = vec!["cold_run".to_string()];
|
||||
}
|
||||
cfg
|
||||
}
|
||||
|
||||
/// 解析 SYNSPEC 阶段配置。
|
||||
///
|
||||
/// 优先级:
|
||||
/// 1. 新版顶层 `synspec_stage:` 块(EngineStageConfig)—— 显式覆盖(含 enabled 开关);
|
||||
/// 2. 兜底 `default_synspec()`(enabled=true,保持旧行为:有大气就跑光谱)。
|
||||
///
|
||||
/// 注:旧版 `synspec: SynspecConfig`(数值参数)不影响阶段启用/策略——它只携带
|
||||
/// 波长范围等数值,由调度器透传到 TaskSpec.synspec_params。如需禁用 SYNSPEC,
|
||||
/// 必须用新版 `synspec_stage: { enabled: false }`。
|
||||
pub fn resolve_synspec_config(&self) -> EngineStageConfig {
|
||||
if let Some(cfg) = &self.synspec_stage {
|
||||
return cfg.clone();
|
||||
}
|
||||
EngineStageConfig::default_synspec()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -213,7 +250,6 @@ fn default_abs_cutoff() -> f64 {
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct GridConfig {
|
||||
pub grid: GridAxesConfig,
|
||||
#[serde(default)]
|
||||
@@ -228,8 +264,7 @@ pub struct GridConfig {
|
||||
#[serde(default = "default_true")]
|
||||
pub seed_step_fallback: bool,
|
||||
/// **已弃用的死字段**:旧版 Python 工具链遗留,无任何代码读取(实际目录以
|
||||
/// `ServerConfig.seeds_dir` / `DCTS_SEEDS_DIR` 为准)。仅因 `deny_unknown_fields`
|
||||
/// 必须能解析而保留。workflow YAML 里仍可写(如 `results: data/seeds`)但被忽略。
|
||||
/// `ServerConfig.seeds_dir` / `DCTS_SEEDS_DIR` 为准)。保留以兼容旧 workflow YAML。
|
||||
#[deprecated(note = "死字段,实际目录以 DCTS_SEEDS_DIR 为准")]
|
||||
#[serde(default)]
|
||||
pub results: Option<String>,
|
||||
@@ -240,6 +275,14 @@ pub struct GridConfig {
|
||||
pub template: Option<String>,
|
||||
pub fort55: Option<String>,
|
||||
pub linelist: Option<String>,
|
||||
/// TLUSTY 阶段独立配置(见 docs/task_engine_decoupling_design.md §3)。
|
||||
/// 缺省 None → `resolve_tlusty_config()` 据旧 `seed_step_fallback` 推断默认链。
|
||||
#[serde(default)]
|
||||
pub tlusty: Option<EngineStageConfig>,
|
||||
/// SYNSPEC 阶段独立配置。命名为 `synspec_stage` 以与上方旧 `synspec: SynspecConfig`
|
||||
///(光谱合成数值参数)区分。缺省 None → `resolve_synspec_config()` 给默认 `[standard]`。
|
||||
#[serde(default)]
|
||||
pub synspec_stage: Option<EngineStageConfig>,
|
||||
}
|
||||
|
||||
fn default_grid_niter() -> Option<i32> {
|
||||
@@ -410,7 +453,7 @@ pub struct NodeConfig {
|
||||
/// 避免随沙盒删除而丢失。与 server 的 `seeds` 目录区分:此处存的是**完整产物**
|
||||
/// (光谱/连续谱/各阶段大气快照等),seeds 只存最小种子集(.7+conv.json)。
|
||||
/// 默认 "data/result",可经 DCTS_RESULT_DIR 覆盖(回退读旧 DCTS_ARCHIVE_DIR)。
|
||||
/// 超过 MAX_RESULT_MODELS 个网格点子目录时按 LRU 删除最旧的。
|
||||
/// 归档**永久保留**,不做 LRU 淘汰(2026-08-02 撤销旧 MAX_RESULT_MODELS=200 上限)。
|
||||
pub result_dir: String,
|
||||
pub heartbeat_sec: u64,
|
||||
}
|
||||
|
||||
@@ -34,9 +34,7 @@ fn parse_fortran_float(s: &str) -> Option<f64> {
|
||||
return Some(v);
|
||||
}
|
||||
// 2. 归一化无-E 记数法:[前导符号?]<尾数>(含小数点或多位数字)[+/-]<指数>
|
||||
let re = NO_E_EXP_RE.get_or_init(|| {
|
||||
Regex::new(r"^([+-]?[\d.]+)([+-]\d+)$").unwrap()
|
||||
});
|
||||
let re = NO_E_EXP_RE.get_or_init(|| Regex::new(r"^([+-]?[\d.]+)([+-]\d+)$").unwrap());
|
||||
if let Some(caps) = re.captures(s) {
|
||||
let normalized = format!("{}E{}", &caps[1], &caps[2]);
|
||||
if let Ok(v) = normalized.parse::<f64>() {
|
||||
@@ -197,9 +195,8 @@ pub fn atmosphere_has_nan(path: &Path) -> bool {
|
||||
let reader = BufReader::new(file);
|
||||
let mut total_lines = 0;
|
||||
let mut bad_lines = 0;
|
||||
let nan_re = NAN_RE.get_or_init(|| {
|
||||
Regex::new(r"(?i)(\bnan\b|\binf(?:inity)?\b|\*{3,})").unwrap()
|
||||
});
|
||||
let nan_re =
|
||||
NAN_RE.get_or_init(|| Regex::new(r"(?i)(\bnan\b|\binf(?:inity)?\b|\*{3,})").unwrap());
|
||||
|
||||
for line in reader.lines().map_while(Result::ok) {
|
||||
total_lines += 1;
|
||||
@@ -274,7 +271,9 @@ mod tests {
|
||||
|
||||
// Rust 的 f64::from_str 接受 "NaN"/"inf",返回 NaN/Inf(非 None)。
|
||||
// 这些在 check_fort9 中会被 is_finite() 判为无效 → converged=false,行为正确。
|
||||
assert!(parse_fortran_float("NaN").map(|v| v.is_nan()).unwrap_or(false));
|
||||
assert!(parse_fortran_float("NaN")
|
||||
.map(|v| v.is_nan())
|
||||
.unwrap_or(false));
|
||||
assert_eq!(parse_fortran_float("inf"), Some(f64::INFINITY));
|
||||
|
||||
// 无法解析的垃圾 → None(调用方 continue 跳过)
|
||||
|
||||
@@ -173,8 +173,13 @@ fn write_if_changed(target_path: &Path, content: &[u8], executable: bool) -> Res
|
||||
fs::set_permissions(&tmp_path, perms)?;
|
||||
}
|
||||
|
||||
fs::rename(&tmp_path, target_path)
|
||||
.with_context(|| format!("原子重命名 {} -> {} 失败", tmp_path.display(), target_path.display()))?;
|
||||
fs::rename(&tmp_path, target_path).with_context(|| {
|
||||
format!(
|
||||
"原子重命名 {} -> {} 失败",
|
||||
tmp_path.display(),
|
||||
target_path.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -332,8 +332,7 @@ mod tests {
|
||||
.filter(|l| {
|
||||
// ions 数据行:含引号且首 token 是整数
|
||||
l.contains('\'')
|
||||
&& l
|
||||
.split_whitespace()
|
||||
&& l.split_whitespace()
|
||||
.next()
|
||||
.map(|t| t.parse::<i32>().is_ok())
|
||||
.unwrap_or(false)
|
||||
|
||||
@@ -280,12 +280,121 @@ impl From<&str> for GridPointStatus {
|
||||
}
|
||||
}
|
||||
|
||||
/// 执行策略(决定如何处理历史记录)。
|
||||
///
|
||||
/// 见 docs/task_engine_decoupling_design.md §2.1:阶段独立配置三维之一。
|
||||
///
|
||||
/// **语义(2026-08-04 修正)**:策略只决定**启动工作流时**对历史终态点(converged/failed)
|
||||
/// 的处理;失败后的策略链回退**只由启动时的策略链(回退优先级排序)驱动**,不受策略门控。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum StagePolicy {
|
||||
/// 跳过已收敛、重试已失败:启动时把已失败点打回 pending 重试,收敛点保留(增量+重试失败)。
|
||||
/// 默认值。
|
||||
#[default]
|
||||
SkipConverged,
|
||||
/// 强制重算(无视历史状态与产物):启动时收敛 + 失败全部打回 pending。
|
||||
ForceRecompute,
|
||||
/// 跳过收敛及失败:启动时收敛和失败点都保留,只算从未计算过的点(最保守增量)。
|
||||
SkipFailed,
|
||||
}
|
||||
|
||||
impl StagePolicy {
|
||||
/// 序列化为 DB 文本列存储用的 snake_case 字符串。
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
StagePolicy::SkipConverged => "skip_converged",
|
||||
StagePolicy::ForceRecompute => "force_recompute",
|
||||
StagePolicy::SkipFailed => "skip_failed",
|
||||
}
|
||||
}
|
||||
|
||||
/// 从 DB 文本列回读;非法值兜底为默认 SkipConverged(防注入与脏数据)。
|
||||
pub fn from_str_lossy(s: &str) -> Self {
|
||||
match s {
|
||||
"force_recompute" => StagePolicy::ForceRecompute,
|
||||
"skip_failed" => StagePolicy::SkipFailed,
|
||||
_ => StagePolicy::SkipConverged,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 独立阶段配置(TLUSTY / SYNSPEC 各一份)。
|
||||
///
|
||||
/// 见 docs/task_engine_decoupling_design.md §3:嵌套式单阶段配置模型,
|
||||
/// 包含三个正交维度:enabled / policy / strategies。
|
||||
///
|
||||
/// 为避免与 `common::config::StageConfig`(迭代步进参数)同名冲突,命名为
|
||||
/// `EngineStageConfig`。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct EngineStageConfig {
|
||||
/// 是否在当前计算流中启用该阶段。
|
||||
#[serde(default = "default_engine_stage_enabled")]
|
||||
pub enabled: bool,
|
||||
/// 决定如何处理历史记录。
|
||||
#[serde(default)]
|
||||
pub policy: StagePolicy,
|
||||
/// 策略链队列(按回退优先级排序),如 `["cold_run", "seed_step"]`。
|
||||
/// 节点总是执行 `strategies[0]`;失败后由服务端弹出首项,下一顺位顶上。
|
||||
#[serde(default)]
|
||||
pub strategies: Vec<String>,
|
||||
}
|
||||
|
||||
fn default_engine_stage_enabled() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
impl EngineStageConfig {
|
||||
/// TLUSTY 阶段默认配置:启用、增量、策略链 `[cold_run, seed_step]`。
|
||||
pub fn default_tlusty() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
policy: StagePolicy::SkipConverged,
|
||||
strategies: vec!["cold_run".to_string(), "seed_step".to_string()],
|
||||
}
|
||||
}
|
||||
|
||||
/// SYNSPEC 阶段默认配置:启用、增量、策略链 `[standard]`。
|
||||
pub fn default_synspec() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
policy: StagePolicy::SkipConverged,
|
||||
strategies: vec!["standard".to_string()],
|
||||
}
|
||||
}
|
||||
|
||||
/// 当前应执行的策略(队列首项)。空链兜底为传入的 fallback。
|
||||
pub fn current_strategy<'a>(&'a self, fallback: &'a str) -> &'a str {
|
||||
self.strategies
|
||||
.first()
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(fallback)
|
||||
}
|
||||
|
||||
/// 是否还含指定策略(任意位置)。用于回退去重等场景。
|
||||
///
|
||||
/// 注(审查 #6):生产回退路径现走 DB 侧策略链弹栈(`pop_stage_strategy_for_fallback`),
|
||||
/// 本方法当前主要用于测试断言与诊断(判断某策略是否仍在链中),保留为公共工具。
|
||||
pub fn has_strategy(&self, name: &str) -> bool {
|
||||
self.strategies.iter().any(|s| s == name)
|
||||
}
|
||||
}
|
||||
|
||||
/// Task execution specification sent to Node
|
||||
///
|
||||
/// 注:`EngineStageConfig` 刻意**不实现 `Default`**——阶段默认值随阶段而异(TLUSTY
|
||||
/// `[cold_run, seed_step]` vs SYNSPEC `[standard]`),无中立的默认语义。构造某阶段的配置请用
|
||||
/// `..EngineStageConfig::default_tlusty()` / `..EngineStageConfig::default_synspec()`,
|
||||
/// 避免把 TLUSTY 默认链误用到 synspec。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TaskSpec {
|
||||
pub task_id: Uuid,
|
||||
pub point_name: String,
|
||||
pub params: GridPointParams,
|
||||
/// **已废弃**:保留以兼容历史 MQ 在途消息与旧节点。新代码应读
|
||||
/// `tlusty_config.strategies[0]` 判定当前 TLUSTY 策略。
|
||||
/// 该字段仍是必填(serde 反序列化要求),调度器在派发时会据
|
||||
/// `tlusty_config.strategies[0]` 同步设置它,保证旧节点能正常工作。
|
||||
pub task_type: TaskType,
|
||||
pub seed_point_name: Option<String>,
|
||||
pub timeout_sec: u64,
|
||||
@@ -297,6 +406,72 @@ pub struct TaskSpec {
|
||||
/// 旧 payload 反序列化时缺省为 0。
|
||||
#[serde(default)]
|
||||
pub wave: i32,
|
||||
/// TLUSTY 阶段独立配置(见 docs/task_engine_decoupling_design.md §3)。
|
||||
/// 旧 payload 反序列化时缺省为 `default_tlusty()`。
|
||||
#[serde(default = "EngineStageConfig::default_tlusty")]
|
||||
pub tlusty_config: EngineStageConfig,
|
||||
/// SYNSPEC 阶段独立配置。旧 payload 反序列化时缺省为 `default_synspec()`。
|
||||
#[serde(default = "EngineStageConfig::default_synspec")]
|
||||
pub synspec_config: EngineStageConfig,
|
||||
/// SYNSPEC 数值参数(波长范围等,对应 `config::SynspecConfig`)。
|
||||
/// 以 `serde_json::Value` 携带避免 models ↔ config 循环依赖;executor 侧
|
||||
/// 反序列化为 `SynspecConfig` 后透传给 runner。None → runner 用硬编码默认。
|
||||
/// 旧 payload 反序列化时缺省为 None(旧节点本就用默认,无回归)。
|
||||
#[serde(default)]
|
||||
pub synspec_params: Option<serde_json::Value>,
|
||||
/// 仅 SYNSPEC-only 场景(TLUSTY 关闭)拉取大气用:显式关联大气网格点名。
|
||||
#[serde(default)]
|
||||
pub atmosphere_ref: Option<String>,
|
||||
}
|
||||
|
||||
impl TaskSpec {
|
||||
/// 旧版兼容归一化:据废弃的 `task_type` 回填 `tlusty_config.strategies` 首项。
|
||||
///
|
||||
/// 修复(审查 #8):serde default 已把 strategies 填为完整默认链 `[cold_run, seed_step]`,
|
||||
/// 故仅判 `is_empty` 无法覆盖「旧 seed_step 消息被误判为 cold_run」的场景。
|
||||
/// 现据 task_type 把首项校正为对应的单策略链(旧消息的 task_type 是权威来源):
|
||||
/// - task_type=SeedStep → `[seed_step]`(旧热启动消息不应被当冷启动重跑);
|
||||
/// - task_type=ColdRun → 保持默认链(cold_run 本就是默认首项)。
|
||||
pub fn normalize_compat(&mut self) {
|
||||
let legacy_first = match self.task_type {
|
||||
TaskType::ColdRun => "cold_run",
|
||||
TaskType::SeedStep => "seed_step",
|
||||
};
|
||||
// 仅当当前 strategies 首项与 task_type 不一致时校正(避免覆盖显式配置)。
|
||||
let needs_fix =
|
||||
self.tlusty_config.strategies.first().map(|s| s.as_str()) != Some(legacy_first);
|
||||
if needs_fix {
|
||||
self.tlusty_config.strategies = vec![legacy_first.to_string()];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 仅供测试夹具构造便利:`TaskSpec::default()` 给出合法占位(task_id 零值、
|
||||
/// 空点/参数、ColdRun、默认阶段配置)。生产代码应显式构造所有字段,避免依赖占位。
|
||||
impl Default for TaskSpec {
|
||||
fn default() -> Self {
|
||||
TaskSpec {
|
||||
task_id: Uuid::nil(),
|
||||
point_name: String::new(),
|
||||
params: GridPointParams {
|
||||
teff: 0.0.into(),
|
||||
logg: 0.0.into(),
|
||||
loghe: 0.0.into(),
|
||||
logc: 0.0.into(),
|
||||
logn: 0.0.into(),
|
||||
logo: 0.0.into(),
|
||||
},
|
||||
task_type: TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 3600,
|
||||
workflow_name: None,
|
||||
wave: 0,
|
||||
tlusty_config: EngineStageConfig::default_tlusty(),
|
||||
synspec_config: EngineStageConfig::default_synspec(),
|
||||
synspec_params: None,
|
||||
atmosphere_ref: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
@@ -331,6 +506,10 @@ pub struct TaskReport {
|
||||
pub elapsed_sec: f64,
|
||||
pub error_message: Option<String>,
|
||||
pub summary_json: String,
|
||||
/// 失败阶段归因(见 docs/task_engine_decoupling_design.md §4.2):`"tlusty"` / `"synspec"`。
|
||||
/// 节点据 ModelSummary 推断;旧节点不携带该字段 → 服务端兜底按 TLUSTY 链回退(兼容)。
|
||||
#[serde(default)]
|
||||
pub failed_stage: Option<String>,
|
||||
}
|
||||
|
||||
/// Node registration request
|
||||
@@ -349,6 +528,22 @@ pub struct NodeHeartbeatRequest {
|
||||
pub memory_usage: f32,
|
||||
}
|
||||
|
||||
/// Node heartbeat response.
|
||||
///
|
||||
/// 设计依据见 docs/dynamic_cpu_slots_design.md:服务端在心跳响应里透传管理员设置的
|
||||
/// `admin_max_slots`(并发槽位配额上限),Worker 据此动态调整本地领用并发数,
|
||||
/// 避免 Pull 模式下被服务端强行拒绝 claim 而陷入空轮询。
|
||||
///
|
||||
/// - `status`:固定 "ok"(401/403 由 HTTP 状态码承载,不会进入反序列化路径)。
|
||||
/// - `admin_max_slots`:管理员强制配额上限(`null` 表示无限制,恢复节点物理槽位上限)。
|
||||
/// `#[serde(default)]` 保证旧服务端(响应体不含此字段)反序列化兜底为 None。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct NodeHeartbeatResponse {
|
||||
pub status: String,
|
||||
#[serde(default)]
|
||||
pub admin_max_slots: Option<i32>,
|
||||
}
|
||||
|
||||
/// Node state in database
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct NodeInfo {
|
||||
@@ -359,6 +554,10 @@ pub struct NodeInfo {
|
||||
pub cpu_usage: f32,
|
||||
pub memory_usage: f32,
|
||||
pub last_heartbeat: DateTime<Utc>,
|
||||
/// 管理员强制并发槽位上限(动态调整 CPU 核数)。None 表示无限制,使用 max_slots。
|
||||
/// 透传给前端供 Dashboard 渲染配额状态,并在心跳响应里下发给 Worker。
|
||||
#[serde(default)]
|
||||
pub admin_max_slots: Option<i32>,
|
||||
}
|
||||
|
||||
/// Single iteration convergence result parsed from fort.9
|
||||
@@ -665,4 +864,109 @@ mod tests {
|
||||
assert_eq!(GridPointStatus::from("failed"), GridPointStatus::Failed);
|
||||
assert_eq!(GridPointStatus::from("unknown"), GridPointStatus::Pending);
|
||||
}
|
||||
|
||||
/// `StagePolicy` 的 snake_case serde 往返 + DB 文本兜底。
|
||||
#[test]
|
||||
fn test_stage_policy_serde_roundtrip() {
|
||||
for p in [
|
||||
StagePolicy::SkipConverged,
|
||||
StagePolicy::ForceRecompute,
|
||||
StagePolicy::SkipFailed,
|
||||
] {
|
||||
let s = serde_json::to_string(&p).unwrap();
|
||||
let back: StagePolicy = serde_json::from_str(&s).unwrap();
|
||||
assert_eq!(p, back);
|
||||
}
|
||||
// snake_case 形态锁定(前端 payload 与 DB 列口径)
|
||||
assert_eq!(
|
||||
serde_json::to_string(&StagePolicy::SkipConverged).unwrap(),
|
||||
"\"skip_converged\""
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&StagePolicy::ForceRecompute).unwrap(),
|
||||
"\"force_recompute\""
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&StagePolicy::SkipFailed).unwrap(),
|
||||
"\"skip_failed\""
|
||||
);
|
||||
// as_str/from_str_lossy 互逆(非法值兜底 SkipConverged)
|
||||
assert_eq!(
|
||||
StagePolicy::from_str_lossy("skip_converged"),
|
||||
StagePolicy::SkipConverged
|
||||
);
|
||||
assert_eq!(
|
||||
StagePolicy::from_str_lossy("garbage"),
|
||||
StagePolicy::SkipConverged
|
||||
);
|
||||
assert_eq!(
|
||||
StagePolicy::from_str_lossy(StagePolicy::ForceRecompute.as_str()),
|
||||
StagePolicy::ForceRecompute
|
||||
);
|
||||
}
|
||||
|
||||
/// `EngineStageConfig` serde 往返 + 默认值(缺字段时 serde default 兜底)。
|
||||
#[test]
|
||||
fn test_engine_stage_config_serde_and_defaults() {
|
||||
let cfg = EngineStageConfig {
|
||||
enabled: false,
|
||||
policy: StagePolicy::ForceRecompute,
|
||||
strategies: vec!["cold_run".to_string(), "seed_step".to_string()],
|
||||
};
|
||||
let json = serde_json::to_string(&cfg).unwrap();
|
||||
let back: EngineStageConfig = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(cfg, back);
|
||||
|
||||
// 空 payload 应产出默认值(enabled=true, policy=skip_converged, strategies=[])
|
||||
let empty: EngineStageConfig = serde_json::from_str("{}").unwrap();
|
||||
assert!(empty.enabled);
|
||||
assert_eq!(empty.policy, StagePolicy::SkipConverged);
|
||||
assert!(empty.strategies.is_empty());
|
||||
|
||||
// current_strategy 空链兜底
|
||||
assert_eq!(empty.current_strategy("cold_run"), "cold_run");
|
||||
assert_eq!(cfg.current_strategy("x"), "cold_run");
|
||||
assert!(cfg.has_strategy("seed_step"));
|
||||
assert!(!cfg.has_strategy("standard"));
|
||||
}
|
||||
|
||||
/// 旧版 MQ 在途消息(仅含 task_type,无 tlusty_config)经 `#[serde(default)]`
|
||||
/// 反序列化后,`normalize_compat()` 应据 task_type 回填 strategies。
|
||||
#[test]
|
||||
fn test_task_spec_normalize_compat_from_legacy_task_type() {
|
||||
let legacy_json = r#"{
|
||||
"task_id": "00000000-0000-0000-0000-000000000001",
|
||||
"point_name": "t20000_g5.0_he-2_c-4_n-4_o-4",
|
||||
"params": {"teff": 20000.0, "logg": 5.0, "loghe": -2.0, "logc": -4.0, "logn": -4.0, "logo": -4.0},
|
||||
"task_type": "seed_step",
|
||||
"seed_point_name": "neighbor",
|
||||
"timeout_sec": 7200,
|
||||
"workflow_name": "wf_a",
|
||||
"wave": 0
|
||||
}"#;
|
||||
let mut spec: TaskSpec = serde_json::from_str(legacy_json).unwrap();
|
||||
// 修复后 normalize_compat 据 task_type 校正首项:旧 seed_step 消息的 strategies
|
||||
// 首项应被校正为 seed_step(而非保留默认链的 cold_run 首项,否则会被误当冷启动)。
|
||||
spec.normalize_compat();
|
||||
assert_eq!(
|
||||
spec.tlusty_config.strategies,
|
||||
vec!["seed_step".to_string()],
|
||||
"旧 seed_step 消息应校正为 [seed_step] 单策略链"
|
||||
);
|
||||
assert_eq!(spec.tlusty_config.current_strategy("cold_run"), "seed_step");
|
||||
|
||||
// 对照:旧 cold_run 消息 → 首项已是 cold_run(默认链首项),无需校正。
|
||||
let mut cold_spec = TaskSpec::default();
|
||||
cold_spec.task_type = TaskType::ColdRun;
|
||||
cold_spec.normalize_compat();
|
||||
assert_eq!(
|
||||
cold_spec
|
||||
.tlusty_config
|
||||
.strategies
|
||||
.first()
|
||||
.map(|s| s.as_str()),
|
||||
Some("cold_run"),
|
||||
"旧 cold_run 消息保持默认链"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,8 @@
|
||||
//!
|
||||
//! 1. **裸名保留**(有独立语义,不以 model_name 为前缀):
|
||||
//! `conv.json`、`fort.8`(synspec 输入大气)、`fort.55`(synspec 控制卡)
|
||||
//! 2. **科学核心**:`<name>.7`、`<name>.spec`、`<name>.cont`、`<name>.iden`、`<name>.log`
|
||||
//! 2. **科学核心**:`<name>.7`、`<name>.spec`、`<name>.cont`、`<name>.iden`、`<name>.log`、
|
||||
//! `<name>.bfac`(TLUSTY 最终 b 因子/非 LTE 偏离因子)、`<name>.emflux`(TLUSTY 最终出射谱 λ–Fλ)
|
||||
//! 3. **阶段快照**:`<name>.<label>.5/.6/.err/.nst/.7`(label ∈ lte/nc/nl/seed_nc)
|
||||
//! 4. **收敛诊断**:`<name>.<label>_chmax*.9`(**唯一保留的 .9**;裸 `<name>.<label>.9`
|
||||
//! 已在 runner 源头停止写出,因其与 `_chmax*.9` 内容完全重复)
|
||||
@@ -21,7 +22,10 @@
|
||||
//! 符号链接、子目录、`.tmp`、`fort.84` 及所有 Tlusty 中间单元均不在白名单内,自然被跳过。
|
||||
|
||||
/// 科学核心产物的文件名后缀(挂在 `<name>.` 之后,无阶段标签)。
|
||||
const SCIENCE_SUFFIXES: &[&str] = &["7", "spec", "cont", "iden", "log"];
|
||||
/// 除 SYNSPEC 产物(spec/cont/iden/log)外,含 runner 快照的 TLUSTY 最终产物:
|
||||
/// `bfac`(b 因子/非 LTE 偏离因子,源 fort.12)与 `emflux`(出射谱 λ–Fλ,源 fort.14),
|
||||
/// 二者若不快照会被 SYNSPEC 覆盖丢失(见 `runner::snapshot_tlusty_outputs`)。
|
||||
const SCIENCE_SUFFIXES: &[&str] = &["7", "spec", "cont", "iden", "log", "bfac", "emflux"];
|
||||
|
||||
/// 阶段快照的文件名后缀(挂在 `<name>.<label>.` 之后)。
|
||||
const STAGE_SNAPSHOT_SUFFIXES: &[&str] = &["5", "6", "err", "nst", "7"];
|
||||
@@ -120,12 +124,19 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_science_core() {
|
||||
for s in ["7", "spec", "cont", "iden", "log"] {
|
||||
for s in ["7", "spec", "cont", "iden", "log", "bfac", "emflux"] {
|
||||
let f = format!("{}.{}", NAME, s);
|
||||
assert!(is_result_worthy(&f, NAME), "{} 应归档", f);
|
||||
}
|
||||
}
|
||||
|
||||
/// TLUSTY 快照产物(b 因子 / 出射谱)应进入白名单,缺失时不误伤其他后缀
|
||||
#[test]
|
||||
fn test_tlusty_snapshots_kept() {
|
||||
assert!(is_result_worthy(&format!("{}.bfac", NAME), NAME));
|
||||
assert!(is_result_worthy(&format!("{}.emflux", NAME), NAME));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stage_snapshots() {
|
||||
// 各阶段标签 × 快照后缀 都应保留
|
||||
|
||||
+170
-25
@@ -113,13 +113,12 @@ async fn run_child_async_with_timeout(
|
||||
timeout_sec: u64,
|
||||
shutdown: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
|
||||
) -> Result<std::process::ExitStatus> {
|
||||
let timeout_fut = tokio::time::timeout(
|
||||
tokio::time::Duration::from_secs(timeout_sec),
|
||||
child.wait(),
|
||||
);
|
||||
let timeout_fut =
|
||||
tokio::time::timeout(tokio::time::Duration::from_secs(timeout_sec), child.wait());
|
||||
|
||||
// 若提供了 shutdown 标志,则与超时/正常结束三路 select;否则只等超时/正常结束。
|
||||
let outcome: Result<std::process::ExitStatus, ShutdownOrTimeout> = if let Some(flag) = shutdown {
|
||||
let outcome: Result<std::process::ExitStatus, ShutdownOrTimeout> = if let Some(flag) = shutdown
|
||||
{
|
||||
let shutdown_watcher = async move {
|
||||
// 轮询 shutdown 标志(10ms 粒度足够灵敏,开销可忽略)。
|
||||
loop {
|
||||
@@ -148,20 +147,12 @@ async fn run_child_async_with_timeout(
|
||||
Ok(status) => Ok(status),
|
||||
Err(ShutdownOrTimeout::Shutdown) => {
|
||||
let _ = child.start_kill();
|
||||
let _ = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
child.wait(),
|
||||
)
|
||||
.await;
|
||||
let _ = tokio::time::timeout(std::time::Duration::from_secs(30), child.wait()).await;
|
||||
anyhow::bail!("节点收到退出信号,子进程已被终止");
|
||||
}
|
||||
Err(ShutdownOrTimeout::Timeout) => {
|
||||
let _ = child.start_kill();
|
||||
let _ = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
child.wait(),
|
||||
)
|
||||
.await;
|
||||
let _ = tokio::time::timeout(std::time::Duration::from_secs(30), child.wait()).await;
|
||||
anyhow::bail!("进程计算超时 (上限: {} 秒)", timeout_sec);
|
||||
}
|
||||
}
|
||||
@@ -173,6 +164,31 @@ enum ShutdownOrTimeout {
|
||||
Timeout,
|
||||
}
|
||||
|
||||
/// 快照 TLUSTY 最终模型的 b 因子与出射谱,防止被 SYNSPEC 覆盖丢失。
|
||||
///
|
||||
/// TLUSTY 在最终迭代(`LFIN=.TRUE.`)经 `OUTPRI` 写出(见 tlusty208.f):
|
||||
/// - `fort.12`:b 因子 / 非 LTE 偏离因子表(头 2I5 + 每深度 TEMP/ELEC/DENS/BFAC,格式 701/702/703)。
|
||||
/// 随后 SYNSPEC 会复用 unit 12 写谱线证认表并覆盖它(runner 再将其存为 `<name>.iden`),
|
||||
/// 故 TLUSTY 的 b 因子若不在此快照即静默丢失。
|
||||
/// - `fort.14`:出射谱(波长 Å + Fλ,格式 614),同样会被 SYNSPEC 的谱线数据覆盖。
|
||||
///
|
||||
/// 在收敛链循环结束(链上最后一次 TLUSTY 运行即最终模型)、SYNSPEC 启动前调用,
|
||||
/// 快照为 `<name>.bfac` / `<name>.emflux`,与科学核心产物一并进入归档白名单
|
||||
/// (见 `result_filter::is_result_worthy` 的 `bfac`/`emflux` 后缀)。
|
||||
/// 文件不存在时静默跳过(TLUSTY 未运行/未写出);IO 错误降级为 warn,不阻断主流程。
|
||||
async fn snapshot_tlusty_outputs(model_dir: &Path, name: &str) {
|
||||
for (src, suffix) in [("fort.12", "bfac"), ("fort.14", "emflux")] {
|
||||
let src_path = model_dir.join(src);
|
||||
if !src_path.is_file() {
|
||||
continue;
|
||||
}
|
||||
let dst = model_dir.join(format!("{}.{}", name, suffix));
|
||||
if let Err(e) = tokio::fs::copy(&src_path, &dst).await {
|
||||
warn!("快照 TLUSTY {} 到 {} 失败: {}", src, dst.display(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ExecutionRunner<'a> {
|
||||
pub runtime: &'a RuntimePaths,
|
||||
pub work_dir: PathBuf,
|
||||
@@ -199,12 +215,19 @@ impl<'a> ExecutionRunner<'a> {
|
||||
custom_chain,
|
||||
seed_atmos,
|
||||
synspec_cfg,
|
||||
true,
|
||||
true,
|
||||
7200,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// 阶段独立配置执行入口(见 docs/task_engine_decoupling_design.md §5)。
|
||||
///
|
||||
/// `tlusty_enabled` / `synspec_enabled` 控制各阶段是否运行:
|
||||
/// - TLUSTY 关闭:跳过 chain 循环,直接以 seed_atmos(或单独拉取的大气)作 final_7;
|
||||
/// - SYNSPEC 关闭:跳过光谱合成块(即便 final_7 存在)。
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn run_model_with_timeout(
|
||||
&self,
|
||||
@@ -214,6 +237,8 @@ impl<'a> ExecutionRunner<'a> {
|
||||
custom_chain: Option<Vec<StageConfig>>,
|
||||
seed_atmos: Option<&Path>,
|
||||
synspec_cfg: Option<&SynspecConfig>,
|
||||
tlusty_enabled: bool,
|
||||
synspec_enabled: bool,
|
||||
timeout_sec: u64,
|
||||
shutdown: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
|
||||
) -> Result<ModelSummary> {
|
||||
@@ -282,7 +307,20 @@ impl<'a> ExecutionRunner<'a> {
|
||||
let mut final_chmax: Option<f64> = None;
|
||||
let mut final_max_relc: Option<f64> = None;
|
||||
|
||||
// 阶段独立配置(见 docs/task_engine_decoupling_design.md §5):
|
||||
// TLUSTY 关闭时跳过整个 chain 循环——current_seed 直接作为 final_7 来源,
|
||||
// 适配「仅 SYNSPEC」场景(用既有大气合成光谱,不重算大气结构)。
|
||||
if tlusty_enabled {
|
||||
info!("TLUSTY 阶段启用:执行 {} 步收敛链", chain.len());
|
||||
} else {
|
||||
info!("TLUSTY 阶段关闭:跳过大气结构计算,直接进入 SYNSPEC 阶段");
|
||||
}
|
||||
|
||||
let tlusty_skipped = !tlusty_enabled;
|
||||
for stage_def in &chain {
|
||||
if tlusty_skipped {
|
||||
break;
|
||||
}
|
||||
let stage_t0 = Instant::now();
|
||||
let metals = stage_def.metals.as_deref().unwrap_or("cno");
|
||||
let input5_text = make_input5(
|
||||
@@ -335,7 +373,8 @@ impl<'a> ExecutionRunner<'a> {
|
||||
.kill_on_drop(true)
|
||||
.spawn()?;
|
||||
|
||||
let status_res = run_child_async_with_timeout(child, timeout_sec, shutdown.clone()).await;
|
||||
let status_res =
|
||||
run_child_async_with_timeout(child, timeout_sec, shutdown.clone()).await;
|
||||
let rc = match status_res {
|
||||
Ok(st) => st.code().unwrap_or(-1),
|
||||
Err(e) => {
|
||||
@@ -440,17 +479,27 @@ impl<'a> ExecutionRunner<'a> {
|
||||
let _ = tokio::fs::copy(model_dir.join("fort.7"), &final_7).await;
|
||||
}
|
||||
|
||||
// TLUSTY 最终 b 因子 + 出射谱快照。必须在此处(SYNSPEC 覆盖 fort.12/fort.14 之前)
|
||||
// 完成:synspec 会复用 unit 12/14 写谱线数据,覆盖 TLUSTY 的最终产物(见上方
|
||||
// snapshot_tlusty_outputs 的注释)。仅 SYNSPEC 场景(tlusty_enabled=false)下
|
||||
// fort.12/14 不存在,函数内按文件是否存在静默跳过。
|
||||
snapshot_tlusty_outputs(&model_dir, name).await;
|
||||
|
||||
let atmo_has_nan = atmosphere_has_nan(&final_7);
|
||||
if atmo_has_nan {
|
||||
final_converged = false;
|
||||
}
|
||||
|
||||
// Run synspec if final .7 atmosphere exists
|
||||
// Run synspec if enabled and final .7 atmosphere exists
|
||||
let mut synspec_rc = None;
|
||||
let mut synspec_err = None;
|
||||
let mut synspec_sec = None;
|
||||
|
||||
if final_7.is_file() {
|
||||
if !synspec_enabled {
|
||||
// SYNSPEC 关闭是合法配置(TLUSTY-only 大气计算),不写入 synspec_error
|
||||
// 以免污染 conv.json 的错误归因——下游把非空 synspec_error 当「光谱有缺陷」。
|
||||
info!("SYNSPEC 阶段关闭:跳过光谱合成(TLUSTY-only 模式)");
|
||||
} else if final_7.is_file() {
|
||||
let syn_t0 = Instant::now();
|
||||
// H10:synspec 输入文件(fort.8 大气 / fort.55 控制卡)写入失败不可静默吞掉。
|
||||
// 历史上用 `let _ =` 忽略错误,磁盘满/inode 耗尽时 synspec 会读到旧/缺失的
|
||||
@@ -480,7 +529,10 @@ impl<'a> ExecutionRunner<'a> {
|
||||
};
|
||||
let fort55_text = generate_fort55_content(synspec_cfg.unwrap_or(&default_cfg));
|
||||
if let Err(e) = tokio::fs::write(&fort55_path, &fort55_text).await {
|
||||
warn!("synspec 输入 fort.55 (控制卡) 写入失败,跳过 synspec: {}", e);
|
||||
warn!(
|
||||
"synspec 输入 fort.55 (控制卡) 写入失败,跳过 synspec: {}",
|
||||
e
|
||||
);
|
||||
synspec_err = Some(format!("fort.55 write failed: {}", e));
|
||||
} else {
|
||||
#[cfg(unix)]
|
||||
@@ -511,7 +563,8 @@ impl<'a> ExecutionRunner<'a> {
|
||||
|
||||
let synspec_timeout_sec = 600_u64.min(timeout_sec);
|
||||
let status_res =
|
||||
run_child_async_with_timeout(child, synspec_timeout_sec, shutdown.clone()).await;
|
||||
run_child_async_with_timeout(child, synspec_timeout_sec, shutdown.clone())
|
||||
.await;
|
||||
let rc = match status_res {
|
||||
Ok(st) => st.code().unwrap_or(-1),
|
||||
Err(e) => {
|
||||
@@ -549,6 +602,19 @@ impl<'a> ExecutionRunner<'a> {
|
||||
synspec_err = Some("No atmosphere .7 produced".to_string());
|
||||
}
|
||||
|
||||
// 收敛判定(见 docs/task_engine_decoupling_design.md §5):
|
||||
// - TLUSTY 启用:final_converged 由 chain 循环内各阶段收敛状态累积得出(既有逻辑)。
|
||||
// - TLUSTY 关闭(仅 SYNSPEC 场景):final_converged 不能恒为 false——否则成功的
|
||||
// 光谱合成任务被误判失败并触发策略回退。此时收敛 = 大气加载干净(无 NaN)且
|
||||
// SYNSPEC 成功(rc=0)或 SYNSPEC 也关闭(TLUSTY-only 等价的纯校验场景,虽罕见)。
|
||||
// 仅 SYNSPEC 场景下大气来自既有产物(非本任务重算),NaN 检查仍必要(产物可能损坏)。
|
||||
if !tlusty_enabled && !atmo_has_nan {
|
||||
final_converged = match synspec_rc {
|
||||
Some(rc) => rc == 0,
|
||||
None => !synspec_enabled, // SYNSPEC 也关闭 → 仅校验大气,干净即收敛
|
||||
};
|
||||
}
|
||||
|
||||
// 清理冗余的裸文件:这些文件的内容已被带阶段标签的快照或重命名的科学产物覆盖,
|
||||
// 保留它们只会与归档里的 <name>.<label>.* / <name>.iden / <name>.cont 等重复(尤其
|
||||
// .spec/.cont 是大文件,双份存储浪费磁盘)。删除后归档目录干净无冗余。
|
||||
@@ -569,6 +635,29 @@ impl<'a> ExecutionRunner<'a> {
|
||||
}
|
||||
|
||||
let elapsed_sec = t0.elapsed().as_secs_f64();
|
||||
|
||||
// 汇总 note(修复审查 #2 后续):半失败点(大气已收敛 + 光谱失败)须让
|
||||
// synspec 的错误可见——此前 synspec rc≠0 时 note 恒为 None,上报的
|
||||
// error_message 为空,attempts 表与详情面板无从排查失败原因。
|
||||
let note = {
|
||||
let mut notes: Vec<String> = Vec::new();
|
||||
if atmo_has_nan {
|
||||
notes.push("Invalidated: atmosphere contains >10% NaN lines".to_string());
|
||||
}
|
||||
if let Some(ref err) = synspec_err {
|
||||
notes.push(format!("synspec error: {}", err));
|
||||
} else if let Some(rc) = synspec_rc {
|
||||
if rc != 0 {
|
||||
notes.push(format!("synspec rc={}", rc));
|
||||
}
|
||||
}
|
||||
if notes.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(notes.join("; "))
|
||||
}
|
||||
};
|
||||
|
||||
let summary = ModelSummary {
|
||||
name: name.to_string(),
|
||||
params: params.clone(),
|
||||
@@ -582,11 +671,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
synspec_error: synspec_err,
|
||||
synspec_sec,
|
||||
elapsed_sec,
|
||||
note: if atmo_has_nan {
|
||||
Some("Invalidated: atmosphere contains >10% NaN lines".to_string())
|
||||
} else {
|
||||
None
|
||||
},
|
||||
note,
|
||||
};
|
||||
|
||||
// Write conv.json
|
||||
@@ -599,6 +684,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::snapshot_tlusty_outputs;
|
||||
use crate::models::{GridAxisValue, GridPointParams};
|
||||
|
||||
#[test]
|
||||
@@ -647,4 +733,63 @@ mod tests {
|
||||
let authoritative_name = point_name; // 即 executor 传入的 task.point_name
|
||||
assert_eq!(authoritative_name, "t20000_g5.0_he-2_c-4_n-4_o-4");
|
||||
}
|
||||
|
||||
/// 快照 TLUSTY b 因子与出射谱:fort.12→`<name>.bfac`、fort.14→`<name>.emflux`,
|
||||
/// 缺失的源文件静默跳过,未列入快照的 fort.13 不受影响。
|
||||
#[tokio::test]
|
||||
async fn test_snapshot_tlusty_outputs() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let name = "t20000_g5.0_he-2_c-4_n-4_o-4";
|
||||
let model_dir = dir.path().join(name);
|
||||
tokio::fs::create_dir_all(&model_dir).await.unwrap();
|
||||
|
||||
// TLUSTY 最终迭代产物:b 因子(fort.12)与出射谱(fort.14)
|
||||
tokio::fs::write(model_dir.join("fort.12"), "bfac payload")
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::fs::write(model_dir.join("fort.14"), "emflux payload")
|
||||
.await
|
||||
.unwrap();
|
||||
// 不参与快照的文件(出射辐射场 fort.13、大气 fort.7)
|
||||
tokio::fs::write(model_dir.join("fort.13"), "emrad payload")
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::fs::write(model_dir.join("fort.7"), "atmo payload")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
snapshot_tlusty_outputs(&model_dir, name).await;
|
||||
|
||||
assert_eq!(
|
||||
tokio::fs::read_to_string(model_dir.join(format!("{}.bfac", name)))
|
||||
.await
|
||||
.unwrap(),
|
||||
"bfac payload"
|
||||
);
|
||||
assert_eq!(
|
||||
tokio::fs::read_to_string(model_dir.join(format!("{}.emflux", name)))
|
||||
.await
|
||||
.unwrap(),
|
||||
"emflux payload"
|
||||
);
|
||||
// fort.13 未列入快照,不应生成 <name>.emrad
|
||||
assert!(!model_dir.join(format!("{}.emrad", name)).exists());
|
||||
// 原 fort.12/fort.14 保留(后续 synspec 覆盖前仍作为单元文件存在)
|
||||
assert!(model_dir.join("fort.12").is_file());
|
||||
assert!(model_dir.join("fort.14").is_file());
|
||||
}
|
||||
|
||||
/// 缺失源文件(仅 SYNSPEC 场景,tlusty 未运行)时快照应是无害 no-op
|
||||
#[tokio::test]
|
||||
async fn test_snapshot_tlusty_outputs_noop_when_missing() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let name = "t20000_g5.0_he-2_c-4_n-4_o-4";
|
||||
let model_dir = dir.path().join(name);
|
||||
tokio::fs::create_dir_all(&model_dir).await.unwrap();
|
||||
|
||||
snapshot_tlusty_outputs(&model_dir, name).await;
|
||||
|
||||
assert!(!model_dir.join(format!("{}.bfac", name)).exists());
|
||||
assert!(!model_dir.join(format!("{}.emflux", name)).exists());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,8 +26,14 @@ pub const MAX_GLOBAL_SEED_DISTANCE: f64 = 3.0;
|
||||
fn directed_cno_distance(cand: &GridPointParams, target: &GridPointParams) -> f64 {
|
||||
const RICH_PENALTY: f64 = 4.0; // 目标比种子富 → 该方向微扰不稳定,重罚
|
||||
const POOR_PENALTY: f64 = 1.0; // 目标比种子贫 → 该方向微扰稳定,轻罚
|
||||
// delta = target − cand:正 = 目标更富(坏方向),负 = 目标更贫(好方向)
|
||||
let penalize = |delta: f64| if delta > 0.0 { delta * RICH_PENALTY } else { -delta * POOR_PENALTY };
|
||||
// delta = target − cand:正 = 目标更富(坏方向),负 = 目标更贫(好方向)
|
||||
let penalize = |delta: f64| {
|
||||
if delta > 0.0 {
|
||||
delta * RICH_PENALTY
|
||||
} else {
|
||||
-delta * POOR_PENALTY
|
||||
}
|
||||
};
|
||||
penalize(target.logc.value() - cand.logc.value())
|
||||
+ penalize(target.logn.value() - cand.logn.value())
|
||||
+ penalize(target.logo.value() - cand.logo.value())
|
||||
@@ -64,7 +70,14 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::models::GridAxisValue;
|
||||
|
||||
fn params(teff: f64, logg: f64, loghe: f64, logc: f64, logn: f64, logo: f64) -> GridPointParams {
|
||||
fn params(
|
||||
teff: f64,
|
||||
logg: f64,
|
||||
loghe: f64,
|
||||
logc: f64,
|
||||
logn: f64,
|
||||
logo: f64,
|
||||
) -> GridPointParams {
|
||||
GridPointParams {
|
||||
teff: GridAxisValue::from_value(teff),
|
||||
logg: GridAxisValue::from_value(logg),
|
||||
@@ -79,7 +92,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_directed_cno_distance_favors_poor_metal_direction() {
|
||||
let seed = params(40000.0, 6.0, -2.0, -2.0, -2.0, -2.0); // 种子 CNO=-6
|
||||
// 贫方向:目标 CNO=-7(种子更富,目标往贫走),|Δ|=1
|
||||
// 贫方向:目标 CNO=-7(种子更富,目标往贫走),|Δ|=1
|
||||
let poor_target = params(40000.0, 6.0, -2.0, -3.0, -2.0, -2.0);
|
||||
// 富方向:目标 CNO=-5(目标更富),|Δ|=1,同一分量、同幅度
|
||||
let rich_target = params(40000.0, 6.0, -2.0, -1.0, -2.0, -2.0);
|
||||
@@ -99,9 +112,9 @@ mod tests {
|
||||
#[test]
|
||||
fn test_exact_family_prefers_poor_direction_seed() {
|
||||
let target = params(40000.0, 6.0, -2.0, -2.0, -2.0, -2.0); // 目标 CNO=-6
|
||||
// 候选A:富方向种子(目标比种子富),CNO 绝对差=1
|
||||
// 候选A:富方向种子(目标比种子富),CNO 绝对差=1
|
||||
let rich_seed = params(40000.0, 6.0, -2.0, -3.0, -2.0, -2.0); // CNO=-7, 目标更富
|
||||
// 候选B:贫方向种子(目标比种子贫),CNO 绝对差=2(更大)
|
||||
// 候选B:贫方向种子(目标比种子贫),CNO 绝对差=2(更大)
|
||||
let poor_seed = params(40000.0, 6.0, -2.0, -1.0, -1.0, -2.0); // CNO=-4, 目标更贫
|
||||
let (_, d_rich) = calculate_seed_distance(&rich_seed, &target);
|
||||
let (_, d_poor) = calculate_seed_distance(&poor_seed, &target);
|
||||
|
||||
+353
-32
@@ -99,6 +99,14 @@ impl SqliteTaskQueue {
|
||||
[],
|
||||
)?;
|
||||
}
|
||||
// H1 修复:历史遗留行(旧版在途任务)的 workflow_name 为 NULL。回填为
|
||||
// '__legacy__'(与主库 grid_points 迁移口径一致),否则新节点领用/上报时
|
||||
// claim/report 无法定向更新 '__legacy__' 网格点,旧任务结算后点永久卡死。
|
||||
// 幂等:新 push 的行恒带 workflow_name,NULL 行只会出现在迁移遗留,重复执行无害。
|
||||
conn.execute(
|
||||
"UPDATE task_queue SET workflow_name = '__legacy__' WHERE workflow_name IS NULL",
|
||||
[],
|
||||
)?;
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_task_queue_status_created ON task_queue(status, created_at)",
|
||||
[],
|
||||
@@ -204,6 +212,15 @@ impl SqliteTaskQueue {
|
||||
}
|
||||
};
|
||||
|
||||
// 旧版兼容归一化(见 models.rs::normalize_compat 文档):
|
||||
// 旧 MQ 在途消息可能只含 task_type、无 tlusty_config 字段。serde default
|
||||
// 会把 tlusty_config.strategies 填为完整默认链 [cold_run, seed_step],
|
||||
// 导致旧 SeedStep 热启动消息首项被误判为 cold_run。normalize_compat
|
||||
// 据 task_type 校正首项策略。在 pop 出队后立即调用,保证节点拿到的是
|
||||
// 语义正确的策略链。对新版消息(已显式设置 tlusty_config)无副作用。
|
||||
let mut task = task;
|
||||
task.normalize_compat();
|
||||
|
||||
// 记录任务归属:claim 时写入领用方 node_id,供 report 阶段校验,
|
||||
// 杜绝「节点 A 领用、节点 B 上报」的跨节点伪造结果投毒。
|
||||
tx.execute(
|
||||
@@ -270,10 +287,15 @@ impl SqliteTaskQueue {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("Queue DB pool error: {}", e))?;
|
||||
// 仅 claimed 态(尚未被 report 清理)且归属匹配才算有效领用
|
||||
// claimed 或 pending 态 + 归属匹配才算有效领用。status 含 'pending' 的缘由:
|
||||
// `requeue_stale_tasks` 会把超时 claimed 行打回 pending(保留 claimed_by_node_id,
|
||||
// 仅清 claimed_at)。若任务实际仍在运行、只是耗时接近 timeout,原节点(仍持
|
||||
// 归属)迟到的上报若被拒绝 → 403 → 昂贵计算结果被静默丢弃、重投重算(代码注释
|
||||
// 记录的竞态,旧实现靠 stale_sec ≥ 3×timeout 缓冲而非根治)。放行 pending 态后
|
||||
// 该竞态消除;重新领用会覆盖 claimed_by_node_id,故跨节点伪造上报仍被拦截。
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT payload FROM task_queue
|
||||
WHERE task_id = ?1 AND claimed_by_node_id = ?2 AND status = 'claimed' LIMIT 1",
|
||||
WHERE task_id = ?1 AND claimed_by_node_id = ?2 AND status IN ('claimed', 'pending') LIMIT 1",
|
||||
)?;
|
||||
let row = stmt.query_row(params![task_id, claimant], |r| r.get::<_, String>(0));
|
||||
match row {
|
||||
@@ -345,7 +367,7 @@ impl SqliteTaskQueue {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 仅清理指定工作流的**未被领取**的排队任务。
|
||||
/// 仅清理指定工作流的**未被领取**的排队任务,并返回被删行的 task_id 列表。
|
||||
///
|
||||
/// 用于 stop_workflow / initialize_grid 按工作流隔离清理,避免在多工作流场景下
|
||||
/// 误清其他工作流的任务。
|
||||
@@ -354,21 +376,55 @@ impl SqliteTaskQueue {
|
||||
/// 必须保留领用凭证,否则节点上报时 verify_task_claim 找不到记录 → 403 →
|
||||
/// 计算结果丢失、网格点永久卡在 running(#6 修复)。
|
||||
/// `claimed` 行会在上报成功后由 remove_task 自然清理。
|
||||
pub async fn clear_queue_by_workflow(&self, workflow_name: &str) -> Result<()> {
|
||||
///
|
||||
/// 返回被删 task_id 供调用方同步清理主库 `tasks` 审计表对应行(2026-08-02 僵尸
|
||||
/// 任务事故修复):此前只删队列行而遗留 tasks 表 pending 行,成为孤儿回收器误判
|
||||
/// 与重复派发涡旋的燃料。调用方拿到 ids 后应调 Database::delete_tasks_by_ids。
|
||||
pub async fn clear_queue_by_workflow(&self, workflow_name: &str) -> Result<Vec<String>> {
|
||||
let pool = self.pool.clone();
|
||||
let wf_owned = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let ids = tokio::task::spawn_blocking(move || -> Result<Vec<String>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("Queue DB pool error: {}", e))?;
|
||||
conn.execute(
|
||||
"DELETE FROM task_queue WHERE workflow_name = ?1 AND status = 'pending'",
|
||||
params![wf_owned],
|
||||
// DELETE...RETURNING 须 prepare + query_map(execute 不返回结果集)。
|
||||
let mut stmt = conn.prepare(
|
||||
"DELETE FROM task_queue WHERE workflow_name = ?1 AND status = 'pending' RETURNING task_id",
|
||||
)?;
|
||||
Ok(())
|
||||
let rows = stmt.query_map(params![wf_owned], |row| row.get::<_, String>(0))?;
|
||||
let mut ids = Vec::new();
|
||||
for r in rows {
|
||||
if let Ok(id) = r {
|
||||
ids.push(id);
|
||||
}
|
||||
}
|
||||
Ok(ids)
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
/// 判断指定 task_id 是否仍有**活**队列行(pending 或 claimed)。
|
||||
///
|
||||
/// 供派发去重与孤儿回收做跨库活性交叉校验:主库 tasks 表的 pending 行可能是
|
||||
/// 从未入队/已被清理的僵尸记录,唯有 task_queue 中存在 pending/claimed 行才证明
|
||||
/// 该任务真在途(排队中或已被节点领用)。
|
||||
pub async fn task_row_exists(&self, task_id: &str) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let id_owned = task_id.to_string();
|
||||
let exists = tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("Queue DB pool error: {}", e))?;
|
||||
let count: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM task_queue WHERE task_id = ?1 AND status IN ('pending', 'claimed')",
|
||||
params![id_owned],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
Ok(count > 0)
|
||||
})
|
||||
.await??;
|
||||
Ok(exists)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -378,6 +434,53 @@ mod tests {
|
||||
use common::models::{GridPointParams, TaskType};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// H1 修复:队列库迁移必须把历史遗留的 NULL workflow_name 行回填为 '__legacy__'
|
||||
/// (与主库 grid_points 迁移口径一致)。否则新节点领用/上报时 claim/report 无法
|
||||
/// 定向更新 '__legacy__' 网格点,旧任务结算后网格点永久卡死。
|
||||
#[tokio::test]
|
||||
async fn test_queue_migration_backfills_null_workflow_name() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let db_path = temp_dir.path().join("legacy_mig.db");
|
||||
|
||||
// 模拟旧版队列库:建表 + 插入一条 workflow_name 为 NULL 的遗留 pending 行。
|
||||
{
|
||||
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE task_queue (
|
||||
task_id TEXT PRIMARY KEY,
|
||||
payload TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
created_at DATETIME NOT NULL,
|
||||
claimed_at DATETIME,
|
||||
workflow_name TEXT,
|
||||
claimed_by_node_id TEXT,
|
||||
wave INTEGER NOT NULL DEFAULT 0
|
||||
);",
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO task_queue (task_id, payload, status, created_at, wave)
|
||||
VALUES ('legacy-1', '{}', 'pending', datetime('now'), 0)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// 打开队列库(触发迁移:补列检查 + NULL workflow_name 回填)。
|
||||
let queue = SqliteTaskQueue::new(&db_path.to_string_lossy()).await.unwrap();
|
||||
|
||||
let wf: String = {
|
||||
let conn = queue.pool.get().unwrap();
|
||||
conn.query_row(
|
||||
"SELECT workflow_name FROM task_queue WHERE task_id = 'legacy-1'",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
assert_eq!(wf, "__legacy__", "NULL workflow_name 应回填为 __legacy__");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_sqlite_task_queue_operations() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
@@ -393,18 +496,19 @@ mod tests {
|
||||
task_id,
|
||||
point_name: "t35000_g5.5_he-1_c-2_n-2_o-2".to_string(),
|
||||
params: GridPointParams {
|
||||
teff: 35000.0.into(),
|
||||
logg: 5.5.into(),
|
||||
loghe: (-1.0).into(),
|
||||
logc: (-2.0).into(),
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
teff: 35000.0.into(),
|
||||
logg: 5.5.into(),
|
||||
loghe: (-1.0).into(),
|
||||
logc: (-2.0).into(),
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
},
|
||||
task_type: TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 3600,
|
||||
workflow_name: Some("test_wf".to_string()),
|
||||
wave: 0,
|
||||
..Default::default()
|
||||
};
|
||||
queue.push_task(&task).await.unwrap();
|
||||
|
||||
@@ -437,12 +541,12 @@ mod tests {
|
||||
|
||||
let task_id = Uuid::new_v4();
|
||||
let params = GridPointParams {
|
||||
teff: 35000.0.into(),
|
||||
logg: 5.5.into(),
|
||||
loghe: (-1.0).into(),
|
||||
logc: (-2.0).into(),
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
teff: 35000.0.into(),
|
||||
logg: 5.5.into(),
|
||||
loghe: (-1.0).into(),
|
||||
logc: (-2.0).into(),
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
};
|
||||
let task = TaskSpec {
|
||||
task_id,
|
||||
@@ -453,6 +557,7 @@ mod tests {
|
||||
timeout_sec: 60,
|
||||
workflow_name: None,
|
||||
wave: 0,
|
||||
..Default::default()
|
||||
};
|
||||
queue.push_task(&task).await.unwrap();
|
||||
|
||||
@@ -486,24 +591,111 @@ mod tests {
|
||||
assert!(claim_after.is_none());
|
||||
}
|
||||
|
||||
/// requeue 竞态回归(修复):任务被 `requeue_stale_tasks` 打回 pending 后(claimed_at
|
||||
/// 清零但 claimed_by_node_id 保留),原节点(仍持归属)迟到的上报必须被放行——否则
|
||||
/// 耗时接近 timeout 的任务会因 403 被静默丢弃计算结果、重投重算(代码注释记录的竞态,
|
||||
/// 旧实现只靠 stale_sec ≥ 3×timeout 缓冲)。重新领用后归属被覆盖,原节点校验失败。
|
||||
#[tokio::test]
|
||||
async fn test_verify_task_claim_accepts_requeued_original_claimant() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let db_path = temp_dir.path().join("claim_requeue.db");
|
||||
let queue = SqliteTaskQueue::new(&db_path.to_string_lossy())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let task_id = Uuid::new_v4();
|
||||
let params = GridPointParams {
|
||||
teff: 35000.0.into(),
|
||||
logg: 5.5.into(),
|
||||
loghe: (-1.0).into(),
|
||||
logc: (-2.0).into(),
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
};
|
||||
let task = TaskSpec {
|
||||
task_id,
|
||||
point_name: params.model_name(),
|
||||
params: params.clone(),
|
||||
task_type: TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 7200,
|
||||
workflow_name: Some("wf_rq".to_string()),
|
||||
wave: 0,
|
||||
..Default::default()
|
||||
};
|
||||
queue.push_task(&task).await.unwrap();
|
||||
|
||||
// node-A 领用 → claimed。
|
||||
assert!(queue.pop_task("node-A").await.unwrap().is_some());
|
||||
|
||||
// 模拟 requeue_stale_tasks(stale=0):超时 claimed 行打回 pending(claimed_by_node_id 保留)。
|
||||
{
|
||||
let pool = queue.pool.clone();
|
||||
let tid = task_id.to_string();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let conn = pool.get().unwrap();
|
||||
conn.execute(
|
||||
"UPDATE task_queue SET status = 'pending', claimed_at = NULL WHERE task_id = ?1",
|
||||
rusqlite::params![tid],
|
||||
)
|
||||
.unwrap();
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// 原节点 node-A 迟到的上报:pending + 归属匹配 → 放行(修复后不再 403)。
|
||||
let claim_a = queue
|
||||
.verify_task_claim(&task_id.to_string(), "node-A")
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
claim_a.map(|(p, w)| (p, w)),
|
||||
Some((params.model_name(), Some("wf_rq".to_string()))),
|
||||
"requeue 后原节点仍持归属,应放行"
|
||||
);
|
||||
|
||||
// 其它节点仍被拒(归属不变)。
|
||||
let claim_b = queue
|
||||
.verify_task_claim(&task_id.to_string(), "node-B")
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(claim_b.is_none(), "非归属节点仍应被拒");
|
||||
|
||||
// 重新领用后归属覆盖:node-B claim 后,node-A 校验失败、node-B 成功。
|
||||
let popped_b = queue.pop_task("node-B").await.unwrap();
|
||||
assert!(popped_b.is_some(), "pending 任务可被 node-B 重新领用");
|
||||
let claim_a2 = queue
|
||||
.verify_task_claim(&task_id.to_string(), "node-A")
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(claim_a2.is_none(), "归属已移交 node-B,node-A 迟报应被拒");
|
||||
let claim_b2 = queue
|
||||
.verify_task_claim(&task_id.to_string(), "node-B")
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(claim_b2.is_some(), "node-B 现持归属,应放行");
|
||||
}
|
||||
|
||||
/// 辅助:构造一个最小 TaskSpec,方便下面两个排序测试。
|
||||
fn mk_task(wf: &str, wave: i32, point_name: &str) -> TaskSpec {
|
||||
TaskSpec {
|
||||
task_id: Uuid::new_v4(),
|
||||
point_name: point_name.to_string(),
|
||||
params: GridPointParams {
|
||||
teff: 35000.0.into(),
|
||||
logg: 5.5.into(),
|
||||
loghe: (-1.0).into(),
|
||||
logc: (-2.0).into(),
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
teff: 35000.0.into(),
|
||||
logg: 5.5.into(),
|
||||
loghe: (-1.0).into(),
|
||||
logc: (-2.0).into(),
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
},
|
||||
task_type: TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 60,
|
||||
workflow_name: Some(wf.to_string()),
|
||||
wave,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -585,10 +777,7 @@ mod tests {
|
||||
|
||||
// 第一次 pop:应跳过 poison(标记为 dead_letter)并返回合法任务
|
||||
let popped = queue.pop_task("n1").await.unwrap();
|
||||
assert!(
|
||||
popped.is_some(),
|
||||
"毒消息不应阻塞合法任务出队"
|
||||
);
|
||||
assert!(popped.is_some(), "毒消息不应阻塞合法任务出队");
|
||||
let task = popped.unwrap();
|
||||
assert_eq!(task.point_name, "ok_point");
|
||||
|
||||
@@ -614,6 +803,64 @@ mod tests {
|
||||
assert!(queue.pop_task("n1").await.unwrap().is_none());
|
||||
}
|
||||
|
||||
/// 旧版 MQ 在途消息兼容(二次审查 critical 修复验证):
|
||||
/// 旧消息只含 task_type=seed_step、无 tlusty_config 字段。serde default 会把
|
||||
/// strategies 填为 [cold_run, seed_step],首项 cold_run 与 task_type 不符。
|
||||
/// pop_task 出队时须调用 normalize_compat() 把首项校正为 seed_step,
|
||||
/// 否则旧热启动消息会被节点误当冷启动执行。
|
||||
#[tokio::test]
|
||||
async fn test_pop_normalizes_legacy_seed_step_message() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let db_path = temp_dir.path().join("legacy.db");
|
||||
let queue = SqliteTaskQueue::new(&db_path.to_string_lossy())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// 直接写一行旧版 payload(仅 task_type,无 tlusty_config/synspec_config 字段)。
|
||||
let legacy_task_id = uuid::Uuid::new_v4();
|
||||
let legacy_payload = serde_json::json!({
|
||||
"task_id": legacy_task_id.to_string(),
|
||||
"point_name": "t35000_g5.5_he-1_c-2_n-2_o-2",
|
||||
"params": {"teff": 35000.0, "logg": 5.5, "loghe": -1.0, "logc": -2.0, "logn": -2.0, "logo": -2.0},
|
||||
"task_type": "seed_step",
|
||||
"seed_point_name": "neighbor_seed",
|
||||
"timeout_sec": 7200,
|
||||
"workflow_name": "wf_legacy",
|
||||
"wave": 0
|
||||
}).to_string();
|
||||
{
|
||||
let pool = queue.pool.clone();
|
||||
let payload_clone = legacy_payload.clone();
|
||||
let tid = legacy_task_id.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool.get().unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO task_queue (task_id, payload, status, created_at, wave)
|
||||
VALUES (?1, ?2, 'pending', datetime('now'), 0)",
|
||||
rusqlite::params![tid, payload_clone],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let popped = queue.pop_task("n1").await.unwrap();
|
||||
let task = popped.expect("旧版消息应能正常出队");
|
||||
// normalize_compat 应把 strategies 校正为 [seed_step](与 task_type 一致)。
|
||||
assert_eq!(
|
||||
task.tlusty_config.strategies,
|
||||
vec!["seed_step".to_string()],
|
||||
"旧 seed_step 消息经 normalize_compat 后首项应为 seed_step"
|
||||
);
|
||||
assert_eq!(
|
||||
task.tlusty_config.current_strategy("cold_run"),
|
||||
"seed_step",
|
||||
"current_strategy 应为 seed_step(而非默认链的 cold_run)"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pop_wave_priority_within_workflow() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
@@ -632,4 +879,78 @@ mod tests {
|
||||
let p2 = queue.pop_task("n2").await.unwrap().unwrap();
|
||||
assert_eq!(p2.point_name, "hard");
|
||||
}
|
||||
|
||||
/// 回归测试(2026-08-02 僵尸任务事故):clear_queue_by_workflow 必须返回被删
|
||||
/// pending 行的 task_id(供调用方同步清理主库 tasks 审计行),且保留 claimed 行、
|
||||
/// 不波及他工作流。
|
||||
#[tokio::test]
|
||||
async fn test_clear_queue_by_workflow_returns_deleted_ids() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let queue = SqliteTaskQueue::new(&temp_dir.path().join("clear_ret.db").to_string_lossy())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// wf_a 两个任务:一个被领用(claimed),一个排队(pending)
|
||||
queue
|
||||
.push_task(&mk_task("wf_a", 0, "a_claimed"))
|
||||
.await
|
||||
.unwrap();
|
||||
queue
|
||||
.push_task(&mk_task("wf_a", 0, "a_pending"))
|
||||
.await
|
||||
.unwrap();
|
||||
// wf_b 一个排队任务(不应被 wf_a 的清理波及)
|
||||
queue
|
||||
.push_task(&mk_task("wf_b", 0, "b_pending"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let claimed = queue.pop_task("node-a").await.unwrap().unwrap();
|
||||
assert_eq!(claimed.point_name, "a_claimed");
|
||||
|
||||
let deleted = queue.clear_queue_by_workflow("wf_a").await.unwrap();
|
||||
assert_eq!(deleted.len(), 1, "仅 a_pending 一行被删");
|
||||
|
||||
// 被删行对应的点名为 a_pending:用剩余可 pop 任务反证(wf_a 已无 pending 行)
|
||||
let next = queue.pop_task("node-b").await.unwrap().unwrap();
|
||||
assert_eq!(
|
||||
next.workflow_name,
|
||||
Some("wf_b".to_string()),
|
||||
"wf_b 行应完好"
|
||||
);
|
||||
|
||||
// claimed 行保留:领用凭证仍在,原节点归属校验通过(#6 设计不动)
|
||||
let verify = queue
|
||||
.verify_task_claim(&claimed.task_id.to_string(), "node-a")
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(verify.is_some(), "claimed 行必须保留");
|
||||
|
||||
// 被删的 id 不再有任何活行
|
||||
for id in &deleted {
|
||||
assert!(!queue.task_row_exists(id).await.unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
/// task_row_exists 三态:pending/claimed 视为活,remove 后为死。
|
||||
#[tokio::test]
|
||||
async fn test_task_row_exists_liveness() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let queue = SqliteTaskQueue::new(&temp_dir.path().join("exists.db").to_string_lossy())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let task = mk_task("wf_e", 0, "point_e");
|
||||
let id = task.task_id.to_string();
|
||||
assert!(!queue.task_row_exists(&id).await.unwrap(), "未入队为死");
|
||||
|
||||
queue.push_task(&task).await.unwrap();
|
||||
assert!(queue.task_row_exists(&id).await.unwrap(), "pending 为活");
|
||||
|
||||
queue.pop_task("node-e").await.unwrap().unwrap();
|
||||
assert!(queue.task_row_exists(&id).await.unwrap(), "claimed 仍为活");
|
||||
|
||||
queue.remove_task(&id).await.unwrap();
|
||||
assert!(!queue.task_row_exists(&id).await.unwrap(), "删除后为死");
|
||||
}
|
||||
}
|
||||
|
||||
+153
-222
@@ -1,7 +1,8 @@
|
||||
use anyhow::Result;
|
||||
use common::result_filter::is_result_worthy;
|
||||
use common::config::SynspecConfig;
|
||||
use common::embedded::{ensure_specific_data_files, RuntimePaths};
|
||||
use common::models::{ModelSummary, TaskSpec, TaskType};
|
||||
use common::models::{ModelSummary, TaskSpec};
|
||||
use common::result_filter::is_result_worthy;
|
||||
use common::runner::ExecutionRunner;
|
||||
use reqwest::Client;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -12,6 +13,7 @@ pub async fn execute_task(
|
||||
server_url: &str,
|
||||
runtime: &RuntimePaths,
|
||||
work_dir: &Path,
|
||||
result_dir: &Path,
|
||||
task: &TaskSpec,
|
||||
shutdown: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
|
||||
) -> Result<(ModelSummary, Option<Vec<u8>>)> {
|
||||
@@ -54,75 +56,112 @@ pub async fn execute_task(
|
||||
let slot_work_dir = work_dir.join(format!("task_{}", task.task_id));
|
||||
tokio::fs::create_dir_all(&slot_work_dir).await?;
|
||||
|
||||
// 2. If seed_step, download seed .7 file from server using atomic file rename.
|
||||
// 调度入口已改为「冷启动优先」,SeedStep 仅作为冷启动失败后的救援任务出现,服务端
|
||||
// 派发前已经 find_best_seed_from_db 确认种子存在于 DB。因此下载失败(缺种子名/HTTP
|
||||
// 错误/网络异常/响应体读取失败)按硬错误处理,直接失败该任务(fail-fast),不再无种
|
||||
// 子继续运行:default_seed_chain 首阶段 seed_nc 的 ltgray="F" 依赖 fort.8,无种子时
|
||||
// Tlusty 在无初始大气下运行必然崩溃。任务失败后由服务端走既有上报路径,
|
||||
// has_seed_step_attempt 阻止重复回退,网格点保持 failed 终态。
|
||||
if task.task_type == TaskType::SeedStep {
|
||||
let seed_name = task.seed_point_name.as_deref().ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"SeedStep 任务 {} 缺少 seed_point_name,无法热启动",
|
||||
task.point_name
|
||||
)
|
||||
})?;
|
||||
let seed_url = format!("{}/api/seed/{}", server_url, seed_name);
|
||||
info!("正在从服务端下载种子大气文件: {}", seed_url);
|
||||
// 2. 种子大气获取(见 docs/task_engine_decoupling_design.md §5):
|
||||
// - TLUSTY 启用 + 策略为 seed_step:下载近邻种子 .7 作热启动种子(既有逻辑)。
|
||||
// - TLUSTY 关闭(仅 SYNSPEC 场景):需拉取目标点 .7 大气作光谱合成输入。
|
||||
// 顺序:本地 result 归档 → server 拉取。
|
||||
// 注:不查沙盒本地——TLUSTY 与 SYNSPEC 在同一任务内串行,种子获取先于 runner,
|
||||
// 全新 slot 内不可能已有目标大气;重试任务的 slot 亦全新(task_id 唯一)。
|
||||
let tlusty_enabled = task.tlusty_config.enabled;
|
||||
let current_strategy = task.tlusty_config.current_strategy("cold_run");
|
||||
let needs_seed_download = (tlusty_enabled && current_strategy == "seed_step")
|
||||
|| (!tlusty_enabled && task.synspec_config.enabled);
|
||||
|
||||
let resp = client.get(&seed_url).send().await.map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"SeedStep 任务 {} 下载种子文件 {} 失败: {}",
|
||||
task.point_name,
|
||||
seed_url,
|
||||
e
|
||||
)
|
||||
})?;
|
||||
if !resp.status().is_success() {
|
||||
anyhow::bail!(
|
||||
"SeedStep 任务 {} 下载种子文件 {} 失败: HTTP {}",
|
||||
task.point_name,
|
||||
seed_url,
|
||||
resp.status()
|
||||
);
|
||||
if needs_seed_download {
|
||||
let seed_name = if !tlusty_enabled {
|
||||
// 仅 SYNSPEC 场景:大气来自目标点本身(atmosphere_ref 或 point_name)。
|
||||
task.atmosphere_ref
|
||||
.clone()
|
||||
.unwrap_or_else(|| task.point_name.clone())
|
||||
} else {
|
||||
// SeedStep 热启动:大气来自近邻种子点。
|
||||
task.seed_point_name.clone().ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"SeedStep 任务 {} 缺少 seed_point_name,无法热启动",
|
||||
task.point_name
|
||||
)
|
||||
})?
|
||||
};
|
||||
|
||||
// 仅 SYNSPEC 场景先查本地 result 归档目录(节点此前算过同点大气,避免 server 拉取)。
|
||||
if !tlusty_enabled {
|
||||
let archived = result_dir
|
||||
.join(&task.point_name)
|
||||
.join(format!("{}.7", task.point_name));
|
||||
if archived.is_file() {
|
||||
info!(
|
||||
"SYNSPEC-only:在本地 result 归档找到大气 {},复用避免 server 拉取",
|
||||
archived.display()
|
||||
);
|
||||
seed_atmos_path = Some(archived);
|
||||
}
|
||||
}
|
||||
let bytes = resp.bytes().await.map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"SeedStep 任务 {} 读取种子文件 {} 响应体失败: {}",
|
||||
task.point_name,
|
||||
seed_url,
|
||||
e
|
||||
)
|
||||
})?;
|
||||
|
||||
let temp_seed_dir = work_dir.join(".seed_cache");
|
||||
tokio::fs::create_dir_all(&temp_seed_dir).await?;
|
||||
// LRU 上限清理:下载新种子前,删除最旧的超出 MAX_SEED_CACHE_FILES 的
|
||||
// .seed.7 文件,防止长期运行后不同种子点累积到 GB 级。同名种子会被
|
||||
// 覆盖写,真正累积的维度是「不同 seed_name」的数量。
|
||||
cleanup_seed_cache(&temp_seed_dir).await;
|
||||
let tmp_path = temp_seed_dir.join(format!(
|
||||
"{}.{}.tmp",
|
||||
seed_name,
|
||||
uuid::Uuid::new_v4().simple()
|
||||
));
|
||||
let final_seed_path = temp_seed_dir.join(format!("{}.seed.7", seed_name));
|
||||
tokio::fs::write(&tmp_path, bytes).await?;
|
||||
tokio::fs::rename(&tmp_path, &final_seed_path).await?;
|
||||
// 归档无 → 向 server 拉取。
|
||||
if seed_atmos_path.is_none() {
|
||||
let seed_url = format!("{}/api/seed/{}", server_url, seed_name);
|
||||
info!(
|
||||
"正在从服务端下载大气文件 ({}, 用途: {}): {}",
|
||||
seed_name,
|
||||
if tlusty_enabled {
|
||||
"TLUSTY 热启动种子"
|
||||
} else {
|
||||
"SYNSPEC 输入大气"
|
||||
},
|
||||
seed_url
|
||||
);
|
||||
|
||||
// 关键:复制一份种子到本任务沙盒私有副本,让 seed_atmos_path
|
||||
// 指向私有副本而非共享缓存。此后 runner 的 current_seed 全程
|
||||
// 只引用沙盒内文件,与 .seed_cache 完全解耦——这样 LRU 清理
|
||||
// (含并发竞争)即便删掉该缓存文件,也不会破坏正在使用该种子
|
||||
// 的 in-flight 任务。.seed_cache 退化为纯粹的下载去重缓存。
|
||||
let private_seed = slot_work_dir.join("seed_atmos.seed.7");
|
||||
tokio::fs::copy(&final_seed_path, &private_seed).await?;
|
||||
seed_atmos_path = Some(private_seed);
|
||||
let resp = client.get(&seed_url).send().await.map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"任务 {} 下载大气文件 {} 失败: {}",
|
||||
task.point_name,
|
||||
seed_url,
|
||||
e
|
||||
)
|
||||
})?;
|
||||
if !resp.status().is_success() {
|
||||
anyhow::bail!(
|
||||
"任务 {} 下载大气文件 {} 失败: HTTP {}",
|
||||
task.point_name,
|
||||
seed_url,
|
||||
resp.status()
|
||||
);
|
||||
}
|
||||
let bytes = resp.bytes().await.map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"任务 {} 读取大气文件 {} 响应体失败: {}",
|
||||
task.point_name,
|
||||
seed_url,
|
||||
e
|
||||
)
|
||||
})?;
|
||||
|
||||
let temp_seed_dir = work_dir.join(".seed_cache");
|
||||
tokio::fs::create_dir_all(&temp_seed_dir).await?;
|
||||
cleanup_seed_cache(&temp_seed_dir).await;
|
||||
let tmp_path = temp_seed_dir.join(format!(
|
||||
"{}.{}.tmp",
|
||||
seed_name,
|
||||
uuid::Uuid::new_v4().simple()
|
||||
));
|
||||
let final_seed_path = temp_seed_dir.join(format!("{}.seed.7", seed_name));
|
||||
tokio::fs::write(&tmp_path, bytes).await?;
|
||||
tokio::fs::rename(&tmp_path, &final_seed_path).await?;
|
||||
|
||||
let private_seed = slot_work_dir.join("seed_atmos.seed.7");
|
||||
tokio::fs::copy(&final_seed_path, &private_seed).await?;
|
||||
seed_atmos_path = Some(private_seed);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. (slot_work_dir 已在种子下载前提前创建,种子私有副本亦已落盘于沙盒内。)
|
||||
|
||||
// 反序列化工作流携带的 SYNSPEC 数值参数(波长范围等)。None → runner 用硬编码默认。
|
||||
let synspec_cfg: Option<SynspecConfig> = task
|
||||
.synspec_params
|
||||
.as_ref()
|
||||
.and_then(|v| serde_json::from_value::<SynspecConfig>(v.clone()).ok());
|
||||
|
||||
let runner = ExecutionRunner::new(runtime, slot_work_dir.clone());
|
||||
let summary = runner
|
||||
.run_model_with_timeout(
|
||||
@@ -135,7 +174,10 @@ pub async fn execute_task(
|
||||
// SeedStep→default_seed_chain),节点端不再做「缺种子回退冷启动链」的降级。
|
||||
None,
|
||||
seed_atmos_path.as_deref(),
|
||||
None,
|
||||
synspec_cfg.as_ref(),
|
||||
// 阶段独立配置开关(见 docs/task_engine_decoupling_design.md §5)。
|
||||
task.tlusty_config.enabled,
|
||||
task.synspec_config.enabled,
|
||||
task.timeout_sec,
|
||||
shutdown,
|
||||
)
|
||||
@@ -199,7 +241,8 @@ pub async fn cleanup_slot_work_dir(slot_work_dir: &Path) -> Result<()> {
|
||||
///
|
||||
/// 保留内容(详见 [`is_result_worthy`]):
|
||||
/// - 裸名:`conv.json`、`fort.8`(synspec 输入大气)、`fort.55`(synspec 控制卡)
|
||||
/// - 科学核心:`<name>.7/.spec/.cont/.iden/.log`
|
||||
/// - 科学核心:`<name>.7/.spec/.cont/.iden/.log`,以及 runner 在 SYNSPEC 覆盖前快照的
|
||||
/// TLUSTY 最终产物:`<name>.bfac`(b 因子/非 LTE 偏离因子)、`<name>.emflux`(出射谱 λ–Fλ)
|
||||
/// - 阶段快照:`<name>.<label>.5/.6/.err/.nst/.7`
|
||||
/// - 收敛诊断:`<name>.<label>_chmax*.9`(**唯一保留的 .9**)
|
||||
///
|
||||
@@ -211,11 +254,7 @@ pub async fn cleanup_slot_work_dir(slot_work_dir: &Path) -> Result<()> {
|
||||
/// `name` 为网格点权威名:取自 summary.name(runner 现用 task.point_name 作权威名),
|
||||
/// 严重失败(runner 抛 Err、无 summary)时回退到 task.point_name,确保失败任务的
|
||||
/// 排错日志也能落盘。
|
||||
pub async fn save_result_artifacts(
|
||||
result_dir: &Path,
|
||||
slot_work_dir: &Path,
|
||||
name: &str,
|
||||
) {
|
||||
pub async fn save_result_artifacts(result_dir: &Path, slot_work_dir: &Path, name: &str) {
|
||||
let src_dir = slot_work_dir.join(name);
|
||||
if !src_dir.is_dir() {
|
||||
// 模型子目录不存在(极早期失败),无可归档内容
|
||||
@@ -276,7 +315,11 @@ pub async fn save_result_artifacts(
|
||||
|
||||
let dest_path = dest_dir.join(&file_name);
|
||||
// 原子写入:先拷到 .result.tmp.<uuid> 再 rename,防止中途崩溃产生半截文件
|
||||
let tmp_path = dest_dir.join(format!("{}.result.tmp.{}", file_name, uuid::Uuid::new_v4().simple()));
|
||||
let tmp_path = dest_dir.join(format!(
|
||||
"{}.result.tmp.{}",
|
||||
file_name,
|
||||
uuid::Uuid::new_v4().simple()
|
||||
));
|
||||
match tokio::fs::copy(&path, &tmp_path).await {
|
||||
Ok(_) => {
|
||||
if let Err(e) = tokio::fs::rename(&tmp_path, &dest_path).await {
|
||||
@@ -284,9 +327,7 @@ pub async fn save_result_artifacts(
|
||||
let _ = tokio::fs::remove_file(&tmp_path).await;
|
||||
warn!(
|
||||
"归档网格点 {} 的文件 {} rename 失败: {}",
|
||||
name,
|
||||
file_name,
|
||||
e
|
||||
name, file_name, e
|
||||
);
|
||||
continue;
|
||||
}
|
||||
@@ -294,12 +335,7 @@ pub async fn save_result_artifacts(
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = tokio::fs::remove_file(&tmp_path).await;
|
||||
warn!(
|
||||
"归档网格点 {} 的文件 {} 拷贝失败: {}",
|
||||
name,
|
||||
file_name,
|
||||
e
|
||||
);
|
||||
warn!("归档网格点 {} 的文件 {} 拷贝失败: {}", name, file_name, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -314,52 +350,9 @@ pub async fn save_result_artifacts(
|
||||
);
|
||||
}
|
||||
|
||||
/// 归档目录保留的网格点(子目录)数量上限。超过则按 mtime 删除最旧的。
|
||||
/// 200 足以覆盖中等规模网格的完整归档;更大网格可经环境变量或常量调整。
|
||||
const MAX_RESULT_MODELS: usize = 200;
|
||||
|
||||
/// LRU 治理归档目录:当网格点子目录数超过 `MAX_RESULT_MODELS` 时,
|
||||
/// 按 mtime 升序删除最旧的若干个子目录,直到不超过上限。
|
||||
/// 仅统计子目录(每个对应一个网格点),忽略散落文件。错误降级为 warn,不阻断主流程。
|
||||
pub async fn cleanup_result_dir(result_dir: &Path) {
|
||||
let mut entries: Vec<(std::time::SystemTime, PathBuf)> =
|
||||
match tokio::fs::read_dir(result_dir).await {
|
||||
Ok(mut rd) => {
|
||||
let mut v = Vec::new();
|
||||
while let Ok(Some(entry)) = rd.next_entry().await {
|
||||
let path = entry.path();
|
||||
// 仅纳入子目录(网格点归档目录),跳过散落文件
|
||||
if !path.is_dir() {
|
||||
continue;
|
||||
}
|
||||
let meta = match entry.metadata().await {
|
||||
Ok(m) => m,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let mtime = meta.modified().unwrap_or(std::time::SystemTime::UNIX_EPOCH);
|
||||
v.push((mtime, path));
|
||||
}
|
||||
v
|
||||
}
|
||||
// 归档目录不存在或不可读:无操作(首次归档尚未创建)
|
||||
Err(_) => return,
|
||||
};
|
||||
|
||||
if entries.len() <= MAX_RESULT_MODELS {
|
||||
return;
|
||||
}
|
||||
|
||||
// 按 mtime 升序(最旧在前),删除超出上限的最旧子目录
|
||||
entries.sort_by_key(|(mtime, _)| *mtime);
|
||||
let to_remove = entries.len().saturating_sub(MAX_RESULT_MODELS);
|
||||
for (_, path) in entries.into_iter().take(to_remove) {
|
||||
if let Err(e) = tokio::fs::remove_dir_all(&path).await {
|
||||
warn!("LRU 清理归档目录 {} 失败: {}", path.display(), e);
|
||||
} else {
|
||||
info!("LRU 清理归档目录: {}", path.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
/// 归档目录不做数量上限治理:所有已算网格点的完整产物(.spec/.cont/.iden/各阶段
|
||||
/// 快照/日志/种子二进制等)一律永久保留,避免 LRU 淘汰导致科学产物丢失
|
||||
/// (2026-08-02 修正:撤销 1bfa240 引入的 MAX_RESULT_MODELS=200 LRU 上限)。
|
||||
|
||||
/// `.seed_cache/` 内保留的 `.seed.7` 文件上限。超过则按 mtime 删除最旧的。
|
||||
/// 典型网格内活跃种子点数量有限,8 足以覆盖常用邻域且把磁盘占用控制在 ~8 个种子文件。
|
||||
@@ -475,40 +468,42 @@ mod tests {
|
||||
|
||||
// 应被归档的白名单产物(科学核心 + 阶段快照 + 收敛诊断 + 裸名保留)
|
||||
let kept_files = [
|
||||
format!("{}.7", summary.name), // 最终大气
|
||||
format!("{}.spec", summary.name), // 合成光谱
|
||||
format!("{}.cont", summary.name), // 连续谱
|
||||
format!("{}.iden", summary.name), // 谱线证认
|
||||
format!("{}.log", summary.name), // synspec 日志
|
||||
"conv.json".to_string(), // 摘要
|
||||
"fort.8".to_string(), // synspec 输入大气(裸名保留)
|
||||
"fort.55".to_string(), // synspec 控制卡(裸名保留)
|
||||
format!("{}.nl.7", summary.name), // nl 阶段大气快照
|
||||
format!("{}.nc.7", summary.name), // nc 阶段大气快照
|
||||
format!("{}.nl.5", summary.name), // nl 阶段输入卡快照
|
||||
format!("{}.nl.6", summary.name), // nl 阶段输出日志快照
|
||||
format!("{}.nl.err", summary.name), // nl 阶段错误日志快照
|
||||
format!("{}.nl.nst", summary.name), // nl 阶段控制卡快照
|
||||
format!("{}.nc.nst", summary.name), // nc 阶段控制卡快照
|
||||
format!("{}.7", summary.name), // 最终大气
|
||||
format!("{}.spec", summary.name), // 合成光谱
|
||||
format!("{}.cont", summary.name), // 连续谱
|
||||
format!("{}.iden", summary.name), // 谱线证认
|
||||
format!("{}.log", summary.name), // synspec 日志
|
||||
format!("{}.bfac", summary.name), // TLUSTY 最终 b 因子(快照 fort.12)
|
||||
format!("{}.emflux", summary.name), // TLUSTY 最终出射谱(快照 fort.14)
|
||||
"conv.json".to_string(), // 摘要
|
||||
"fort.8".to_string(), // synspec 输入大气(裸名保留)
|
||||
"fort.55".to_string(), // synspec 控制卡(裸名保留)
|
||||
format!("{}.nl.7", summary.name), // nl 阶段大气快照
|
||||
format!("{}.nc.7", summary.name), // nc 阶段大气快照
|
||||
format!("{}.nl.5", summary.name), // nl 阶段输入卡快照
|
||||
format!("{}.nl.6", summary.name), // nl 阶段输出日志快照
|
||||
format!("{}.nl.err", summary.name), // nl 阶段错误日志快照
|
||||
format!("{}.nl.nst", summary.name), // nl 阶段控制卡快照
|
||||
format!("{}.nc.nst", summary.name), // nc 阶段控制卡快照
|
||||
format!("{}.nl_chmax0.001.9", summary.name), // nl 收敛诊断(唯一保留的 .9)
|
||||
];
|
||||
// 应被白名单过滤掉的文件:Tlusty 中间单元、裸的 runner 已清理文件、
|
||||
// 无 _chmax 的重复 .9 快照、未知后缀
|
||||
let skipped_files: [String; 14] = [
|
||||
"fort.1".to_string(), // 空单元
|
||||
"fort.13".to_string(), // Tlusty NLTE 跃迁频率网格
|
||||
"fort.18".to_string(), // Tlusty 大气结构内部表
|
||||
"fort.22".to_string(), // Tlusty 中间大气副本
|
||||
"fort.82".to_string(), // Tlusty 运行时诊断表
|
||||
"fort.95".to_string(), // Tlusty 旧模型定义副本
|
||||
"fort.84".to_string(), // NATOMS 崩溃缓存
|
||||
"residue.tmp".to_string(), // 原子写入残留
|
||||
"nst".to_string(), // 裸 nst(runner 已改名为 <name>.<label>.nst)
|
||||
"fort.9".to_string(), // 裸 fort.9(runner 已删,内容在 _chmax.9)
|
||||
"fort.12".to_string(), // 裸 fort.12(已 copy 为 .iden)
|
||||
"fort.17".to_string(), // 裸 fort.17(已 copy 为 .cont)
|
||||
format!("{}.nl.9", summary.name), // 无 _chmax 的 .9 快照(与 _chmax.9 重复)
|
||||
format!("{}.unknown", summary.name), // 未知后缀
|
||||
"fort.1".to_string(), // 空单元
|
||||
"fort.13".to_string(), // Tlusty 出射辐射场 (FREQ/FLUX/FH),未快照,丢弃
|
||||
"fort.18".to_string(), // Tlusty 大气结构内部表
|
||||
"fort.22".to_string(), // Tlusty 中间大气副本
|
||||
"fort.82".to_string(), // Tlusty 运行时诊断表
|
||||
"fort.95".to_string(), // Tlusty 旧模型定义副本
|
||||
"fort.84".to_string(), // NATOMS 崩溃缓存
|
||||
"residue.tmp".to_string(), // 原子写入残留
|
||||
"nst".to_string(), // 裸 nst(runner 已改名为 <name>.<label>.nst)
|
||||
"fort.9".to_string(), // 裸 fort.9(runner 已删,内容在 _chmax.9)
|
||||
"fort.12".to_string(), // 裸 fort.12(已 copy 为 .iden)
|
||||
"fort.17".to_string(), // 裸 fort.17(已 copy 为 .cont)
|
||||
format!("{}.nl.9", summary.name), // 无 _chmax 的 .9 快照(与 _chmax.9 重复)
|
||||
format!("{}.unknown", summary.name), // 未知后缀
|
||||
];
|
||||
for f in kept_files.iter() {
|
||||
tokio::fs::write(model_dir.join(f), "payload")
|
||||
@@ -535,19 +530,11 @@ mod tests {
|
||||
assert!(dest_dir.is_dir(), "归档目标目录应被创建");
|
||||
// 验证白名单产物都被拷贝
|
||||
for f in &kept_files {
|
||||
assert!(
|
||||
dest_dir.join(f).is_file(),
|
||||
"白名单产物 {} 应被归档",
|
||||
f
|
||||
);
|
||||
assert!(dest_dir.join(f).is_file(), "白名单产物 {} 应被归档", f);
|
||||
}
|
||||
// 验证非白名单文件未进归档
|
||||
for f in &skipped_files {
|
||||
assert!(
|
||||
!dest_dir.join(f).exists(),
|
||||
"非白名单文件 {} 应被跳过",
|
||||
f
|
||||
);
|
||||
assert!(!dest_dir.join(f).exists(), "非白名单文件 {} 应被跳过", f);
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
@@ -561,65 +548,9 @@ mod tests {
|
||||
let mut rd = tokio::fs::read_dir(&dest_dir).await.unwrap();
|
||||
while let Ok(Some(e)) = rd.next_entry().await {
|
||||
let name = e.file_name().to_string_lossy().to_string();
|
||||
assert!(
|
||||
!name.contains(".result.tmp"),
|
||||
"不应残留 tmp 文件: {}",
|
||||
name
|
||||
);
|
||||
assert!(!name.contains(".result.tmp"), "不应残留 tmp 文件: {}", name);
|
||||
}
|
||||
|
||||
let _ = tokio::fs::remove_dir_all(&root).await;
|
||||
}
|
||||
|
||||
/// 验证 LRU 治理:超过上限时按 mtime 删最旧的子目录
|
||||
#[tokio::test]
|
||||
async fn test_cleanup_result_dir() {
|
||||
let result_dir =
|
||||
std::env::temp_dir().join(format!("test_result_lru_{}", uuid::Uuid::new_v4()));
|
||||
tokio::fs::create_dir_all(&result_dir).await.unwrap();
|
||||
|
||||
// 创建 MAX+10 个子目录,按创建顺序递增 mtime(每个 sleep 制造可测的时间差)。
|
||||
// model_0000 最早创建(最旧),model_0209 最新创建。
|
||||
let total = MAX_RESULT_MODELS + 10;
|
||||
for i in 0..total {
|
||||
let dir = result_dir.join(format!("model_{:04}", i));
|
||||
tokio::fs::create_dir_all(&dir).await.unwrap();
|
||||
tokio::fs::write(dir.join("marker"), format!("{}", i))
|
||||
.await
|
||||
.unwrap();
|
||||
// 10ms 间隔足以让多数文件系统的 mtime 分辨出先后顺序
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
|
||||
cleanup_result_dir(&result_dir).await;
|
||||
|
||||
let mut remaining: Vec<String> = Vec::new();
|
||||
let mut rd = tokio::fs::read_dir(&result_dir).await.unwrap();
|
||||
while let Ok(Some(e)) = rd.next_entry().await {
|
||||
if e.path().is_dir() {
|
||||
remaining.push(e.file_name().to_string_lossy().to_string());
|
||||
}
|
||||
}
|
||||
// 清理后剩余数量应恰为上限
|
||||
assert_eq!(
|
||||
remaining.len(),
|
||||
MAX_RESULT_MODELS,
|
||||
"清理后应剩余 {} 个,实际 {} 个",
|
||||
MAX_RESULT_MODELS,
|
||||
remaining.len()
|
||||
);
|
||||
// 最旧的那批(model_0000~model_0009)应被删除,最新的 MAX 个应保留
|
||||
remaining.sort();
|
||||
assert!(
|
||||
!remaining.contains(&"model_0000".to_string()),
|
||||
"最旧的 model_0000 应被 LRU 删除"
|
||||
);
|
||||
assert!(
|
||||
remaining.contains(&format!("model_{:04}", total - 1)),
|
||||
"最新的 model_{:04} 应被保留",
|
||||
total - 1
|
||||
);
|
||||
|
||||
let _ = tokio::fs::remove_dir_all(&result_dir).await;
|
||||
}
|
||||
}
|
||||
|
||||
+211
-27
@@ -4,6 +4,46 @@ use reqwest::multipart::{Form, Part};
|
||||
use reqwest::Client;
|
||||
use tracing::{info, warn};
|
||||
|
||||
/// 失败阶段归因(见 docs/task_engine_decoupling_design.md §4.2):从 summary + 任务阶段开关
|
||||
/// 推断本次失败发生在 TLUSTY 还是 SYNSPEC,供服务端按对应策略链弹栈回退。
|
||||
///
|
||||
/// 判定顺序(顺序敏感):
|
||||
/// - `converged=false` 时:大气未收敛。若 TLUSTY 启用 → 归因 TLUSTY(含"TLUSTY 产出大气
|
||||
/// 失败导致 synspec 无输入"的级联场景);TLUSTY 关闭 → 归因 SYNSPEC(synspec-only 任务
|
||||
/// 的 converged 由 synspec_rc 决定)。
|
||||
/// - `converged=true` 但 synspec 有错误/非零 rc:归因 SYNSPEC(大气已收敛、光谱失败)。
|
||||
fn infer_failed_stage(task: &TaskSpec, summary: &ModelSummary) -> Option<String> {
|
||||
if !summary.converged {
|
||||
return if task.tlusty_config.enabled {
|
||||
Some("tlusty".to_string())
|
||||
} else {
|
||||
Some("synspec".to_string())
|
||||
};
|
||||
}
|
||||
if summary.synspec_error.is_some() || matches!(summary.synspec_rc, Some(rc) if rc != 0) {
|
||||
return Some("synspec".to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// 任务整体成败判定(修复审查 #2 后续)。
|
||||
///
|
||||
/// 旧实现 `if s.converged { Completed } else { Failed }` 只看大气收敛:当 TLUSTY 收敛但
|
||||
/// SYNSPEC 失败(rc≠0 / synspec_error)时,`converged` 仍为 true(它是大气收敛标志),
|
||||
/// 任务被误报为 Completed → 服务端 `record_task_report` 按 converged 吸收为终态:
|
||||
/// 半失败点(大气好、光谱坏)既不回退重试,错误也无从查证。
|
||||
///
|
||||
/// 新判定:**半失败 = 失败**。`converged` 仍保留 true 随上报(大气产物有效,服务端仍会
|
||||
/// 保存 .7 作种子),但 status=Failed 让服务端弹 synspec 链重试,且错误对运维可见。
|
||||
fn derive_report_status(s: &ModelSummary) -> TaskStatus {
|
||||
let synspec_failed = s.synspec_error.is_some() || matches!(s.synspec_rc, Some(rc) if rc != 0);
|
||||
if s.converged && !synspec_failed {
|
||||
TaskStatus::Completed
|
||||
} else {
|
||||
TaskStatus::Failed
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn report_result(
|
||||
client: &Client,
|
||||
server_url: &str,
|
||||
@@ -13,33 +53,40 @@ pub async fn report_result(
|
||||
) -> Result<()> {
|
||||
let report_url = format!("{}/api/task/report", server_url);
|
||||
|
||||
let (status, converged, max_relc, atmo_has_nan, elapsed_sec, err_msg, summary_json, seed_bytes) =
|
||||
match exec_res {
|
||||
Ok((s, s_bytes)) => (
|
||||
if s.converged {
|
||||
TaskStatus::Completed
|
||||
} else {
|
||||
TaskStatus::Failed
|
||||
},
|
||||
s.converged,
|
||||
s.final_max_relc,
|
||||
s.atmosphere_has_nan,
|
||||
s.elapsed_sec,
|
||||
s.note.clone(),
|
||||
serde_json::to_string(&s).unwrap_or_default(),
|
||||
s_bytes,
|
||||
),
|
||||
Err(e) => (
|
||||
TaskStatus::Failed,
|
||||
false,
|
||||
None,
|
||||
false,
|
||||
0.0,
|
||||
Some(e.clone()),
|
||||
serde_json::json!({"error": e}).to_string(),
|
||||
None,
|
||||
),
|
||||
};
|
||||
let (
|
||||
status,
|
||||
converged,
|
||||
max_relc,
|
||||
atmo_has_nan,
|
||||
elapsed_sec,
|
||||
err_msg,
|
||||
summary_json,
|
||||
seed_bytes,
|
||||
failed_stage,
|
||||
) = match exec_res {
|
||||
Ok((s, s_bytes)) => (
|
||||
derive_report_status(&s),
|
||||
s.converged,
|
||||
s.final_max_relc,
|
||||
s.atmosphere_has_nan,
|
||||
s.elapsed_sec,
|
||||
s.note.clone(),
|
||||
serde_json::to_string(&s).unwrap_or_default(),
|
||||
s_bytes,
|
||||
infer_failed_stage(task, &s),
|
||||
),
|
||||
Err(e) => (
|
||||
TaskStatus::Failed,
|
||||
false,
|
||||
None,
|
||||
false,
|
||||
0.0,
|
||||
Some(e.clone()),
|
||||
serde_json::json!({"error": e}).to_string(),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
};
|
||||
|
||||
let report = TaskReport {
|
||||
task_id: task.task_id,
|
||||
@@ -53,6 +100,7 @@ pub async fn report_result(
|
||||
elapsed_sec,
|
||||
error_message: err_msg,
|
||||
summary_json,
|
||||
failed_stage,
|
||||
};
|
||||
|
||||
let report_bytes = serde_json::to_vec(&report)?;
|
||||
@@ -93,6 +141,17 @@ pub async fn report_result(
|
||||
);
|
||||
anyhow::bail!("node token 失效 (HTTP {}),结果未上报", status);
|
||||
}
|
||||
// M3 修复:409 = 任务已被其他节点重新领用(超时重投后重领)或已由他节点结算。
|
||||
// 这是迟到的冗余结果,服务端据此弃掉本次上报——**不是凭据问题**,切勿误报
|
||||
// "token 失效"误导运维换 token;也无需重试(重试必再 409)。产物仍会归档,
|
||||
// 返回 Ok 让 worker 正常收尾(归档 + 清理沙盒 + 归还 slot)。
|
||||
if status.as_u16() == 409 {
|
||||
info!(
|
||||
"任务 {} 上报被服务端拒绝 (HTTP 409):任务已被其他节点重新领用或已结算,本次结果丢弃(不重试,产物仍归档)",
|
||||
task.task_id
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
warn!(
|
||||
"向服务端上报任务 {} 结果失败 (尝试 {}/{}): HTTP {}",
|
||||
task.task_id, attempt, max_attempts, status
|
||||
@@ -119,3 +178,128 @@ pub async fn report_result(
|
||||
task.task_id
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use common::models::GridPointParams;
|
||||
|
||||
fn mk_task(tlusty_enabled: bool) -> TaskSpec {
|
||||
TaskSpec {
|
||||
task_id: uuid::Uuid::new_v4(),
|
||||
point_name: "t20000_g5.0_he-2_c-4_n-4_o-4".to_string(),
|
||||
params: GridPointParams {
|
||||
teff: 20000.0.into(),
|
||||
logg: 5.0.into(),
|
||||
loghe: (-2.0).into(),
|
||||
logc: (-4.0).into(),
|
||||
logn: (-4.0).into(),
|
||||
logo: (-4.0).into(),
|
||||
},
|
||||
tlusty_config: common::models::EngineStageConfig {
|
||||
enabled: tlusty_enabled,
|
||||
..common::models::EngineStageConfig::default_tlusty()
|
||||
},
|
||||
..TaskSpec::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn mk_summary(
|
||||
converged: bool,
|
||||
synspec_rc: Option<i32>,
|
||||
synspec_error: Option<&str>,
|
||||
) -> ModelSummary {
|
||||
ModelSummary {
|
||||
name: "t20000_g5.0_he-2_c-4_n-4_o-4".to_string(),
|
||||
params: GridPointParams {
|
||||
teff: 20000.0.into(),
|
||||
logg: 5.0.into(),
|
||||
loghe: (-2.0).into(),
|
||||
logc: (-4.0).into(),
|
||||
logn: (-4.0).into(),
|
||||
logo: (-4.0).into(),
|
||||
},
|
||||
stages: vec![],
|
||||
converged,
|
||||
final_max_relc: None,
|
||||
final_chmax: None,
|
||||
seed: None,
|
||||
atmosphere_has_nan: false,
|
||||
synspec_rc,
|
||||
synspec_error: synspec_error.map(|s| s.to_string()),
|
||||
synspec_sec: None,
|
||||
elapsed_sec: 10.0,
|
||||
note: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 失败阶段归因(docs/task_engine_decoupling_design.md §4.2 语义):
|
||||
/// 大气未收敛 → TLUSTY(tlusty 启用时);tlusty 关闭 → SYNSPEC;
|
||||
/// 大气已收敛但 synspec 失败 → SYNSPEC;全成功 → None。
|
||||
#[test]
|
||||
fn test_infer_failed_stage() {
|
||||
// tlusty 启用 + 大气未收敛(synspec 无输入级联)→ tlusty
|
||||
assert_eq!(
|
||||
infer_failed_stage(
|
||||
&mk_task(true),
|
||||
&mk_summary(false, None, Some("No atmosphere .7 produced"))
|
||||
),
|
||||
Some("tlusty".to_string())
|
||||
);
|
||||
// tlusty 关闭 + synspec 失败(converged=false)→ synspec
|
||||
assert_eq!(
|
||||
infer_failed_stage(&mk_task(false), &mk_summary(false, Some(1), None)),
|
||||
Some("synspec".to_string())
|
||||
);
|
||||
// tlusty 启用 + 大气收敛但 synspec rc≠0 → synspec
|
||||
assert_eq!(
|
||||
infer_failed_stage(&mk_task(true), &mk_summary(true, Some(2), None)),
|
||||
Some("synspec".to_string())
|
||||
);
|
||||
// tlusty 启用 + 大气收敛 + synspec_error(如 fort.8 写入失败)→ synspec
|
||||
assert_eq!(
|
||||
infer_failed_stage(
|
||||
&mk_task(true),
|
||||
&mk_summary(true, None, Some("fort.8 copy failed"))
|
||||
),
|
||||
Some("synspec".to_string())
|
||||
);
|
||||
// 全成功 → None
|
||||
assert_eq!(
|
||||
infer_failed_stage(&mk_task(true), &mk_summary(true, Some(0), None)),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
/// 半失败判定(修复审查 #2 后续):
|
||||
/// 大气收敛 + 光谱失败 → 任务整体 Failed(否则服务端吸收为 converged 终态,无重试);
|
||||
/// 大气收敛 + 光谱成功 → Completed;大气未收敛 → Failed。
|
||||
#[test]
|
||||
fn test_derive_report_status_semi_failed() {
|
||||
// 大气收敛 + synspec rc≠0 → Failed(半失败点不再被误报成功)
|
||||
assert_eq!(
|
||||
derive_report_status(&mk_summary(true, Some(2), None)),
|
||||
TaskStatus::Failed
|
||||
);
|
||||
// 大气收敛 + synspec_error(如 fort.8 写入失败)→ Failed
|
||||
assert_eq!(
|
||||
derive_report_status(&mk_summary(true, None, Some("fort.8 copy failed"))),
|
||||
TaskStatus::Failed
|
||||
);
|
||||
// 大气收敛 + synspec 成功(rc=0)→ Completed
|
||||
assert_eq!(
|
||||
derive_report_status(&mk_summary(true, Some(0), None)),
|
||||
TaskStatus::Completed
|
||||
);
|
||||
// TLUSTY-only 模式(synspec 关闭,无 rc 无 error)+ 大气收敛 → Completed
|
||||
assert_eq!(
|
||||
derive_report_status(&mk_summary(true, None, None)),
|
||||
TaskStatus::Completed
|
||||
);
|
||||
// 大气未收敛(无论光谱状态)→ Failed
|
||||
assert_eq!(
|
||||
derive_report_status(&mk_summary(false, Some(0), None)),
|
||||
TaskStatus::Failed
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+62
-13
@@ -3,11 +3,11 @@ use crate::reporter::report_result;
|
||||
use anyhow::Result;
|
||||
use common::config::NodeConfig;
|
||||
use common::embedded::RuntimePaths;
|
||||
use common::models::{NodeHeartbeatRequest, NodeRegisterRequest, TaskSpec};
|
||||
use common::models::{NodeHeartbeatRequest, NodeHeartbeatResponse, NodeRegisterRequest, TaskSpec};
|
||||
use reqwest::Client;
|
||||
use serde_json::Value;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicI32, Ordering};
|
||||
use std::sync::atomic::{AtomicI32, AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use tokio::time::{sleep, Duration};
|
||||
use tracing::{info, warn};
|
||||
@@ -56,15 +56,23 @@ pub struct NodeWorker {
|
||||
client: Client,
|
||||
runtime: RuntimePaths,
|
||||
active_slots: Arc<AtomicI32>,
|
||||
/// 生效并发槽位上限:心跳线程据此写入,领用主循环据此读取。
|
||||
///
|
||||
/// 取自 `min(admin_max_slots, physical_max_slots)`;管理员未设配额时等于物理上限
|
||||
/// (见 docs/dynamic_cpu_slots_design.md §3.2)。用 `Arc<AtomicUsize>` 在心跳线程与
|
||||
/// 领用线程间共享,无需 Mutex:单写者(心跳线程)单读者(领用线程),原子读写即可。
|
||||
effective_max_slots: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl NodeWorker {
|
||||
pub fn new(config: NodeConfig, runtime: RuntimePaths, client: Client) -> Self {
|
||||
let physical = config.max_slots;
|
||||
Self {
|
||||
config,
|
||||
client,
|
||||
runtime,
|
||||
active_slots: Arc::new(AtomicI32::new(0)),
|
||||
effective_max_slots: Arc::new(AtomicUsize::new(physical)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -206,6 +214,8 @@ impl NodeWorker {
|
||||
let hb_url = format!("{}/api/node/heartbeat", self.config.server_url);
|
||||
let hb_node_id = self.config.node_id.clone();
|
||||
let hb_slots = self.active_slots.clone();
|
||||
let hb_effective = self.effective_max_slots.clone();
|
||||
let hb_physical = self.config.max_slots;
|
||||
let hb_interval = self.config.heartbeat_sec;
|
||||
let hb_runtime_dir = self.config.runtime_dir.clone();
|
||||
|
||||
@@ -280,6 +290,34 @@ impl NodeWorker {
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
// 解析心跳响应体,提取管理员配额(见 docs/dynamic_cpu_slots_design.md
|
||||
// §3.2):计算生效配额并更新 effective_max_slots。反序列化失败不致命
|
||||
// ——旧服务端响应体可能不含 admin_max_slots 字段,保持现有配额不变即可;
|
||||
// 但真正解析失败(服务端改了响应形状/网关篡改 body)须留日志,
|
||||
// 否则运维无法定位「配额为何没生效」(审查修复 N1)。
|
||||
if status.is_success() {
|
||||
match resp.json::<NodeHeartbeatResponse>().await {
|
||||
Ok(hb_resp) => {
|
||||
let target = match hb_resp.admin_max_slots {
|
||||
Some(admin_limit) => {
|
||||
std::cmp::min(admin_limit.max(0) as usize, hb_physical)
|
||||
}
|
||||
None => hb_physical,
|
||||
};
|
||||
let prev = hb_effective.swap(target, Ordering::AcqRel);
|
||||
if prev != target {
|
||||
info!(
|
||||
"节点 {} 并发槽位配额已调整: {} → {}(admin_max_slots={:?}, physical={})",
|
||||
hb_node_id, prev, target, hb_resp.admin_max_slots, hb_physical
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => warn!(
|
||||
"节点 {} 心跳响应解析失败 (HTTP {}),保持当前配额: {}",
|
||||
hb_node_id, status, e
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => warn!("节点 {} 心跳上报失败: {}", hb_node_id, e),
|
||||
}
|
||||
@@ -375,7 +413,7 @@ impl NodeWorker {
|
||||
}
|
||||
|
||||
let active = self.active_slots.load(Ordering::Acquire);
|
||||
if (active as usize) < self.config.max_slots {
|
||||
if (active as usize) < self.effective_max_slots.load(Ordering::Acquire) {
|
||||
match self.claim_task().await {
|
||||
Ok(ClaimOutcome::Task(task)) => {
|
||||
if was_disconnected {
|
||||
@@ -397,39 +435,50 @@ impl NodeWorker {
|
||||
slots_counter.fetch_add(1, Ordering::AcqRel);
|
||||
// RAII 守卫仅负责 drop 时 -1(含 panic 展栈),杜绝 spawned future
|
||||
// panic 导致的活动 slot 永久泄漏。
|
||||
let slot_guard = SlotGuard { counter: slots_counter.clone() };
|
||||
let slot_guard = SlotGuard {
|
||||
counter: slots_counter.clone(),
|
||||
};
|
||||
|
||||
tokio::spawn(async move {
|
||||
// 把守卫移入 future,确保任务结束(含 panic)时归还 slot。
|
||||
let _slot_guard = slot_guard;
|
||||
let slot_work_dir = work_dir.join(format!("task_{}", task.task_id));
|
||||
let res =
|
||||
execute_task(&client, &server_url, &runtime, &work_dir, &task, Some(shutdown.clone()))
|
||||
.await
|
||||
.map_err(|e| e.to_string());
|
||||
let res = execute_task(
|
||||
&client,
|
||||
&server_url,
|
||||
&runtime,
|
||||
&work_dir,
|
||||
&result_dir,
|
||||
&task,
|
||||
Some(shutdown.clone()),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| e.to_string());
|
||||
|
||||
// 在上报前克隆 summary,供上报成功后的归档使用(归档目录名取
|
||||
// summary.name,与 executor 内部 model_sub_dir 路径口径一致)。
|
||||
// 严重失败(runner 抛 Err)时 summary_opt=None,此时回退到 task.point_name
|
||||
// (二者均源自 params.model_name())作为归档目录名,确保失败任务的
|
||||
// 排错日志也能落盘而非随沙盒删除丢失。
|
||||
let summary_opt =
|
||||
res.as_ref().ok().map(|(s, _)| s.clone());
|
||||
let result_name =
|
||||
summary_opt.as_ref().map(|s| s.name.clone()).unwrap_or_else(|| task.point_name.clone());
|
||||
let summary_opt = res.as_ref().ok().map(|(s, _)| s.clone());
|
||||
let result_name = summary_opt
|
||||
.as_ref()
|
||||
.map(|s| s.name.clone())
|
||||
.unwrap_or_else(|| task.point_name.clone());
|
||||
|
||||
let report_res =
|
||||
report_result(&client, &server_url, &node_id, &task, res).await;
|
||||
if report_res.is_ok() {
|
||||
// 上报成功后、清理沙盒前,先把完整产物归档到持久目录,
|
||||
// 避免随沙盒删除丢失(.spec/.cont/.iden/各阶段快照/日志等)。
|
||||
// 归档不做 LRU 淘汰:所有已算网格点产物一律永久保留
|
||||
// (2026-08-02 修正:撤销 1bfa240 引入的 200 目录上限)。
|
||||
crate::executor::save_result_artifacts(
|
||||
&result_dir,
|
||||
&slot_work_dir,
|
||||
&result_name,
|
||||
)
|
||||
.await;
|
||||
crate::executor::cleanup_result_dir(&result_dir).await;
|
||||
if let Err(e) =
|
||||
crate::executor::cleanup_slot_work_dir(&slot_work_dir).await
|
||||
{
|
||||
|
||||
@@ -191,3 +191,103 @@ pub async fn enable_node(
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// POST /api/admin/nodes/:node_id/quota — 设置节点并发槽位配额(动态调整 CPU 核数)。
|
||||
///
|
||||
/// 见 `docs/dynamic_cpu_slots_design.md`:管理员通过本接口下发配额上限,服务端写入
|
||||
/// `nodes.admin_max_slots`,Worker 在下一次心跳响应里取回并据此调整本地领用并发数。
|
||||
/// 不中断正在运行的任务,实现平滑降级。
|
||||
///
|
||||
/// 请求体:`{"admin_max_slots": 4}`(`null` 表示解除限制,恢复物理 `max_slots`;`0` 表示
|
||||
/// 暂停接新任务)。配额合法后立即落库,Worker 下次心跳(`heartbeat_sec` 秒内)即生效。
|
||||
#[derive(serde::Deserialize)]
|
||||
pub struct SetNodeQuotaRequest {
|
||||
/// 双层 Option 区分「字段缺失」与「显式 null」:
|
||||
/// - 外层 `None`(JSON 字段缺失)→ 400(防止畸形输入被静默当作清除配额)
|
||||
/// - 外层 `Some(None)`(JSON `null`)→ 清除限制
|
||||
/// - 外层 `Some(Some(n))`(JSON 数字)→ 设为 n(n >= 0)
|
||||
///
|
||||
/// 注:serde 对嵌套 `Option<Option<T>>` 默认把 `null` 也映射为外层 `None`
|
||||
/// (与缺失字段不可区分),必须用 `deserialize_with` 显式区分(审查修复 M1)。
|
||||
#[serde(default, deserialize_with = "deserialize_quota")]
|
||||
pub admin_max_slots: Option<Option<i32>>,
|
||||
}
|
||||
|
||||
/// 自定义反序列化:字段**存在**时把 `null` 映射为 `Some(None)`(清除),数字映射为
|
||||
/// `Some(Some(n))`(设值)。字段**缺失**时 serde 走 `#[serde(default)]`(外层 None),
|
||||
/// 与显式 null 语义区分开。
|
||||
fn deserialize_quota<'de, D>(deserializer: D) -> Result<Option<Option<i32>>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let inner: Option<i32> = serde::Deserialize::deserialize(deserializer)?;
|
||||
Ok(Some(inner))
|
||||
}
|
||||
|
||||
pub async fn set_node_quota(
|
||||
State(state): State<AppState>,
|
||||
AxumPath(node_id): AxumPath<String>,
|
||||
Json(req): Json<SetNodeQuotaRequest>,
|
||||
) -> Result<impl IntoResponse, crate::api::AppError> {
|
||||
if !is_valid_node_id(&node_id) {
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"非法的节点 ID 参数".to_string(),
|
||||
));
|
||||
}
|
||||
// 字段缺失(`{}`、字段名拼错等)是畸形输入而非「清除」:显式拒绝,避免破坏性默认值。
|
||||
// 审查修复(M1):旧实现 `Option<i32>` 在缺字段时 serde 默认 None,会把任何畸形请求
|
||||
// 静默当成「清除配额」执行,客户端只见 200「已清除限制」。
|
||||
let admin_max_slots = match req.admin_max_slots {
|
||||
None => {
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"请求体缺少 admin_max_slots 字段(传 null 清除限制,传非负整数设置配额)"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
Some(v) => v,
|
||||
};
|
||||
// 配额非负校验:0 合法(暂停接新任务),仅拒绝负数。
|
||||
if let Some(n) = admin_max_slots {
|
||||
if n < 0 {
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"admin_max_slots 不能为负数(0 = 暂停接新任务,null = 清除限制)".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
// 节点须存在(防幽灵 node_id 写入空更新)
|
||||
match state.db.get_node_exists(&node_id).await {
|
||||
Ok(false) => {
|
||||
return Err(crate::api::AppError::NotFound(format!(
|
||||
"节点 '{}' 不存在",
|
||||
node_id
|
||||
)));
|
||||
}
|
||||
Ok(true) => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
match state
|
||||
.db
|
||||
.set_node_admin_max_slots(&node_id, admin_max_slots)
|
||||
.await
|
||||
{
|
||||
Ok(true) => {
|
||||
let desc = match admin_max_slots {
|
||||
Some(n) => format!("配额上限设为 {}(下一次心跳后生效)", n),
|
||||
None => "已清除配额限制(恢复物理 max_slots,下一次心跳后生效)".to_string(),
|
||||
};
|
||||
info!("管理员已调整节点 {} 的并发槽位配额:{}", node_id, desc);
|
||||
Ok((
|
||||
StatusCode::OK,
|
||||
Json(json!({
|
||||
"success": true,
|
||||
"message": format!("节点 '{}' {}", node_id, desc),
|
||||
})),
|
||||
))
|
||||
}
|
||||
Ok(false) => Err(crate::api::AppError::NotFound(format!(
|
||||
"节点 '{}' 不存在",
|
||||
node_id
|
||||
))),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,10 +127,7 @@ pub async fn check_auth() -> impl IntoResponse {
|
||||
/// 一致的 `extract_token_from_headers`,同时支持 Authorization: Bearer 与 X-API-Key、
|
||||
/// 拒绝空值)并从 `admin_sessions` 中移除,使该 token 在服务端立即失效(而非等 24h 过期)。
|
||||
/// 这样即便 token 已被窃取,登出操作也能立即阻断重放。
|
||||
pub async fn logout(
|
||||
State(state): State<AppState>,
|
||||
headers: HeaderMap,
|
||||
) -> impl IntoResponse {
|
||||
pub async fn logout(State(state): State<AppState>, headers: HeaderMap) -> impl IntoResponse {
|
||||
// 与 auth_middleware 口径一致地提取 token(支持 X-API-Key、拒绝空值)。
|
||||
let token = crate::api::extract_token_from_headers(&headers);
|
||||
|
||||
|
||||
@@ -181,7 +181,10 @@ pub async fn heartbeat_node(
|
||||
));
|
||||
}
|
||||
match state.db.heartbeat_node(&req).await {
|
||||
Ok(_) => Ok(Json(json!({"status": "ok"}))),
|
||||
Ok(admin_max_slots) => Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"admin_max_slots": admin_max_slots,
|
||||
}))),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
+109
-32
@@ -40,9 +40,23 @@ pub async fn claim_task(
|
||||
Ok(Some(task)) => {
|
||||
// 多工作流分区:mark_grid_point_running 须带 workflow_name,避免按 name 全局更新
|
||||
// 误改其他工作流的同名点。TaskSpec.workflow_name 在调度时已绑定。
|
||||
let wf = task.workflow_name.as_deref().unwrap_or("");
|
||||
if let Err(e) = state.db.mark_grid_point_running(&task.point_name, wf).await {
|
||||
warn!("领用任务 {} 后同步变更为 running 状态遇到异常: {}. 后置 stale 定时自取检索引索将介入修复维护", task.task_id, e);
|
||||
// 终态守卫(2026-08-02 涡旋事故修复):仅 pending/queued → running;返回
|
||||
// false 表示点已在 running 或终态(迟到/重复领用),记录后照常下发任务
|
||||
// (队列凭证有效,计算结果仍会被 record_task_report 的终态守卫正确吸收)。
|
||||
// 旧版在途任务 payload 无 workflow_name(None)→ 归一到主库迁移回填的
|
||||
// '__legacy__' 标记,使 mark_grid_point_running 能命中 legacy 网格点(H1 修复)。
|
||||
let wf = crate::db::normalize_workflow_name(task.workflow_name.as_deref());
|
||||
match state.db.mark_grid_point_running(&task.point_name, &wf).await {
|
||||
Ok(false) => {
|
||||
info!(
|
||||
"领用任务 {}(网格点 {})时点已非 pending/queued 态,跳过 running 标记(迟到/重复领用)",
|
||||
task.task_id, task.point_name
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("领用任务 {} 后同步变更为 running 状态遇到异常: {}. 后置 stale 定时自取检索引索将介入修复维护", task.task_id, e);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok((StatusCode::OK, Json(json!({"status": "ok", "task": task}))))
|
||||
}
|
||||
@@ -127,6 +141,11 @@ pub async fn report_task(
|
||||
// 2. 上报的 point_name 必须与该 task 绑定的 point_name 一致(防跨点上报)。
|
||||
// 3. 忽略 body 里声称的 node_id,统一以鉴权 node_id 写库(修复审计归因断裂)。
|
||||
// 4. 取 task 绑定的 workflow_name,用于定向更新该工作流的 grid_points(多工作流分区)。
|
||||
// 幂等吸收:verify_task_claim 落空有两种可能——(a) 首轮上报已完成且 MQ 领用行已被
|
||||
// remove_task 清除,但响应在链路上丢失,节点重试;(b) 真·伪造。凭 tasks 表已结算记录
|
||||
// 区分:若该任务确已由本节点结算(find_settled_task_claim),按幂等重放处理(补写种子、
|
||||
// 返回 200),避免节点误判「token 失效」而中止并导致种子/结算丢失。
|
||||
let mut idempotent = false;
|
||||
let (claimed_point, claimed_workflow) = match state
|
||||
.queue
|
||||
.verify_task_claim(&report.task_id.to_string(), &auth_node.node_id)
|
||||
@@ -134,13 +153,37 @@ pub async fn report_task(
|
||||
{
|
||||
Ok(Some((p, w))) => (p, w),
|
||||
Ok(None) => {
|
||||
warn!(
|
||||
"任务归属校验失败:node={} 上报 task_id={} 但未领用或已被清理",
|
||||
auth_node.node_id, report.task_id
|
||||
);
|
||||
return Err(crate::api::AppError::Forbidden(
|
||||
"任务未由本节点领用或已上报过".to_string(),
|
||||
));
|
||||
match state
|
||||
.db
|
||||
.find_settled_task_claim(&report.task_id.to_string(), &auth_node.node_id)
|
||||
.await
|
||||
{
|
||||
Ok(Some((p, w))) => {
|
||||
info!(
|
||||
"任务 {} 已由本节点结算(幂等重放上报),吸收处理并补写种子",
|
||||
report.task_id
|
||||
);
|
||||
idempotent = true;
|
||||
(p, w)
|
||||
}
|
||||
Ok(None) => {
|
||||
// M3 修复:归属校验落空 ≠ token 失效。verify 落空且非本节点已结算,
|
||||
// 通常是因为任务被 requeue_stale_tasks 重投后被**其他节点**重新领用
|
||||
// (或已由他节点结算)——迟到冗余结果应被识别为"结果被弃",而非引导
|
||||
// 运维去换 token。返回 409 Conflict 与 401/403(真·鉴权失败)区分开。
|
||||
warn!(
|
||||
"任务归属校验失败:node={} 上报 task_id={} 未由本节点领用/结算(可能已被其他节点重新领用或已结算)",
|
||||
auth_node.node_id, report.task_id
|
||||
);
|
||||
return Err(crate::api::AppError::Conflict(
|
||||
"任务未由本节点领用:可能已被其他节点重新领用或已结算".to_string(),
|
||||
));
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("校验任务已结算状态数据库异常: {}", e);
|
||||
return Err(crate::api::AppError::Internal(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("校验任务归属数据库异常: {}", e);
|
||||
@@ -159,7 +202,9 @@ pub async fn report_task(
|
||||
// 统一以鉴权 node_id 覆盖 body 里的 node_id,保证归因可信
|
||||
report.node_id = auth_node.node_id.clone();
|
||||
// workflow_name 以领用记录为准(claim 时从 TaskSpec 落库),body 无权声称。
|
||||
let workflow_name = claimed_workflow.unwrap_or_default();
|
||||
// 旧版任务(payload 无 workflow_name)归一到 '__legacy__',定向更新主库 legacy 网格点
|
||||
// (H1 修复:否则 record_task_report 按 workflow_name='' 更新 0 行,点永久卡死)。
|
||||
let workflow_name = crate::db::normalize_workflow_name(claimed_workflow.as_deref());
|
||||
|
||||
let name = report.point_name.clone();
|
||||
|
||||
@@ -187,19 +232,31 @@ pub async fn report_task(
|
||||
};
|
||||
|
||||
// Record in DB(带 workflow_name 定向更新该工作流的 grid_points)
|
||||
if let Err(e) = state.db.record_task_report(&report, &workflow_name).await {
|
||||
// DB 错误细节进日志,对客户端只返回通用消息(避免泄露表结构/内部错误给未授权方)
|
||||
tracing::error!("记录网格点 {} 任务结果到数据库失败: {}", name, e);
|
||||
return Err(crate::api::AppError::Internal(e));
|
||||
}
|
||||
// state_changed:网格点状态是否实际迁移。被终态守卫吸收的重复报告返回 false,
|
||||
// 下方种子回退据此跳过,杜绝重复失败报告触发多余 seed_step(2026-08-02 事故修复)。
|
||||
// 幂等重放(idempotent=true):首轮已结算,跳过结算与队列清理(终态守卫已吸收)。
|
||||
let state_changed = if idempotent {
|
||||
false
|
||||
} else {
|
||||
match state.db.record_task_report(&report, &workflow_name).await {
|
||||
Ok(changed) => changed,
|
||||
Err(e) => {
|
||||
// DB 错误细节进日志,对客户端只返回通用消息(避免泄露表结构/内部错误给未授权方)
|
||||
tracing::error!("记录网格点 {} 任务结果到数据库失败: {}", name, e);
|
||||
return Err(crate::api::AppError::Internal(e));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Clean up task from task_queue table to prevent queue DB bloat
|
||||
if let Err(e) = state.queue.remove_task(&report.task_id.to_string()).await {
|
||||
tracing::warn!(
|
||||
"从任务队列中清理已上报任务记录 {} 失败: {}",
|
||||
report.task_id,
|
||||
e
|
||||
);
|
||||
if !idempotent {
|
||||
if let Err(e) = state.queue.remove_task(&report.task_id.to_string()).await {
|
||||
tracing::warn!(
|
||||
"从任务队列中清理已上报任务记录 {} 失败: {}",
|
||||
report.task_id,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 采用原子写入模式保持 conv.json 与核心二进制数据完整落地后才揭晓真实文件名
|
||||
@@ -240,19 +297,29 @@ pub async fn report_task(
|
||||
}
|
||||
}
|
||||
|
||||
if !report.converged
|
||||
|| report.atmosphere_has_nan
|
||||
|| report.status == TaskStatus::Failed
|
||||
|| report.status == TaskStatus::Timeout
|
||||
if state_changed
|
||||
&& (!report.converged
|
||||
|| report.atmosphere_has_nan
|
||||
|| report.status == TaskStatus::Failed
|
||||
|| report.status == TaskStatus::Timeout)
|
||||
{
|
||||
// Task did not succeed -> check if seed_step fallback should be triggered
|
||||
info!("网格点 {} 计算未成功完成,检查种子回退机制...", name);
|
||||
// Task did not succeed -> check if strategy chain fallback should be triggered.
|
||||
// 仅在网格点状态实际迁移时才检查回退:被终态守卫吸收的迟到/重复失败报告
|
||||
// (state_changed=false)不再触发,避免重复派发(2026-08-02 涡旋事故修复)。
|
||||
// 失败阶段归因(docs/task_engine_decoupling_design.md §4.2):节点据 summary 推断
|
||||
// 失败发生在 TLUSTY 还是 SYNSPEC,服务端弹对应策略链;旧节点不携带该字段 →
|
||||
// 兜底 "tlusty"(行为与旧版一致)。
|
||||
let failed_stage = report.failed_stage.as_deref().unwrap_or("tlusty");
|
||||
info!(
|
||||
"网格点 {} 计算未成功完成(失败阶段: {}),检查策略链回退机制...",
|
||||
name, failed_stage
|
||||
);
|
||||
if let Err(e) = state
|
||||
.scheduler
|
||||
.trigger_seed_step_fallback(¶ms, &name, &workflow_name)
|
||||
.trigger_strategy_fallback(¶ms, &name, &workflow_name, failed_stage)
|
||||
.await
|
||||
{
|
||||
warn!("网格点 {} 触发种子回退机制失败: {}", name, e);
|
||||
warn!("网格点 {} 触发策略链回退机制失败: {}", name, e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -402,9 +469,17 @@ pub async fn import_seed(
|
||||
// 1. 幂等写入 grid_points(ON CONFLICT DO NOTHING):无需事先 start 工作流。
|
||||
// 用权威 name(旧 conv.json 的源精度真名),而非从 params 重推——导入路径的
|
||||
// params 来自旧 JSON(无源文本,model_name() 会失真)。
|
||||
// wave 修复(2026-08-04):此前硬编码 wave=0,导入点全部被错误归入第一波,
|
||||
// 前端难度波次推进显示错误。现按 initialize_grid 同口径计算波次(该工作流内
|
||||
// cno_sum 严格小于本点的去重值个数),新导入点落库即归入正确波次。
|
||||
let wave = state
|
||||
.db
|
||||
.compute_wave_for_cno_sum(&workflow_name, params.cno_sum())
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
if let Err(e) = state
|
||||
.db
|
||||
.upsert_grid_point_named(&name, ¶ms, 0, &workflow_name)
|
||||
.upsert_grid_point_named(&name, ¶ms, wave, &workflow_name)
|
||||
.await
|
||||
{
|
||||
tracing::error!("历史种子导入:upsert grid_points {} 失败: {}", name, e);
|
||||
@@ -469,7 +544,9 @@ pub async fn import_seed(
|
||||
|
||||
info!(
|
||||
"历史种子导入完成:网格点 {} (workflow={}, converged={}, success_method={}, max_relc={:?})",
|
||||
name, workflow_name, converged,
|
||||
name,
|
||||
workflow_name,
|
||||
converged,
|
||||
success_method.as_deref().unwrap_or("(default cold_run)"),
|
||||
max_relc
|
||||
);
|
||||
|
||||
@@ -96,11 +96,36 @@ pub async fn save_workflow(
|
||||
}
|
||||
|
||||
// Validate YAML config string(用源精度解析,校验 + 保留 grid 轴书写小数位)
|
||||
if let Err(e) = GridConfig::from_yaml_str(&req.config_yaml) {
|
||||
return Err(crate::api::AppError::BadRequest(format!(
|
||||
"无效的 YAML 配置: {}",
|
||||
e
|
||||
)));
|
||||
let cfg = match GridConfig::from_yaml_str(&req.config_yaml) {
|
||||
Ok(cfg) => cfg,
|
||||
Err(e) => {
|
||||
return Err(crate::api::AppError::BadRequest(format!(
|
||||
"无效的 YAML 配置: {}",
|
||||
e
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
// 阶段配置合法性校验(见 docs/task_engine_decoupling_design.md §3,修复审查 #4/#5):
|
||||
// - 启用的阶段必须配置 ≥1 个策略(空链 → 调度无顺位可派,任务必失败);
|
||||
// - 至少一个阶段启用(双关 → 无任何计算可执行,任务必失败)。
|
||||
// 校验基于 resolve_* 的最终生效配置(含旧字段推断),口径与调度器一致。
|
||||
let tlusty_cfg = cfg.resolve_tlusty_config();
|
||||
let synspec_cfg = cfg.resolve_synspec_config();
|
||||
if !tlusty_cfg.enabled && !synspec_cfg.enabled {
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"TLUSTY 与 SYNSPEC 阶段均被禁用:至少应启用一个计算阶段".to_string(),
|
||||
));
|
||||
}
|
||||
if tlusty_cfg.enabled && tlusty_cfg.strategies.is_empty() {
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"TLUSTY 阶段已启用但策略链为空:至少需要 1 个策略(如 cold_run)".to_string(),
|
||||
));
|
||||
}
|
||||
if synspec_cfg.enabled && synspec_cfg.strategies.is_empty() {
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"SYNSPEC 阶段已启用但策略链为空:至少需要 1 个策略(如 standard)".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// 检查被编辑的工作流是否正处于激活运行中
|
||||
@@ -304,7 +329,13 @@ pub struct ProgressQuery {
|
||||
///
|
||||
/// - `series`:窗口内的计数快照(超 300 点自动降采样,首末点保留);
|
||||
/// - `now`:服务端当前 UTC 时刻(同 ts 格式),供前端把曲线右缘锚定为“现在”、横轴按真实时间铺开;
|
||||
/// - `rate_per_hour`:窗口首末 converged 增量 ÷ 时长(快照 <2 条或时长 ≤0 为 null);
|
||||
/// - `rate_per_hour`:**最近 2 小时**的平均收敛速率(点/小时)。取末快照往前 2h 子窗口的
|
||||
/// 首末 converged 增量 ÷ 实际跨度;子窗口不足 2 个快照时回退整窗。只看近 2h 是因为速率
|
||||
/// 首要用于 ETA,应由当前吞吐驱动,而非被整窗(最长 24h)的早期快慢/停滞抹平;
|
||||
/// - `done_rate_per_hour`:终态完成(converged+failed)的同口径近 2h 平均速率,即队列实际
|
||||
/// 清空速率,供前端 ETA 使用(剩余点数已同时扣除 converged 与 failed,口径须一致);
|
||||
/// - `rate_span_hours`:速率统计的实际时间跨度(近 2h 子窗口首末间隔,≤2h;回退整窗时为
|
||||
/// 整窗跨度)。前端须按实际跨度展示,避免把短数据跨度误读为固定 2h 或整窗平均;
|
||||
/// - `stalled_minutes`:终态数(converged+failed)最后一次增长到窗口末端的分钟数
|
||||
/// (用于"进度停滞"预警;快照 <2 条为 null)。
|
||||
pub async fn get_workflow_progress(
|
||||
@@ -346,21 +377,72 @@ pub async fn get_workflow_progress(
|
||||
// SQLite datetime('now') 为 UTC 'YYYY-MM-DD HH:MM:SS'
|
||||
let parse_ts = |ts: &str| chrono::NaiveDateTime::parse_from_str(ts, "%Y-%m-%d %H:%M:%S").ok();
|
||||
|
||||
let rate_per_hour: Option<f64> = match (series.first(), series.last()) {
|
||||
(Some(first), Some(last)) if series.len() >= 2 => {
|
||||
match (parse_ts(&first.ts), parse_ts(&last.ts)) {
|
||||
(Some(t0), Some(t1)) => {
|
||||
let dh = (t1 - t0).num_seconds() as f64 / 3600.0;
|
||||
if dh > 0.0 {
|
||||
Some((last.converged - first.converged) as f64 / dh)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
// 经验速率 = 最近 2 小时的平均速率(点/小时)。
|
||||
// 为什么只看近 2h 而非整窗:速率的首要用途是 ETA(“剩余点还要多久”),应由**当前
|
||||
// 吞吐**驱动。整窗(最长 24h)平均会把早期的快/慢、中途停滞一并抹平,对“接下来”的
|
||||
// 预测失真;近 2h 平均更贴近当下的真实处理速度。
|
||||
const RATE_WINDOW_SEC: i64 = 2 * 3600;
|
||||
|
||||
// 近 2h 子窗口起点下标:升序序列中首个「距末快照 ≤2h」的快照。用末快照(而非 now)
|
||||
// 锚定,使停滞期不被算进窗口尾部。子窗口不足 2 个快照(极慢/刚启动/长期停滞)时
|
||||
// 回退整窗,尽量仍给出数值。
|
||||
let recent_start: usize = if series.len() >= 2 {
|
||||
let s = match parse_ts(&series[series.len() - 1].ts) {
|
||||
Some(t_last) => series
|
||||
.iter()
|
||||
.position(|p| {
|
||||
parse_ts(&p.ts)
|
||||
.map(|t| (t_last - t).num_seconds() <= RATE_WINDOW_SEC)
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(0),
|
||||
None => 0,
|
||||
};
|
||||
if series.len() - s < 2 {
|
||||
0
|
||||
} else {
|
||||
s
|
||||
}
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
// 子窗口首末增量 ÷ 实际跨度 = 时段平均速率(弦斜率)。跨度即 rate_span_hours(≤2h,
|
||||
// 回退整窗时为整窗跨度),前端据此展示“近 X 小时平均”,不暗示固定 2h 或整窗。
|
||||
let avg_per_hour = |count: fn(&crate::db::ProgressPoint) -> i64| -> Option<f64> {
|
||||
let sub = &series[recent_start..];
|
||||
if sub.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
let first = sub.first()?;
|
||||
let last = sub.last()?;
|
||||
let t0 = parse_ts(&first.ts)?;
|
||||
let t1 = parse_ts(&last.ts)?;
|
||||
let dh = (t1 - t0).num_seconds() as f64 / 3600.0;
|
||||
if dh > 0.0 {
|
||||
Some((count(last) - count(first)) as f64 / dh)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
let rate_per_hour = avg_per_hour(|p| p.converged);
|
||||
let done_rate_per_hour = avg_per_hour(|p| p.converged + p.failed);
|
||||
|
||||
// 速率统计的实际时间跨度(近 2h 子窗口首末间隔,≤2h;回退整窗时为整窗跨度)。
|
||||
let rate_span_hours: Option<f64> = {
|
||||
let sub = &series[recent_start..];
|
||||
match (sub.first(), sub.last()) {
|
||||
(Some(first), Some(last)) if sub.len() >= 2 => {
|
||||
match (parse_ts(&first.ts), parse_ts(&last.ts)) {
|
||||
(Some(t0), Some(t1)) => {
|
||||
Some(((t1 - t0).num_seconds() as f64 / 3600.0).max(0.0))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let stalled_minutes: Option<f64> = if series.len() >= 2 {
|
||||
@@ -396,10 +478,12 @@ pub async fn get_workflow_progress(
|
||||
message: "成功获取进度时间序列".to_string(),
|
||||
data: Some(serde_json::json!({
|
||||
"hours": hours,
|
||||
"now": now,
|
||||
"series": series,
|
||||
"rate_per_hour": rate_per_hour,
|
||||
"stalled_minutes": stalled_minutes,
|
||||
"now": now,
|
||||
"series": series,
|
||||
"rate_per_hour": rate_per_hour,
|
||||
"done_rate_per_hour": done_rate_per_hour,
|
||||
"rate_span_hours": rate_span_hours,
|
||||
"stalled_minutes": stalled_minutes,
|
||||
})),
|
||||
}),
|
||||
))
|
||||
@@ -481,6 +565,13 @@ pub async fn get_workflow_points(
|
||||
"teff" => format!("gp.teff {dir}, gp.wave ASC, gp.cno_sum ASC"),
|
||||
"max_relc" => format!("t.max_relc IS NULL ASC, t.max_relc {dir}, gp.wave ASC"),
|
||||
"attempts" => format!("gp.attempt_count {dir}, gp.wave ASC, gp.cno_sum ASC"),
|
||||
// 耗时取最近一次尝试的真实墙钟(与列表展示同口径 COALESCE),NULL(从未派发)靠后。
|
||||
"elapsed" => {
|
||||
format!(
|
||||
"COALESCE(t.elapsed_sec, gp.last_elapsed_sec) IS NULL ASC, \
|
||||
COALESCE(t.elapsed_sec, gp.last_elapsed_sec) {dir}, gp.wave ASC"
|
||||
)
|
||||
}
|
||||
"last_completed_at" => {
|
||||
format!("t.completed_at IS NULL ASC, t.completed_at {dir}, gp.wave ASC")
|
||||
}
|
||||
@@ -594,9 +685,26 @@ pub async fn stop_workflow(
|
||||
match state.db.update_workflow_status(&name, "paused").await {
|
||||
Ok(_) => {
|
||||
// 多工作流分区:清理与重置都限定在本工作流内,避免误伤其他并发运行的工作流。
|
||||
// - clear_queue_by_workflow:只删本工作流的排队任务。
|
||||
// - clear_queue_by_workflow:只删本工作流的排队任务,返回被删 task_id。
|
||||
// - delete_tasks_by_ids:同步清理主库 tasks 表对应 pending 行(2026-08-02
|
||||
// 涡旋事故修复:此前只删队列行遗留僵尸 pending 行,成为重复派发燃料)。
|
||||
// - reset_queued_grid_points_to_pending(&name):只把本工作流的 queued 点打回 pending。
|
||||
let _ = state.queue.clear_queue_by_workflow(&name).await;
|
||||
match state.queue.clear_queue_by_workflow(&name).await {
|
||||
Ok(cleared_ids) if !cleared_ids.is_empty() => {
|
||||
if let Err(e) = state.db.delete_tasks_by_ids(&cleared_ids).await {
|
||||
tracing::warn!(
|
||||
"暂停工作流 {} 时同步清理 {} 条被删队列任务的 tasks 历史行失败: {}",
|
||||
name,
|
||||
cleared_ids.len(),
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("暂停工作流 {} 时清理排队任务失败: {}", name, e);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
let _ = state.db.reset_queued_grid_points_to_pending(&name).await;
|
||||
Ok((
|
||||
StatusCode::OK,
|
||||
|
||||
+1767
-77
File diff suppressed because it is too large
Load Diff
+37
-25
@@ -106,22 +106,23 @@ async fn main() -> Result<()> {
|
||||
);
|
||||
for (wf_name, wf_yaml) in &stuck {
|
||||
match GridConfig::from_yaml_str(wf_yaml) {
|
||||
Ok(cfg) => {
|
||||
match scheduler.initialize_grid(&cfg, wf_name).await {
|
||||
Ok(_) => {
|
||||
let _ = db.update_workflow_status(wf_name, "running").await;
|
||||
info!("启动恢复:工作流 {} 已完成重新初始化并切回 running", wf_name);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"启动恢复:工作流 {} 重新初始化失败,回退为 idle: {}",
|
||||
wf_name,
|
||||
e
|
||||
);
|
||||
let _ = db.update_workflow_status(wf_name, "idle").await;
|
||||
}
|
||||
Ok(cfg) => match scheduler.initialize_grid(&cfg, wf_name).await {
|
||||
Ok(_) => {
|
||||
let _ = db.update_workflow_status(wf_name, "running").await;
|
||||
info!(
|
||||
"启动恢复:工作流 {} 已完成重新初始化并切回 running",
|
||||
wf_name
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"启动恢复:工作流 {} 重新初始化失败,回退为 idle: {}",
|
||||
wf_name,
|
||||
e
|
||||
);
|
||||
let _ = db.update_workflow_status(wf_name, "idle").await;
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"启动恢复:工作流 {} 的 YAML 配置解析失败,回退为 idle: {}",
|
||||
@@ -188,8 +189,11 @@ async fn main() -> Result<()> {
|
||||
let mut by_wf: std::collections::HashMap<String, Vec<String>> =
|
||||
std::collections::HashMap::new();
|
||||
for (point, wf) in &requeued {
|
||||
// 旧版任务(payload 无 workflow_name)归一到 '__legacy__',
|
||||
// 使 reset_specific_grid_points_to_pending 命中 legacy 网格点
|
||||
// (H1 修复:否则按 '' 更新 0 行,重投后的 legacy 点永远重置不回 pending)。
|
||||
by_wf
|
||||
.entry(wf.clone().unwrap_or_default())
|
||||
.entry(server::db::normalize_workflow_name(wf.as_deref()))
|
||||
.or_default()
|
||||
.push(point.clone());
|
||||
}
|
||||
@@ -218,17 +222,23 @@ async fn main() -> Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
// 孤儿 running 点回收(#6 修复兜底):queue 行已消失(误删/崩溃丢队列)
|
||||
// 但 grid_points 仍卡在 running 的点,requeue_stale_tasks 找不到它们,
|
||||
// 在此按 tasks 表的 stale pending 记录兜底重置为 pending,让调度器重新派发。
|
||||
match bg_db_clone.reset_orphaned_running_points(stale_sec).await {
|
||||
// 孤儿点回收(#6 修复兜底,2026-08-02 涡旋事故重构):queue 凭证已消失
|
||||
// (误删/崩溃丢队列/insert 后 push 前崩溃)但 grid_points 仍卡在
|
||||
// running/queued 的点,requeue_stale_tasks 找不到它们。经 MQ 活性交叉
|
||||
// 校验确认真孤儿后重置为 pending 让调度器重新派发,并清除作为判据的
|
||||
// 僵尸 tasks 行(旧实现仅凭 tasks 表 stale pending 行判定,僵尸行使
|
||||
// 判据恒真 → 重复派发涡旋,已废弃)。
|
||||
match bg_scheduler_clone.reclaim_orphaned_points(stale_sec).await {
|
||||
Ok(reset) => {
|
||||
if reset > 0 {
|
||||
info!("已回收 {} 个孤儿 running 网格点(领用凭证丢失,重置为 pending)", reset);
|
||||
info!(
|
||||
"已回收 {} 个孤儿网格点(领用凭证丢失,重置为 pending)",
|
||||
reset
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("回收孤儿 running 网格点失败: {}", e);
|
||||
tracing::warn!("回收孤儿网格点失败: {}", e);
|
||||
has_error = true;
|
||||
}
|
||||
}
|
||||
@@ -402,6 +412,10 @@ async fn main() -> Result<()> {
|
||||
"/admin/nodes/:node_id/enable",
|
||||
post(api::admin::enable_node),
|
||||
)
|
||||
.route(
|
||||
"/admin/nodes/:node_id/quota",
|
||||
post(api::admin::set_node_quota),
|
||||
)
|
||||
.layer(DefaultBodyLimit::max(DEFAULT_BODY_LIMIT));
|
||||
|
||||
// 合并两个子 router:各自携带自己的 body limit,互不覆盖。
|
||||
@@ -431,9 +445,7 @@ async fn main() -> Result<()> {
|
||||
let auth_layer = axum::middleware::from_fn_with_state(state.clone(), api::auth_middleware);
|
||||
api_router.layer(auth_layer).layer(rate_limit_layer)
|
||||
} else {
|
||||
info!(
|
||||
"DCTS_AUTH_DISABLE=1 已生效:服务端运行在无鉴权模式(仅限本地调试,切勿用于生产)。"
|
||||
);
|
||||
info!("DCTS_AUTH_DISABLE=1 已生效:服务端运行在无鉴权模式(仅限本地调试,切勿用于生产)。");
|
||||
api_router
|
||||
};
|
||||
|
||||
|
||||
+2121
-59
File diff suppressed because it is too large
Load Diff
@@ -1466,7 +1466,8 @@ async fn test_import_seed_admin_endpoint() {
|
||||
|
||||
// 4. 未收敛点 → 200,但不写 .7、grid_points 维持 pending(未建 converged)。
|
||||
let conv_fail = make_legacy_conv_json("t20000_g5.0_he-2_c-4_n-4_o-4_fail", false);
|
||||
let body_bytes = make_import_multipart("boundary4", &conv_fail, b"WONT_BE_USED", "x.7", "cold_run");
|
||||
let body_bytes =
|
||||
make_import_multipart("boundary4", &conv_fail, b"WONT_BE_USED", "x.7", "cold_run");
|
||||
let res = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
@@ -1538,8 +1539,13 @@ async fn test_import_seed_python_legacy_conv_json() {
|
||||
|
||||
let name = "t20000_g5.0_he-2_c-4_n-4_o-4";
|
||||
let conv = make_python_legacy_conv_json(name);
|
||||
let body_bytes =
|
||||
make_import_multipart("boundaryL", &conv, b"FAKE_ATMOS_7", &format!("{name}.7"), "cold_run");
|
||||
let body_bytes = make_import_multipart(
|
||||
"boundaryL",
|
||||
&conv,
|
||||
b"FAKE_ATMOS_7",
|
||||
&format!("{name}.7"),
|
||||
"cold_run",
|
||||
);
|
||||
let res = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
@@ -1667,6 +1673,7 @@ async fn test_node_disable_enable_flow() {
|
||||
timeout_sec: 60,
|
||||
workflow_name: Some("wf_test".to_string()),
|
||||
wave: 0,
|
||||
..Default::default()
|
||||
};
|
||||
queue.push_task(&task).await.unwrap();
|
||||
|
||||
@@ -1883,6 +1890,232 @@ async fn test_node_disable_enable_flow() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_admin_set_node_quota_and_heartbeat_sync() {
|
||||
// 验证动态 CPU 槽位配额下发链路(见 docs/dynamic_cpu_slots_design.md):
|
||||
// admin POST /quota 设 admin_max_slots → 落库 nodes.admin_max_slots →
|
||||
// list_nodes_with_credentials 返回配额 → 心跳响应体透传 admin_max_slots。
|
||||
// 覆盖:设值 / 清除(null) / 负数 400 / 不存在节点 404。
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let db_path = temp_dir.path().join("quota_db.db");
|
||||
let queue_db_path = temp_dir.path().join("quota_queue.db");
|
||||
let seeds_dir = temp_dir.path().join("results");
|
||||
std::fs::create_dir_all(&seeds_dir).unwrap();
|
||||
|
||||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||||
let queue = Arc::new(
|
||||
SqliteTaskQueue::new(&queue_db_path.to_string_lossy())
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let scheduler = Arc::new(GridScheduler::new(db.clone(), queue.clone()));
|
||||
|
||||
// 注册并审批 node-quota-test,颁发 token
|
||||
let reg = common::models::NodeRegisterRequest {
|
||||
node_id: "node-quota-test".to_string(),
|
||||
max_slots: 4,
|
||||
};
|
||||
db.register_node(®).await.unwrap();
|
||||
let token = db.approve_node("node-quota-test").await.unwrap();
|
||||
|
||||
let state = AppState {
|
||||
db: db.clone(),
|
||||
queue: queue.clone(),
|
||||
scheduler,
|
||||
seeds_dir: seeds_dir.to_string_lossy().to_string(),
|
||||
rate_limiter: server::api::rate_limit::RateLimiter::new(
|
||||
5,
|
||||
std::time::Duration::from_secs(300),
|
||||
),
|
||||
admin_token: Some("admin-secret".to_string()),
|
||||
auth_disabled: false,
|
||||
admin_sessions: std::sync::Arc::new(tokio::sync::RwLock::new(
|
||||
std::collections::HashMap::new(),
|
||||
)),
|
||||
};
|
||||
|
||||
let api_router = axum::Router::new()
|
||||
.route(
|
||||
"/node/heartbeat",
|
||||
axum::routing::post(server::api::node::heartbeat_node),
|
||||
)
|
||||
.route(
|
||||
"/admin/nodes/:node_id/quota",
|
||||
axum::routing::post(server::api::admin::set_node_quota),
|
||||
);
|
||||
let auth_layer =
|
||||
axum::middleware::from_fn_with_state(state.clone(), server::api::auth_middleware);
|
||||
let app = axum::Router::new()
|
||||
.nest("/api", api_router.layer(auth_layer))
|
||||
.with_state(state);
|
||||
|
||||
// 1. 基线心跳:admin_max_slots 应为 null(无配额限制)
|
||||
let hb = serde_json::json!({"node_id":"node-quota-test","active_slots":0,"cpu_usage":10.0,"memory_usage":20.0});
|
||||
let res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/node/heartbeat")
|
||||
.header("authorization", format!("Bearer {}", token))
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(serde_json::to_vec(&hb).unwrap()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
let body: serde_json::Value = serde_json::from_slice(
|
||||
&axum::body::to_bytes(res.into_body(), usize::MAX)
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(body["status"], "ok");
|
||||
assert!(
|
||||
body["admin_max_slots"].is_null(),
|
||||
"未设配额时心跳响应 admin_max_slots 应为 null"
|
||||
);
|
||||
|
||||
// 2. admin 设配额 = 2 → 200
|
||||
let res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/admin/nodes/node-quota-test/quota")
|
||||
.header("authorization", "Bearer admin-secret")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
serde_json::to_vec(&serde_json::json!({"admin_max_slots": 2})).unwrap(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
|
||||
// 2a. 配额落库(list_nodes_with_credentials)
|
||||
let nodes = db.list_nodes_with_credentials().await.unwrap();
|
||||
let me = nodes
|
||||
.iter()
|
||||
.find(|n| n.node_id == "node-quota-test")
|
||||
.unwrap();
|
||||
assert_eq!(me.admin_max_slots, Some(2), "配额应已落库为 Some(2)");
|
||||
|
||||
// 3. 心跳响应透传 admin_max_slots = 2
|
||||
let res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/node/heartbeat")
|
||||
.header("authorization", format!("Bearer {}", token))
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(serde_json::to_vec(&hb).unwrap()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
let body: serde_json::Value = serde_json::from_slice(
|
||||
&axum::body::to_bytes(res.into_body(), usize::MAX)
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(body["admin_max_slots"], 2, "心跳响应应透传配额 2");
|
||||
|
||||
// 3a. 请求体缺 admin_max_slots 字段(畸形输入)→ 400,绝不能静默当作「清除配额」
|
||||
// (审查修复 M1:Option<i32> 缺字段默认 None 会把畸形请求误判为清除限制)。
|
||||
let res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/admin/nodes/node-quota-test/quota")
|
||||
.header("authorization", "Bearer admin-secret")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
serde_json::to_vec(&serde_json::json!({})).unwrap(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
res.status(),
|
||||
StatusCode::BAD_REQUEST,
|
||||
"缺字段应 400 而非静默清除配额"
|
||||
);
|
||||
// 畸形请求无副作用:既有配额仍为 Some(2),未被静默清除
|
||||
let nodes = db.list_nodes_with_credentials().await.unwrap();
|
||||
let me = nodes
|
||||
.iter()
|
||||
.find(|n| n.node_id == "node-quota-test")
|
||||
.unwrap();
|
||||
assert_eq!(me.admin_max_slots, Some(2), "畸形请求不应改动既有配额");
|
||||
|
||||
// 4. 设 null 清除配额 → 心跳响应 admin_max_slots = null
|
||||
let res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/admin/nodes/node-quota-test/quota")
|
||||
.header("authorization", "Bearer admin-secret")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
serde_json::to_vec(&serde_json::json!({"admin_max_slots": null})).unwrap(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
let nodes = db.list_nodes_with_credentials().await.unwrap();
|
||||
let me = nodes
|
||||
.iter()
|
||||
.find(|n| n.node_id == "node-quota-test")
|
||||
.unwrap();
|
||||
assert_eq!(me.admin_max_slots, None, "清除后配额应为 None");
|
||||
|
||||
// 5. 负数 → 400
|
||||
let res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/admin/nodes/node-quota-test/quota")
|
||||
.header("authorization", "Bearer admin-secret")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
serde_json::to_vec(&serde_json::json!({"admin_max_slots": -1})).unwrap(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::BAD_REQUEST, "负数配额应 400");
|
||||
|
||||
// 6. 不存在节点 → 404
|
||||
let res = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/admin/nodes/ghost-node/quota")
|
||||
.header("authorization", "Bearer admin-secret")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
serde_json::to_vec(&serde_json::json!({"admin_max_slots": 1})).unwrap(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::NOT_FOUND, "不存在节点应 404");
|
||||
}
|
||||
|
||||
// ===== 工作流执行观测端点测试(stats / points / point detail) =====
|
||||
|
||||
/// 测试专用:走真实写路径派发并回报一个网格点任务。
|
||||
@@ -1907,6 +2140,7 @@ async fn dispatch_and_report(
|
||||
timeout_sec: 7200,
|
||||
workflow_name: Some(wf.to_string()),
|
||||
wave: 0,
|
||||
..Default::default()
|
||||
};
|
||||
db.insert_task(&spec).await.unwrap();
|
||||
let report = common::models::TaskReport {
|
||||
@@ -1929,6 +2163,7 @@ async fn dispatch_and_report(
|
||||
Some("nl stage diverged".to_string())
|
||||
},
|
||||
summary_json: "{}".to_string(),
|
||||
failed_stage: None,
|
||||
};
|
||||
db.record_task_report(&report, wf).await.unwrap();
|
||||
}
|
||||
@@ -2844,3 +3079,340 @@ async fn test_wf_progress_endpoint() {
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
/// 构造仅含 report 字段的 multipart body(模拟节点上报,不带 seed_file)。
|
||||
fn make_report_only_multipart(boundary: &str, report_json: &str) -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes());
|
||||
body.extend_from_slice(b"Content-Disposition: form-data; name=\"report\"\r\n");
|
||||
body.extend_from_slice(b"Content-Type: application/json\r\n\r\n");
|
||||
body.extend_from_slice(report_json.as_bytes());
|
||||
body.extend_from_slice(b"\r\n");
|
||||
body.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes());
|
||||
body
|
||||
}
|
||||
|
||||
/// 端到端回归(2026-08-02 涡旋事故):已收敛点收到迟到的重复失败报告时,
|
||||
/// 走完整 HTTP 链路(claim → report)后状态保持 converged,且不触发种子回退
|
||||
/// (state_changed=false 跳过 fallback;纵有可用种子也不产生 seed_step 任务)。
|
||||
#[tokio::test]
|
||||
async fn test_duplicate_failure_report_cannot_flip_converged() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let db_path = temp_dir.path().join("flip_db.db");
|
||||
let queue_db_path = temp_dir.path().join("flip_queue.db");
|
||||
let seeds_dir = temp_dir.path().join("results");
|
||||
std::fs::create_dir_all(&seeds_dir).unwrap();
|
||||
|
||||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||||
let queue = Arc::new(
|
||||
SqliteTaskQueue::new(&queue_db_path.to_string_lossy())
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let scheduler = Arc::new(GridScheduler::new(db.clone(), queue.clone()));
|
||||
|
||||
// 预置节点与 token。
|
||||
let reg = common::models::NodeRegisterRequest {
|
||||
node_id: "node-flip".to_string(),
|
||||
max_slots: 2,
|
||||
};
|
||||
db.register_node(®).await.unwrap();
|
||||
let token = db.issue_node_token("node-flip").await.unwrap();
|
||||
|
||||
let state = AppState {
|
||||
db: db.clone(),
|
||||
queue: queue.clone(),
|
||||
scheduler,
|
||||
seeds_dir: seeds_dir.to_string_lossy().to_string(),
|
||||
rate_limiter: server::api::rate_limit::RateLimiter::new(
|
||||
100,
|
||||
std::time::Duration::from_secs(300),
|
||||
),
|
||||
admin_token: None,
|
||||
auth_disabled: false,
|
||||
admin_sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
|
||||
};
|
||||
let api_router = axum::Router::new()
|
||||
.route(
|
||||
"/task/claim",
|
||||
axum::routing::post(server::api::task::claim_task),
|
||||
)
|
||||
.route(
|
||||
"/task/report",
|
||||
axum::routing::post(server::api::task::report_task),
|
||||
);
|
||||
let auth_layer =
|
||||
axum::middleware::from_fn_with_state(state.clone(), server::api::auth_middleware);
|
||||
let app = axum::Router::new()
|
||||
.nest("/api", api_router.layer(auth_layer))
|
||||
.with_state(state);
|
||||
|
||||
// 工作流 running + 网格点 + 可用种子(若回退被错误触发,必然能匹配到种子并派发任务,
|
||||
// 使"无 seed_step 任务"断言成为强证据)。
|
||||
db.upsert_workflow("wf_flip", None, "", "running")
|
||||
.await
|
||||
.unwrap();
|
||||
let params = common::models::GridPointParams {
|
||||
teff: 35000.0.into(),
|
||||
logg: 5.5.into(),
|
||||
loghe: (-1.0).into(),
|
||||
logc: (-2.0).into(),
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
};
|
||||
let name = params.model_name();
|
||||
db.upsert_grid_point(¶ms, 0, "wf_flip").await.unwrap();
|
||||
db.insert_seed_named(&name, ¶ms, "/tmp/flip_seed.7")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// ---- 任务 A:正常收敛 ----
|
||||
let task_a = common::models::TaskSpec {
|
||||
task_id: uuid::Uuid::new_v4(),
|
||||
point_name: name.clone(),
|
||||
params: params.clone(),
|
||||
task_type: common::models::TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 7200,
|
||||
workflow_name: Some("wf_flip".to_string()),
|
||||
wave: 0,
|
||||
..Default::default()
|
||||
};
|
||||
db.insert_task(&task_a).await.unwrap();
|
||||
queue.push_task(&task_a).await.unwrap();
|
||||
|
||||
let claim_res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/task/claim")
|
||||
.header("authorization", format!("Bearer {}", token))
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(claim_res.status(), StatusCode::OK);
|
||||
|
||||
let report_a = common::models::TaskReport {
|
||||
task_id: task_a.task_id,
|
||||
point_name: name.clone(),
|
||||
params: Some(params.clone()),
|
||||
node_id: "node-flip".to_string(),
|
||||
status: common::models::TaskStatus::Completed,
|
||||
converged: true,
|
||||
max_relc: Some(0.0005),
|
||||
atmosphere_has_nan: false,
|
||||
elapsed_sec: 120.0,
|
||||
error_message: None,
|
||||
summary_json: "{}".to_string(),
|
||||
failed_stage: None,
|
||||
};
|
||||
let body_a =
|
||||
make_report_only_multipart("flipbound1", &serde_json::to_string(&report_a).unwrap());
|
||||
let report_res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/task/report")
|
||||
.header("authorization", format!("Bearer {}", token))
|
||||
.header("content-type", "multipart/form-data; boundary=flipbound1")
|
||||
.body(Body::from(body_a))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(report_res.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
db.get_grid_point_status(&name, "wf_flip")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.0,
|
||||
"converged"
|
||||
);
|
||||
|
||||
// ---- 任务 B:迟到的重复失败报告(涡旋残留任务的典型行为)----
|
||||
let task_b = common::models::TaskSpec {
|
||||
task_id: uuid::Uuid::new_v4(),
|
||||
point_name: name.clone(),
|
||||
params: params.clone(),
|
||||
task_type: common::models::TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 7200,
|
||||
workflow_name: Some("wf_flip".to_string()),
|
||||
wave: 0,
|
||||
..Default::default()
|
||||
};
|
||||
db.insert_task(&task_b).await.unwrap();
|
||||
queue.push_task(&task_b).await.unwrap();
|
||||
|
||||
let claim_b = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/task/claim")
|
||||
.header("authorization", format!("Bearer {}", token))
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(claim_b.status(), StatusCode::OK);
|
||||
|
||||
let report_b = common::models::TaskReport {
|
||||
task_id: task_b.task_id,
|
||||
point_name: name.clone(),
|
||||
params: Some(params.clone()),
|
||||
node_id: "node-flip".to_string(),
|
||||
status: common::models::TaskStatus::Failed,
|
||||
converged: false,
|
||||
max_relc: Some(9.5e5),
|
||||
atmosphere_has_nan: false,
|
||||
elapsed_sec: 130.0,
|
||||
error_message: Some("nl stage diverged".to_string()),
|
||||
summary_json: "{}".to_string(),
|
||||
failed_stage: None,
|
||||
};
|
||||
let body_b =
|
||||
make_report_only_multipart("flipbound2", &serde_json::to_string(&report_b).unwrap());
|
||||
let report_b_res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/task/report")
|
||||
.header("authorization", format!("Bearer {}", token))
|
||||
.header("content-type", "multipart/form-data; boundary=flipbound2")
|
||||
.body(Body::from(body_b))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
report_b_res.status(),
|
||||
StatusCode::OK,
|
||||
"上报本身应成功(吸收)"
|
||||
);
|
||||
|
||||
// 核心断言:converged 不被翻黑,且未触发任何种子回退任务。
|
||||
assert_eq!(
|
||||
db.get_grid_point_status(&name, "wf_flip")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.0,
|
||||
"converged",
|
||||
"迟到失败报告不得翻黑 converged 点"
|
||||
);
|
||||
assert!(
|
||||
db.has_pending_tasks_for_point(&name, "wf_flip", Some("seed_step"))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"不得因迟到失败报告派发 seed_step"
|
||||
);
|
||||
assert!(
|
||||
!db.has_seed_step_attempt(&name, "wf_flip").await.unwrap(),
|
||||
"全程不应产生任何 seed_step 行"
|
||||
);
|
||||
}
|
||||
|
||||
/// save_workflow 阶段配置合法性校验(修复审查 #4/#5):
|
||||
/// - 双阶段全关 → 400(无任何计算可执行);
|
||||
/// - 启用阶段空策略链 → 400(调度无顺位可派,任务必失败);
|
||||
/// - 合法配置(synspec-only 场景 B / 默认双开)→ 200。
|
||||
#[tokio::test]
|
||||
async fn test_save_workflow_validates_stage_configs() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let db_path = temp_dir.path().join("wfval_db.db");
|
||||
let queue_db_path = temp_dir.path().join("wfval_queue.db");
|
||||
let seeds_dir = temp_dir.path().join("results");
|
||||
std::fs::create_dir_all(&seeds_dir).unwrap();
|
||||
|
||||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||||
let queue = Arc::new(
|
||||
SqliteTaskQueue::new(&queue_db_path.to_string_lossy())
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let scheduler = Arc::new(GridScheduler::new(db.clone(), queue.clone()));
|
||||
let state = AppState {
|
||||
db: db.clone(),
|
||||
queue,
|
||||
scheduler,
|
||||
seeds_dir: seeds_dir.to_string_lossy().to_string(),
|
||||
rate_limiter: server::api::rate_limit::RateLimiter::new(
|
||||
5,
|
||||
std::time::Duration::from_secs(300),
|
||||
),
|
||||
admin_token: Some("admin-secret".to_string()),
|
||||
auth_disabled: false,
|
||||
admin_sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
|
||||
};
|
||||
let app = axum::Router::new()
|
||||
.route(
|
||||
"/api/workflows",
|
||||
axum::routing::post(server::api::workflow::save_workflow),
|
||||
)
|
||||
.with_state(state);
|
||||
|
||||
let base_yaml = "grid:\n teff: [35000]\n logg: [5.5]\n loghe: [-1]\n logc: [-2]\n logn: [-2]\n logo: [-2]";
|
||||
let post_save = |name: &str, yaml: &str| {
|
||||
let app = app.clone();
|
||||
let body = serde_json::json!({
|
||||
"name": name,
|
||||
"description": null,
|
||||
"config_yaml": yaml,
|
||||
});
|
||||
async move {
|
||||
app.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/workflows")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(serde_json::to_vec(&body).unwrap()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
};
|
||||
|
||||
// 1. 双阶段全关 → 400
|
||||
let both_off = format!(
|
||||
"{}\ntlusty:\n enabled: false\n policy: skip_converged\n strategies: [cold_run, seed_step]\n\
|
||||
synspec_stage:\n enabled: false\n policy: skip_converged\n strategies: [standard]\n",
|
||||
base_yaml
|
||||
);
|
||||
let res = post_save("wf_val_a", &both_off).await;
|
||||
assert_eq!(res.status(), StatusCode::BAD_REQUEST, "双阶段全关应 400");
|
||||
|
||||
// 2. 启用阶段空策略链 → 400
|
||||
let empty_chain = format!(
|
||||
"{}\ntlusty:\n enabled: true\n policy: skip_converged\n strategies: []\n",
|
||||
base_yaml
|
||||
);
|
||||
let res = post_save("wf_val_b", &empty_chain).await;
|
||||
assert_eq!(
|
||||
res.status(),
|
||||
StatusCode::BAD_REQUEST,
|
||||
"启用阶段空策略链应 400"
|
||||
);
|
||||
|
||||
// 3. 合法:TLUSTY 关 + SYNSPEC 启(设计 §2.2 场景 B:仅更新光谱)→ 200
|
||||
let syn_only = format!(
|
||||
"{}\ntlusty:\n enabled: false\n policy: skip_converged\n strategies: [cold_run, seed_step]\n\
|
||||
synspec_stage:\n enabled: true\n policy: force_recompute\n strategies: [standard]\n",
|
||||
base_yaml
|
||||
);
|
||||
let res = post_save("wf_val_c", &syn_only).await;
|
||||
assert_eq!(res.status(), StatusCode::OK, "synspec-only 合法配置应 200");
|
||||
|
||||
// 4. 合法:无阶段块(默认双开)→ 200
|
||||
let res = post_save("wf_val_d", base_yaml).await;
|
||||
assert_eq!(res.status(), StatusCode::OK, "默认配置应 200");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user