feat(all): 数据库模块化拆分与版本化迁移、任务引擎命名体系收敛、物理输出校验加固与用户配置接通
- server/db: 拆 4929 行 db.rs 单体为 db/ 目录，migrations.rs 引入 PRAGMA user_version
版本化迁移运行器（M1~M13）
- 任务引擎 Phase 6/7b/7c 改名收敛：EngineStageConfig→PhaseConfig、StagePolicy→ResumePolicy、
Converged→Completed、删除 task_type 列、success_method 拆 tlusty_/synspec_ 双列、
新增 tlusty_status/synspec_status 半失败阶段守卫
- 科学正确性加固：conv_check 任意行 NaN/Inf/溢出判无效（0 行容忍）、新增 spec_is_valid
校验 SYNSPEC 脏谱、itek_history 逐次迭代全量保真、fmt_abn powf 溢出饱和
- 用户配置真正接通：tlusty_chain/tlusty_input 由死字段经 调度器→TaskSpec→executor→runner
透传生效；config 加载期 validate + deny_unknown_fields + 解析失败记 warn
- 调度修复：H1 活锁（pending_strategies 跳过已失败策略）、种子查找错误不再静默降级冷启动
- dashboard: 阶段配置面板 tlusty_stage/synspec_stage、"已完成"标签、迭代诊断展示
- docs: 新增 database_refactor_design.md，同步 database/api/PIPELINE/workflow_detail
This commit is contained in:
+142
-38
@@ -1,9 +1,9 @@
|
||||
use crate::config::{StageConfig, SynspecConfig};
|
||||
use crate::conv_check::{atmosphere_has_nan, check_fort9};
|
||||
use crate::config::{ChainStep, SynspecInput, TlustyInput};
|
||||
use crate::conv_check::{atmosphere_has_nan, check_fort9, extract_failure_hint, spec_is_valid};
|
||||
use crate::embedded::RuntimePaths;
|
||||
use crate::fort55_writer::generate_fort55_content;
|
||||
use crate::gen_input5::make_input5;
|
||||
use crate::models::{GridPointParams, ModelSummary, StageSummary, TaskType};
|
||||
use crate::models::{GridPointParams, ModelSummary, StepSummary};
|
||||
use crate::nst_writer::generate_nst_content;
|
||||
use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -13,9 +13,9 @@ use tokio::fs::File;
|
||||
use tokio::process::Command as AsyncCommand;
|
||||
use tracing::{info, warn};
|
||||
|
||||
pub fn default_cold_chain() -> Vec<StageConfig> {
|
||||
pub fn default_cold_chain() -> Vec<ChainStep> {
|
||||
vec![
|
||||
StageConfig {
|
||||
ChainStep {
|
||||
label: "lte".to_string(),
|
||||
lte: "T".to_string(),
|
||||
ltgray: "T".to_string(),
|
||||
@@ -30,7 +30,7 @@ pub fn default_cold_chain() -> Vec<StageConfig> {
|
||||
iacc: None,
|
||||
orelax: None,
|
||||
},
|
||||
StageConfig {
|
||||
ChainStep {
|
||||
label: "nc".to_string(),
|
||||
lte: "F".to_string(),
|
||||
ltgray: "F".to_string(),
|
||||
@@ -45,7 +45,7 @@ pub fn default_cold_chain() -> Vec<StageConfig> {
|
||||
iacc: None,
|
||||
orelax: None,
|
||||
},
|
||||
StageConfig {
|
||||
ChainStep {
|
||||
label: "nl".to_string(),
|
||||
lte: "F".to_string(),
|
||||
ltgray: "F".to_string(),
|
||||
@@ -63,9 +63,9 @@ pub fn default_cold_chain() -> Vec<StageConfig> {
|
||||
]
|
||||
}
|
||||
|
||||
pub fn default_seed_chain() -> Vec<StageConfig> {
|
||||
pub fn default_seed_chain() -> Vec<ChainStep> {
|
||||
vec![
|
||||
StageConfig {
|
||||
ChainStep {
|
||||
label: "seed_nc".to_string(),
|
||||
lte: "F".to_string(),
|
||||
ltgray: "F".to_string(),
|
||||
@@ -80,7 +80,7 @@ pub fn default_seed_chain() -> Vec<StageConfig> {
|
||||
iacc: None,
|
||||
orelax: None,
|
||||
},
|
||||
StageConfig {
|
||||
ChainStep {
|
||||
label: "nl".to_string(),
|
||||
lte: "F".to_string(),
|
||||
ltgray: "F".to_string(),
|
||||
@@ -98,6 +98,19 @@ pub fn default_seed_chain() -> Vec<StageConfig> {
|
||||
]
|
||||
}
|
||||
|
||||
/// 按当前策略选默认执行链（`custom_chain` 为 None/空时的兜底）。
|
||||
///
|
||||
/// Phase 6（P8）起取代废弃的 task_type 匹配：`"seed_step"` → 种子热启动链，其余策略
|
||||
/// （`cold_run` 等）→ 冷启动链。executor 现优先使用 TaskSpec.tlusty_chain_params
|
||||
/// （用户 YAML `tlusty_chain:` 配置），None/空才回退本函数的默认链。
|
||||
pub fn default_chain_for_strategy(current_strategy: &str) -> Vec<ChainStep> {
|
||||
if current_strategy == "seed_step" {
|
||||
default_seed_chain()
|
||||
} else {
|
||||
default_cold_chain()
|
||||
}
|
||||
}
|
||||
|
||||
/// 运行子进程，带超时与优雅退出（shutdown）感知。
|
||||
///
|
||||
/// 三种终止路径：
|
||||
@@ -199,19 +212,21 @@ impl<'a> ExecutionRunner<'a> {
|
||||
Self { runtime, work_dir }
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)] // 透传全参给 run_model_with_timeout（后者同 allow）
|
||||
pub async fn run_model(
|
||||
&self,
|
||||
params: &GridPointParams,
|
||||
name: &str,
|
||||
task_type: TaskType,
|
||||
custom_chain: Option<Vec<StageConfig>>,
|
||||
current_strategy: &str,
|
||||
custom_chain: Option<Vec<ChainStep>>,
|
||||
seed_atmos: Option<&Path>,
|
||||
synspec_cfg: Option<&SynspecConfig>,
|
||||
synspec_cfg: Option<&SynspecInput>,
|
||||
tlusty_input: Option<&TlustyInput>,
|
||||
) -> Result<ModelSummary> {
|
||||
self.run_model_with_timeout(
|
||||
params,
|
||||
name,
|
||||
task_type,
|
||||
current_strategy,
|
||||
custom_chain,
|
||||
seed_atmos,
|
||||
synspec_cfg,
|
||||
@@ -219,6 +234,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
true,
|
||||
7200,
|
||||
None,
|
||||
tlusty_input,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -228,19 +244,24 @@ impl<'a> ExecutionRunner<'a> {
|
||||
/// `tlusty_enabled` / `synspec_enabled` 控制各阶段是否运行：
|
||||
/// - TLUSTY 关闭：跳过 chain 循环，直接以 seed_atmos（或单独拉取的大气）作 final_7；
|
||||
/// - SYNSPEC 关闭：跳过光谱合成块（即便 final_7 存在）。
|
||||
///
|
||||
/// Phase 6（P8）：`current_strategy` 取代废弃的 `task_type`——执行链由
|
||||
/// `custom_chain`（executor 按 `tlusty_config.strategies[0]` 显式推导）决定；
|
||||
/// 该参数仅用于日志与 custom_chain=None 时的兜底（"seed_step"→种子链，否则冷启动链）。
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn run_model_with_timeout(
|
||||
&self,
|
||||
params: &GridPointParams,
|
||||
name: &str,
|
||||
task_type: TaskType,
|
||||
custom_chain: Option<Vec<StageConfig>>,
|
||||
current_strategy: &str,
|
||||
custom_chain: Option<Vec<ChainStep>>,
|
||||
seed_atmos: Option<&Path>,
|
||||
synspec_cfg: Option<&SynspecConfig>,
|
||||
synspec_cfg: Option<&SynspecInput>,
|
||||
tlusty_enabled: bool,
|
||||
synspec_enabled: bool,
|
||||
timeout_sec: u64,
|
||||
shutdown: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
|
||||
tlusty_input: Option<&TlustyInput>,
|
||||
) -> Result<ModelSummary> {
|
||||
// `name` 取自权威的 TaskSpec.point_name（DB 的 grid_points.name 列，源精度正确），
|
||||
// 而非 params.model_name()。原因：服务端把 GridPointParams 存成 6 个 REAL 数值列，
|
||||
@@ -255,7 +276,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
let model_dir = self.work_dir.join(name);
|
||||
tokio::fs::create_dir_all(&model_dir).await?;
|
||||
|
||||
info!("开始物理计算网格模型 {} (类型: {:?})", name, task_type);
|
||||
info!("开始物理计算网格模型 {} (策略: {})", name, current_strategy);
|
||||
let t0 = Instant::now();
|
||||
|
||||
// 1. Data directory symlink setup
|
||||
@@ -296,10 +317,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
let _ = tokio::fs::remove_file(&fort84).await;
|
||||
}
|
||||
|
||||
let chain = custom_chain.unwrap_or_else(|| match task_type {
|
||||
TaskType::ColdRun => default_cold_chain(),
|
||||
TaskType::SeedStep => default_seed_chain(),
|
||||
});
|
||||
let chain = custom_chain.unwrap_or_else(|| default_chain_for_strategy(current_strategy));
|
||||
|
||||
let mut stage_summaries = Vec::new();
|
||||
let mut current_seed: Option<PathBuf> = seed_atmos.map(|p| p.to_path_buf());
|
||||
@@ -329,13 +347,14 @@ impl<'a> ExecutionRunner<'a> {
|
||||
&stage_def.ltgray,
|
||||
metals,
|
||||
stage_def.ilvlin,
|
||||
tlusty_input,
|
||||
);
|
||||
|
||||
let input5_path = model_dir.join(format!("{}.5", name));
|
||||
tokio::fs::write(&input5_path, &input5_text).await?;
|
||||
|
||||
// Write nst file
|
||||
let nst_text = generate_nst_content(stage_def);
|
||||
let nst_text = generate_nst_content(stage_def, tlusty_input);
|
||||
tokio::fs::write(model_dir.join("nst"), &nst_text).await?;
|
||||
|
||||
// Prepare fort.8 for this stage
|
||||
@@ -386,7 +405,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
let fort9 = model_dir.join("fort.9");
|
||||
let fort7 = model_dir.join("fort.7");
|
||||
|
||||
let mut stage_summary = StageSummary {
|
||||
let mut stage_summary = StepSummary {
|
||||
label: stage_def.label.clone(),
|
||||
chmax: stage_def.chmax,
|
||||
lte: stage_def.lte.clone(),
|
||||
@@ -397,6 +416,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
last_iter: None,
|
||||
worst_depth: None,
|
||||
n_depths: None,
|
||||
itek_history: Vec::new(),
|
||||
};
|
||||
|
||||
if rc == 0 && fort7.is_file() {
|
||||
@@ -410,15 +430,50 @@ impl<'a> ExecutionRunner<'a> {
|
||||
stage_summary.last_iter = res.last_iter;
|
||||
stage_summary.worst_depth = Some(res.worst_depth);
|
||||
stage_summary.n_depths = Some(res.n_depths);
|
||||
// itek 全量保真（Phase 5b）：逐次迭代诊断随 summary_json/conv.json 落库。
|
||||
stage_summary.itek_history = res.itek_history;
|
||||
|
||||
// 漏洞5修复：发散时从 fort.6 提取求解器 STOP 行（SOLVE/SOLVES/RYBSOL）
|
||||
// 作为 note，提升归因质量。仅未收敛且无既有 note 时补（避免覆盖错误信息）。
|
||||
if !res.converged && stage_summary.note.is_none() {
|
||||
let fort6 = model_dir.join(format!("{}.6", name));
|
||||
if let Some(h) = extract_failure_hint(&fort6) {
|
||||
stage_summary.note = Some(format!("未收敛 [{}]", h));
|
||||
}
|
||||
}
|
||||
|
||||
// Save fort.9 snapshot
|
||||
let snap_name = format!("{}.{}_chmax{}.9", name, stage_def.label, eff_chmax);
|
||||
let _ = tokio::fs::copy(&fort9, model_dir.join(snap_name)).await;
|
||||
} else {
|
||||
// NITER=0 grey start without fort.9
|
||||
stage_summary.converged = true;
|
||||
stage_summary.best_max_relc = Some(0.0);
|
||||
stage_summary.note = Some("NITER=0 grey start".to_string());
|
||||
// fort.9 缺失：按 stage_def.niter 区分两种场景（漏洞2进阶修复）。
|
||||
// - niter==0：合法 grey start（lte 阶段不迭代，TLUSTY 不写 fort.9）。
|
||||
// converged=true 保留 grey start 语义；best_max_relc=None 不虚构
|
||||
// （避免污染 final_max_relc/种子选择）。
|
||||
// - niter>0：异常——配了迭代却无 fort.9，通常是 TLUSTY 启动失败
|
||||
// （call quit，如 temp 越界）或 IO 异常。判 converged=false，
|
||||
// 避免把崩溃误判为收敛。此前两种场景共用无校验分支无法区分。
|
||||
if stage_def.niter == 0 {
|
||||
stage_summary.converged = true;
|
||||
stage_summary.best_max_relc = None;
|
||||
stage_summary.note = Some("NITER=0 grey start".to_string());
|
||||
} else {
|
||||
stage_summary.converged = false;
|
||||
stage_summary.best_max_relc = None;
|
||||
// 补 fort.6 失败诊断（call quit 留言），便于排查启动失败原因。
|
||||
let fort6 = model_dir.join(format!("{}.6", name));
|
||||
let hint = extract_failure_hint(&fort6);
|
||||
stage_summary.note = Some(match hint {
|
||||
Some(h) => format!(
|
||||
"stage {} 配置 NITER={} 但 fort.9 缺失 [{}]",
|
||||
stage_def.label, stage_def.niter, h
|
||||
),
|
||||
None => format!(
|
||||
"stage {} 配置 NITER={} 但 fort.9 缺失（TLUSTY 未完成迭代）",
|
||||
stage_def.label, stage_def.niter
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Copy fort.7 as stage seed
|
||||
@@ -426,7 +481,14 @@ impl<'a> ExecutionRunner<'a> {
|
||||
let _ = tokio::fs::copy(&fort7, &stage_seed_path).await;
|
||||
current_seed = Some(stage_seed_path);
|
||||
} else {
|
||||
stage_summary.note = Some(format!("tlusty rc={} or missing fort.7", rc));
|
||||
// 漏洞5修复：fort.7 缺失分支（输入错误、temp 越界等 call quit 场景），
|
||||
// 从 fort.6 尾部提取 call quit / stop 留言补进 note，便于排查。
|
||||
let fort6 = model_dir.join(format!("{}.6", name));
|
||||
let hint = extract_failure_hint(&fort6);
|
||||
stage_summary.note = Some(match hint {
|
||||
Some(h) => format!("tlusty rc={} or missing fort.7 [{}]", rc, h),
|
||||
None => format!("tlusty rc={} or missing fort.7", rc),
|
||||
});
|
||||
}
|
||||
|
||||
// 快照本阶段的同名输入/输出文件，带阶段标签保留。
|
||||
@@ -485,7 +547,15 @@ impl<'a> ExecutionRunner<'a> {
|
||||
// fort.12/14 不存在，函数内按文件是否存在静默跳过。
|
||||
snapshot_tlusty_outputs(&model_dir, name).await;
|
||||
|
||||
let atmo_has_nan = atmosphere_has_nan(&final_7);
|
||||
// L2 修复：`atmosphere_has_nan` 对**缺失**文件返回 false（语义是"无 NaN"而非"有效"），
|
||||
// 与**空文件返回 true** 语义不对称。缺失最终大气 = 无可判定收敛的干净大气 →
|
||||
// 在此显式判定为无效（NaN），与空文件语义对齐。调用前提：final_7 应在收敛链产出；
|
||||
// 若缺失（如种子拷贝失败、TLUSTY 崩溃未写 fort.7），本守卫强制最终不收敛。
|
||||
let atmo_has_nan = if final_7.is_file() {
|
||||
atmosphere_has_nan(&final_7)
|
||||
} else {
|
||||
true
|
||||
};
|
||||
if atmo_has_nan {
|
||||
final_converged = false;
|
||||
}
|
||||
@@ -518,7 +588,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
let _ = tokio::fs::remove_file(&fort55_path).await;
|
||||
let _ = tokio::fs::remove_file(&fort19_path).await;
|
||||
|
||||
let default_cfg = SynspecConfig {
|
||||
let default_cfg = SynspecInput {
|
||||
wstart: 1400.0,
|
||||
wend: 1410.0,
|
||||
imode: 0,
|
||||
@@ -577,11 +647,22 @@ impl<'a> ExecutionRunner<'a> {
|
||||
|
||||
// Copy/move outputs: fort.7 (Synspec spectrum) -> .spec, fort.17 -> .cont, fort.12 -> .iden
|
||||
if model_dir.join("fort.7").is_file() {
|
||||
let _ = tokio::fs::rename(
|
||||
model_dir.join("fort.7"),
|
||||
model_dir.join(format!("{}.spec", name)),
|
||||
)
|
||||
.await;
|
||||
let spec_path = model_dir.join(format!("{}.spec", name));
|
||||
let _ = tokio::fs::rename(model_dir.join("fort.7"), &spec_path).await;
|
||||
|
||||
// 漏洞1修复（P0）：SYNSPEC .spec 内容校验。
|
||||
// gfortran 下 SYNSPEC 几乎所有错误路径 rc=0，旧代码只做 is_file() 存在性
|
||||
// 检查，导致脏谱（NaN/Inf/行数不足/全零）被当作 Completed 归档——全链路
|
||||
// 最大的科学正确性风险。命中无效则置 synspec_rc 非零 + synspec_error 描述，
|
||||
// 让 reporter 判 Failed 并触发 synspec 策略链回退。
|
||||
// 守卫 synspec_err.is_none()：避免覆盖上游 fort.8/fort.55 复制失败的既有 err。
|
||||
if synspec_err.is_none() {
|
||||
if let Some(reason) = spec_is_valid(&spec_path) {
|
||||
warn!("spec 校验失败: {}", reason);
|
||||
synspec_rc = Some(1);
|
||||
synspec_err = Some(reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
if model_dir.join("fort.17").is_file() {
|
||||
let _ = tokio::fs::copy(
|
||||
@@ -642,7 +723,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
let note = {
|
||||
let mut notes: Vec<String> = Vec::new();
|
||||
if atmo_has_nan {
|
||||
notes.push("Invalidated: atmosphere contains >10% NaN lines".to_string());
|
||||
notes.push("Invalidated: atmosphere contains NaN/Inf lines".to_string());
|
||||
}
|
||||
if let Some(ref err) = synspec_err {
|
||||
notes.push(format!("synspec error: {}", err));
|
||||
@@ -662,7 +743,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
name: name.to_string(),
|
||||
params: params.clone(),
|
||||
stages: stage_summaries,
|
||||
converged: final_converged,
|
||||
result_valid: final_converged,
|
||||
final_max_relc,
|
||||
final_chmax,
|
||||
seed: seed_atmos.map(|p| p.to_string_lossy().to_string()),
|
||||
@@ -687,6 +768,29 @@ mod tests {
|
||||
use super::snapshot_tlusty_outputs;
|
||||
use crate::models::{GridAxisValue, GridPointParams};
|
||||
|
||||
/// Phase 6（P8）：执行链按当前策略派生——seed_step 走种子热启动链，其余走冷启动链。
|
||||
#[test]
|
||||
fn test_default_chain_for_strategy() {
|
||||
let labels = |chain: Vec<super::ChainStep>| -> Vec<String> {
|
||||
chain.into_iter().map(|s| s.label).collect()
|
||||
};
|
||||
// seed_step → 种子热启动链（seed_nc/nl）。
|
||||
assert_eq!(
|
||||
labels(super::default_chain_for_strategy("seed_step")),
|
||||
vec!["seed_nc".to_string(), "nl".to_string()]
|
||||
);
|
||||
// 其余策略（cold_run / 未知如 standard）→ 冷启动链（lte/nc/nl）。
|
||||
assert_eq!(
|
||||
labels(super::default_chain_for_strategy("cold_run")),
|
||||
vec!["lte".to_string(), "nc".to_string(), "nl".to_string()]
|
||||
);
|
||||
assert_eq!(
|
||||
labels(super::default_chain_for_strategy("standard")),
|
||||
vec!["lte".to_string(), "nc".to_string(), "nl".to_string()],
|
||||
"未知策略兜底冷启动链（synspec-only strategies[0] 等）"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_synspec_timeout_calculation() {
|
||||
let long_tlusty_timeout: u64 = 7200;
|
||||
|
||||
Reference in New Issue
Block a user