use crate::config::{ChainStep, SynspecInput, TlustyInput}; use crate::conv_check::{atmosphere_has_nan, check_fort9, extract_failure_hint, spec_is_valid}; use crate::embedded::RuntimePaths; use crate::fort55_writer::generate_fort55_content; use crate::gen_input5::make_input5; use crate::models::{GridPointParams, ModelSummary, StepSummary}; use crate::nst_writer::generate_nst_content; use anyhow::Result; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::Instant; use tokio::fs::File; use tokio::process::Command as AsyncCommand; use tracing::{info, warn}; pub fn default_cold_chain() -> Vec { vec![ ChainStep { label: "lte".to_string(), lte: "T".to_string(), ltgray: "T".to_string(), ilvlin: 0, require_converged: false, niter: 0, chmax: None, itek: None, metals: Some("cno".to_string()), ichang: None, idlte: None, iacc: None, orelax: None, }, ChainStep { label: "nc".to_string(), lte: "F".to_string(), ltgray: "F".to_string(), ilvlin: 0, require_converged: false, niter: 10, chmax: None, itek: None, metals: Some("cno".to_string()), ichang: None, idlte: None, iacc: None, orelax: None, }, ChainStep { label: "nl".to_string(), lte: "F".to_string(), ltgray: "F".to_string(), ilvlin: 100, require_converged: true, niter: 100, chmax: None, itek: None, metals: Some("cno".to_string()), ichang: None, idlte: None, iacc: None, orelax: None, }, ] } pub fn default_seed_chain() -> Vec { vec![ ChainStep { label: "seed_nc".to_string(), lte: "F".to_string(), ltgray: "F".to_string(), ilvlin: 0, require_converged: false, niter: 20, chmax: None, itek: None, metals: Some("cno".to_string()), ichang: Some(0), idlte: None, iacc: None, orelax: None, }, ChainStep { label: "nl".to_string(), lte: "F".to_string(), ltgray: "F".to_string(), ilvlin: 100, require_converged: true, niter: 100, chmax: None, itek: None, metals: Some("cno".to_string()), ichang: Some(0), idlte: None, iacc: None, orelax: None, }, ] } /// 按当前策略选默认执行链(`custom_chain` 为 None/空时的兜底)。 /// /// Phase 6(P8)起取代废弃的 task_type 匹配:`"seed_step"` → 种子热启动链,其余策略 /// (`cold_run` 等)→ 冷启动链。executor 现优先使用 TaskSpec.tlusty_chain_params /// (用户 YAML `tlusty_chain:` 配置),None/空才回退本函数的默认链。 pub fn default_chain_for_strategy(current_strategy: &str) -> Vec { if current_strategy == "seed_step" { default_seed_chain() } else { default_cold_chain() } } /// 运行子进程,带超时与优雅退出(shutdown)感知。 /// /// 三种终止路径: /// 1. 子进程正常结束 → 返回 ExitStatus。 /// 2. 超时(timeout_sec)→ SIGKILL 子进程 + 二级 30s 等待 reap,超时则放弃 Child(kill_on_drop 兜底)。 /// 3. shutdown 信号(节点收到 SIGTERM/SIGINT)→ 立即 SIGKILL 子进程并快速返回 Err, /// 让上层尽快退出(在途任务的结果会丢失,由服务端 stale 重投兜底)。 /// /// 历史 bug:超时 kill 后 `child.wait().await` 无二级超时,Fortran 进程若卡死 /// (OpenMP hang / ptrace)会使 wait 永久阻塞,超时机制名存实亡、slot 永久泄漏。 async fn run_child_async_with_timeout( mut child: tokio::process::Child, timeout_sec: u64, shutdown: Option>, ) -> Result { let timeout_fut = tokio::time::timeout(tokio::time::Duration::from_secs(timeout_sec), child.wait()); // 若提供了 shutdown 标志,则与超时/正常结束三路 select;否则只等超时/正常结束。 let outcome: Result = if let Some(flag) = shutdown { let shutdown_watcher = async move { // 轮询 shutdown 标志(10ms 粒度足够灵敏,开销可忽略)。 loop { if flag.load(std::sync::atomic::Ordering::Acquire) { return; } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } }; tokio::select! { biased; // 优先响应 shutdown _ = shutdown_watcher => Err(ShutdownOrTimeout::Shutdown), r = timeout_fut => match r { Ok(res) => Ok(res?), Err(_) => Err(ShutdownOrTimeout::Timeout), }, } } else { match timeout_fut.await { Ok(res) => Ok(res?), Err(_) => Err(ShutdownOrTimeout::Timeout), } }; match outcome { Ok(status) => Ok(status), Err(ShutdownOrTimeout::Shutdown) => { let _ = child.start_kill(); let _ = tokio::time::timeout(std::time::Duration::from_secs(30), child.wait()).await; anyhow::bail!("节点收到退出信号,子进程已被终止"); } Err(ShutdownOrTimeout::Timeout) => { let _ = child.start_kill(); let _ = tokio::time::timeout(std::time::Duration::from_secs(30), child.wait()).await; anyhow::bail!("进程计算超时 (上限: {} 秒)", timeout_sec); } } } #[derive(Debug)] enum ShutdownOrTimeout { Shutdown, Timeout, } /// 快照 TLUSTY 最终模型的 b 因子与出射谱,防止被 SYNSPEC 覆盖丢失。 /// /// TLUSTY 在最终迭代(`LFIN=.TRUE.`)经 `OUTPRI` 写出(见 tlusty208.f): /// - `fort.12`:b 因子 / 非 LTE 偏离因子表(头 2I5 + 每深度 TEMP/ELEC/DENS/BFAC,格式 701/702/703)。 /// 随后 SYNSPEC 会复用 unit 12 写谱线证认表并覆盖它(runner 再将其存为 `.iden`), /// 故 TLUSTY 的 b 因子若不在此快照即静默丢失。 /// - `fort.14`:出射谱(波长 Å + Fλ,格式 614),同样会被 SYNSPEC 的谱线数据覆盖。 /// /// 在收敛链循环结束(链上最后一次 TLUSTY 运行即最终模型)、SYNSPEC 启动前调用, /// 快照为 `.bfac` / `.emflux`,与科学核心产物一并进入归档白名单 /// (见 `result_filter::is_result_worthy` 的 `bfac`/`emflux` 后缀)。 /// 文件不存在时静默跳过(TLUSTY 未运行/未写出);IO 错误降级为 warn,不阻断主流程。 async fn snapshot_tlusty_outputs(model_dir: &Path, name: &str) { for (src, suffix) in [("fort.12", "bfac"), ("fort.14", "emflux")] { let src_path = model_dir.join(src); if !src_path.is_file() { continue; } let dst = model_dir.join(format!("{}.{}", name, suffix)); if let Err(e) = tokio::fs::copy(&src_path, &dst).await { warn!("快照 TLUSTY {} 到 {} 失败: {}", src, dst.display(), e); } } } pub struct ExecutionRunner<'a> { pub runtime: &'a RuntimePaths, pub work_dir: PathBuf, } impl<'a> ExecutionRunner<'a> { pub fn new(runtime: &'a RuntimePaths, work_dir: PathBuf) -> Self { Self { runtime, work_dir } } #[allow(clippy::too_many_arguments)] // 透传全参给 run_model_with_timeout(后者同 allow) pub async fn run_model( &self, params: &GridPointParams, name: &str, current_strategy: &str, custom_chain: Option>, seed_atmos: Option<&Path>, synspec_cfg: Option<&SynspecInput>, tlusty_input: Option<&TlustyInput>, ) -> Result { self.run_model_with_timeout( params, name, current_strategy, custom_chain, seed_atmos, synspec_cfg, true, true, 7200, None, tlusty_input, ) .await } /// 阶段独立配置执行入口(见 docs/task_engine_decoupling_design.md §5)。 /// /// `tlusty_enabled` / `synspec_enabled` 控制各阶段是否运行: /// - TLUSTY 关闭:跳过 chain 循环,直接以 seed_atmos(或单独拉取的大气)作 final_7; /// - SYNSPEC 关闭:跳过光谱合成块(即便 final_7 存在)。 /// /// Phase 6(P8):`current_strategy` 取代废弃的 `task_type`——执行链由 /// `custom_chain`(executor 按 `tlusty_config.strategies[0]` 显式推导)决定; /// 该参数仅用于日志与 custom_chain=None 时的兜底("seed_step"→种子链,否则冷启动链)。 #[allow(clippy::too_many_arguments)] pub async fn run_model_with_timeout( &self, params: &GridPointParams, name: &str, current_strategy: &str, custom_chain: Option>, seed_atmos: Option<&Path>, synspec_cfg: Option<&SynspecInput>, tlusty_enabled: bool, synspec_enabled: bool, timeout_sec: u64, shutdown: Option>, tlusty_input: Option<&TlustyInput>, ) -> Result { // `name` 取自权威的 TaskSpec.point_name(DB 的 grid_points.name 列,源精度正确), // 而非 params.model_name()。原因:服务端把 GridPointParams 存成 6 个 REAL 数值列, // 回读时用 from_value() 反推文本会丢精度("5.0"→"5"),导致 params.model_name() // 产出错误名(g5 而非 g5.0)。point_name 走独立 TEXT 列,精度全程保留。 // 下游(沙盒子目录、各阶段快照、conv.json.name、归档目录)全部用此 name, // 故只需在此处用权威 name 即可让整条链精度正确。 // // 历史:此处曾把 params.model_name() 与 name 对比并 warn 不一致。但该不一致是 // DB REAL 列回读丢精度的已知现象(runner 端无法修复,根治需改 DB schema 存原文), // 且 runner 已全程采用权威 name,对比结果不参与任何决策——故移除这段噪音 warn。 let model_dir = self.work_dir.join(name); tokio::fs::create_dir_all(&model_dir).await?; info!("开始物理计算网格模型 {} (策略: {})", name, current_strategy); let t0 = Instant::now(); // 1. Data directory symlink setup let link_data = model_dir.join("data"); if tokio::fs::symlink_metadata(&link_data).await.is_ok() || link_data.exists() { let _ = tokio::fs::remove_file(&link_data).await; } #[cfg(unix)] { let abs_data_dir = tokio::fs::canonicalize(&self.runtime.data_dir) .await .unwrap_or_else(|_| self.runtime.data_dir.clone()); if let Err(e) = std::os::unix::fs::symlink(&abs_data_dir, &link_data) { warn!("构建 data 数据集软链时发生提示性告警: {}", e); } } // 2. Initial fort.8 seed setup let fort8 = model_dir.join("fort.8"); if fort8.exists() { let _ = tokio::fs::remove_file(&fort8).await; } if let Some(seed_path) = seed_atmos { if seed_path.is_file() { if let Err(e) = tokio::fs::copy(seed_path, &fort8).await { warn!( "向工作沙盒引导填载首期收敛模型种子 fort.8 发生复制错误: {}", e ); } } } // Clean fort.84 residue to prevent NATOMS Fortran crash let fort84 = model_dir.join("fort.84"); if fort84.exists() { let _ = tokio::fs::remove_file(&fort84).await; } let chain = custom_chain.unwrap_or_else(|| default_chain_for_strategy(current_strategy)); let mut stage_summaries = Vec::new(); let mut current_seed: Option = seed_atmos.map(|p| p.to_path_buf()); let mut final_converged = false; let mut final_chmax: Option = None; let mut final_max_relc: Option = None; // 阶段独立配置(见 docs/task_engine_decoupling_design.md §5): // TLUSTY 关闭时跳过整个 chain 循环——current_seed 直接作为 final_7 来源, // 适配「仅 SYNSPEC」场景(用既有大气合成光谱,不重算大气结构)。 if tlusty_enabled { info!("TLUSTY 阶段启用:执行 {} 步收敛链", chain.len()); } else { info!("TLUSTY 阶段关闭:跳过大气结构计算,直接进入 SYNSPEC 阶段"); } let tlusty_skipped = !tlusty_enabled; for stage_def in &chain { if tlusty_skipped { break; } let stage_t0 = Instant::now(); let metals = stage_def.metals.as_deref().unwrap_or("cno"); let input5_text = make_input5( params, &stage_def.lte, &stage_def.ltgray, metals, stage_def.ilvlin, tlusty_input, ); let input5_path = model_dir.join(format!("{}.5", name)); tokio::fs::write(&input5_path, &input5_text).await?; // Write nst file let nst_text = generate_nst_content(stage_def, tlusty_input); tokio::fs::write(model_dir.join("nst"), &nst_text).await?; // Prepare fort.8 for this stage if stage_def.ltgray == "T" { if fort8.exists() { let _ = tokio::fs::remove_file(&fort8).await; } } else if let Some(ref s_path) = current_seed { if s_path.is_file() { if let Err(e) = tokio::fs::copy(s_path, &fort8).await { warn!( "阶段 {} 重载候选近邻推算种子模型期间发生文件复制异常: {}", stage_def.label, e ); } } } // Run tlusty.exe let fin = File::open(&input5_path).await?.into_std().await; let fout = File::create(model_dir.join(format!("{}.6", name))) .await? .into_std() .await; let ferr = File::create(model_dir.join(format!("{}.err", name))) .await? .into_std() .await; let child = AsyncCommand::new(&self.runtime.tlusty_exe) .current_dir(&model_dir) .stdin(Stdio::from(fin)) .stdout(Stdio::from(fout)) .stderr(Stdio::from(ferr)) .kill_on_drop(true) .spawn()?; let status_res = run_child_async_with_timeout(child, timeout_sec, shutdown.clone()).await; let rc = match status_res { Ok(st) => st.code().unwrap_or(-1), Err(e) => { warn!("tlusty 运行失败/超时: {}", e); -1 } }; let fort9 = model_dir.join("fort.9"); let fort7 = model_dir.join("fort.7"); let mut stage_summary = StepSummary { label: stage_def.label.clone(), chmax: stage_def.chmax, lte: stage_def.lte.clone(), converged: false, best_max_relc: None, elapsed_sec: stage_t0.elapsed().as_secs_f64(), note: None, last_iter: None, worst_depth: None, n_depths: None, itek_history: Vec::new(), }; if rc == 0 && fort7.is_file() { let eff_chmax = stage_def.chmax.unwrap_or(0.001); if fort9.is_file() { let res = check_fort9(&fort9, eff_chmax); stage_summary.converged = res.converged; stage_summary.best_max_relc = Some(res.max_relc); // 携带迭代诊断量进 conv.json(旧版在此处丢弃): // 迭代数/最差深度点供详情页阶段链展示收敛难度。 stage_summary.last_iter = res.last_iter; stage_summary.worst_depth = Some(res.worst_depth); stage_summary.n_depths = Some(res.n_depths); // itek 全量保真(Phase 5b):逐次迭代诊断随 summary_json/conv.json 落库。 stage_summary.itek_history = res.itek_history; // 漏洞5修复:发散时从 fort.6 提取求解器 STOP 行(SOLVE/SOLVES/RYBSOL) // 作为 note,提升归因质量。仅未收敛且无既有 note 时补(避免覆盖错误信息)。 if !res.converged && stage_summary.note.is_none() { let fort6 = model_dir.join(format!("{}.6", name)); if let Some(h) = extract_failure_hint(&fort6) { stage_summary.note = Some(format!("未收敛 [{}]", h)); } } // Save fort.9 snapshot let snap_name = format!("{}.{}_chmax{}.9", name, stage_def.label, eff_chmax); let _ = tokio::fs::copy(&fort9, model_dir.join(snap_name)).await; } else { // fort.9 缺失:按 stage_def.niter 区分两种场景(漏洞2进阶修复)。 // - niter==0:合法 grey start(lte 阶段不迭代,TLUSTY 不写 fort.9)。 // converged=true 保留 grey start 语义;best_max_relc=None 不虚构 // (避免污染 final_max_relc/种子选择)。 // - niter>0:异常——配了迭代却无 fort.9,通常是 TLUSTY 启动失败 // (call quit,如 temp 越界)或 IO 异常。判 converged=false, // 避免把崩溃误判为收敛。此前两种场景共用无校验分支无法区分。 if stage_def.niter == 0 { stage_summary.converged = true; stage_summary.best_max_relc = None; stage_summary.note = Some("NITER=0 grey start".to_string()); } else { stage_summary.converged = false; stage_summary.best_max_relc = None; // 补 fort.6 失败诊断(call quit 留言),便于排查启动失败原因。 let fort6 = model_dir.join(format!("{}.6", name)); let hint = extract_failure_hint(&fort6); stage_summary.note = Some(match hint { Some(h) => format!( "stage {} 配置 NITER={} 但 fort.9 缺失 [{}]", stage_def.label, stage_def.niter, h ), None => format!( "stage {} 配置 NITER={} 但 fort.9 缺失(TLUSTY 未完成迭代)", stage_def.label, stage_def.niter ), }); } } // Copy fort.7 as stage seed let stage_seed_path = model_dir.join(format!("{}.{}.7", name, stage_def.label)); let _ = tokio::fs::copy(&fort7, &stage_seed_path).await; current_seed = Some(stage_seed_path); } else { // 漏洞5修复:fort.7 缺失分支(输入错误、temp 越界等 call quit 场景), // 从 fort.6 尾部提取 call quit / stop 留言补进 note,便于排查。 let fort6 = model_dir.join(format!("{}.6", name)); let hint = extract_failure_hint(&fort6); stage_summary.note = Some(match hint { Some(h) => format!("tlusty rc={} or missing fort.7 [{}]", rc, h), None => format!("tlusty rc={} or missing fort.7", rc), }); } // 快照本阶段的同名输入/输出文件,带阶段标签保留。 // 背景:.5/.6/.err/nst 在每阶段用同名文件覆盖,若不快照则只有最后阶段(nl) // 的版本能存活到归档,nc 等中间阶段的日志/输入会丢失。失败阶段的日志对排错 // 尤其重要,因此此处无条件(不论 rc 是否为 0)快照。 // 命名风格与上方 .7/.9 快照一致:.