use crate::config::{ChainStep, SynspecInput, TlustyInput}; use crate::conv_check::{ atmosphere_has_nan, check_bfactor, check_convergence_trace, check_emflux_bolometric, check_energy_conservation, check_fort9, check_temperature_structure, extract_failure_hint, spec_is_valid, }; use crate::embedded::RuntimePaths; use crate::fort55_writer::generate_fort55_content; use crate::gen_input5::make_input5; use crate::models::{GridPointParams, ModelSummary, StepSummary}; use crate::nst_writer::generate_nst_content; use anyhow::Result; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::Instant; use tokio::fs::File; use tokio::process::Command as AsyncCommand; use tracing::{info, warn}; /// 阶梯/延拓阶段(Teff 自适应延拓、固定梯、丰度轴延拓)的迭代下限。 /// /// 这些阶段克隆自 nl 步(NITER=100),而慢收敛 waypoint 常在 chmax=0.001 门槛前 /// 耗尽迭代——生产实证(2026-08-21,t60000_g5.0_he-4_c-4_n-4_o-1):N 轴延拓 /// 在 n=-3.53 处 best=0.002/0.005 卡满 100 迭代且仍在单调下降(迭代饥饿而非 /// 发散),二分触底 0.025 dex 后被误判为轴折叠。TLUSTY 收敛即停,本上限只给 /// 慢阶段 3× 余量,不影响快阶段;真发散步会提前 STOP in SOLVE,也不受损。 const LADDER_STAGE_NITER: i32 = 300; pub fn default_cold_chain() -> Vec { vec![ ChainStep { label: "lte".to_string(), lte: "T".to_string(), ltgray: "T".to_string(), ilvlin: 0, require_converged: false, niter: 0, chmax: None, itek: None, ichang: None, idlte: None, iacc: None, orelax: None, dpsilg: None, popzer: None, }, ChainStep { label: "nc".to_string(), lte: "F".to_string(), ltgray: "F".to_string(), ilvlin: 0, require_converged: false, niter: 10, chmax: None, itek: None, ichang: None, idlte: None, iacc: None, orelax: None, dpsilg: None, popzer: None, }, ChainStep { label: "nl".to_string(), lte: "F".to_string(), ltgray: "F".to_string(), ilvlin: 100, require_converged: true, niter: 100, chmax: None, itek: None, ichang: None, idlte: None, iacc: None, orelax: None, dpsilg: None, popzer: None, }, ] } pub fn default_seed_chain() -> Vec { vec![ ChainStep { label: "seed_nc".to_string(), lte: "F".to_string(), ltgray: "F".to_string(), ilvlin: 0, require_converged: false, niter: 20, chmax: None, itek: None, ichang: Some(0), idlte: None, iacc: None, orelax: Some(0.3), dpsilg: None, popzer: None, }, ChainStep { label: "nl".to_string(), lte: "F".to_string(), ltgray: "F".to_string(), ilvlin: 100, require_converged: true, niter: 100, chmax: None, itek: None, ichang: Some(0), idlte: None, iacc: None, orelax: Some(0.5), dpsilg: None, popzer: None, }, ] } /// 稳定化种子步进链(策略 `seed_step_stab`)。 /// /// 面向难收敛角落(2026-08-18 实测 20kK He 富大气 He I/II 电离前沿布居极限环): /// 同物理族(同 Teff/logg/logHe)不同 CNO 的已收敛邻居种子 + POPZER 微布居置零 + /// DPSILG λ 算子欠松弛。代表点 t20000_g6.5_he2_c-4_n-3_o-4 其余配置组合全部发散, /// 本配方收敛且五重物理硬门全过(emflux 1.0013、首末比 4.5e5)。 pub fn default_seed_stab_chain() -> Vec { default_seed_chain() .into_iter() .map(|mut s| { s.dpsilg = Some(3.0); s.popzer = Some(1e-10); s }) .collect() } /// 按当前策略选默认执行链(`custom_chain` 为 None/空时的兜底)。 /// /// Phase 6(P8)起取代废弃的 task_type 匹配:`"seed_step"` → 种子热启动链, /// `"seed_step_stab"` → 稳定化种子链(POPZER+DPSILG),其余策略(`cold_run` 等) /// → 冷启动链。executor 现优先使用 TaskSpec.tlusty_chain_params /// (用户 YAML `tlusty_chain:` 配置),None/空才回退本函数的默认链。 pub fn default_chain_for_strategy(current_strategy: &str) -> Vec { if current_strategy == "seed_step" { default_seed_chain() } else if current_strategy == "seed_step_stab" { default_seed_stab_chain() } else { default_cold_chain() } } /// ladder 步进规划(纯函数):给定种子参数与目标参数,产出中间步坐标。 /// /// 规则(2026-08-17 实测验证:test/20260817_failed400/ladder_*): /// - 只沿与种子差值更大的轴步进(归一化:Δlogg/0.25 vs Δteff/2500); /// - 步长上限 Δlogg=0.25、ΔTeff=2500K,中间步数 ≤4; /// - 间隔已在单步收敛域内(Δlogg≤0.25 且 ΔTeff≤2500)→ 返回空(无需 ladder, /// 该场景本就不该触发); /// - 均匀切分:n = ceil(归一化间隔),每步走 gap/n。 /// /// 返回 (teff, logg, label) 列表,label 如 `ladder_g6.25` / `ladder_t57.5k`。 /// 目标步不在其中(由调用方用原 ChainStep 跑目标参数)。 pub fn plan_ladder_steps( from_teff: f64, from_logg: f64, to_teff: f64, to_logg: f64, ) -> Vec<(f64, f64, String)> { let d_logg = (to_logg - from_logg).abs(); let d_teff = (to_teff - from_teff).abs(); // 归一化到"单步上限"的单位数 let n_logg = (d_logg / 0.25).ceil() as usize; let n_teff = (d_teff / 2500.0).ceil() as usize; // 双轴都在单步收敛域内(含恰好等于上限)→ 无需 ladder(direct nl 即可覆盖) if n_logg <= 1 && n_teff <= 1 { return Vec::new(); } // 轴选择:需要更多步的轴(间隔更远 = 收敛域外的主因) let use_logg = n_logg >= n_teff; let n = if use_logg { n_logg } else { n_teff }.clamp(1, 4); (1..=n) .map(|k| { let f = k as f64 / n as f64; let (t, g) = if use_logg { (to_teff, from_logg + f * (to_logg - from_logg)) } else { (from_teff + f * (to_teff - from_teff), to_logg) }; let label = if use_logg { format!("ladder_g{}", (g * 100.0).round() / 100.0) } else { format!("ladder_t{}k", (t / 1000.0 * 10.0).round() / 10.0) }; (t, g, label) }) .collect() } /// 运行子进程,带超时与优雅退出(shutdown)感知。 /// /// 三种终止路径: /// 1. 子进程正常结束 → 返回 ExitStatus。 /// 2. 超时(timeout_sec)→ SIGKILL 子进程 + 二级 30s 等待 reap,超时则放弃 Child(kill_on_drop 兜底)。 /// 3. shutdown 信号(节点收到 SIGTERM/SIGINT)→ 立即 SIGKILL 子进程并快速返回 Err, /// 让上层尽快退出(在途任务的结果会丢失,由服务端 stale 重投兜底)。 /// /// 历史 bug:超时 kill 后 `child.wait().await` 无二级超时,Fortran 进程若卡死 /// (OpenMP hang / ptrace)会使 wait 永久阻塞,超时机制名存实亡、slot 永久泄漏。 async fn run_child_async_with_timeout( mut child: tokio::process::Child, timeout_sec: u64, shutdown: Option>, ) -> Result { let timeout_fut = tokio::time::timeout(tokio::time::Duration::from_secs(timeout_sec), child.wait()); // 若提供了 shutdown 标志,则与超时/正常结束三路 select;否则只等超时/正常结束。 let outcome: Result = if let Some(flag) = shutdown { let shutdown_watcher = async move { // 轮询 shutdown 标志(10ms 粒度足够灵敏,开销可忽略)。 loop { if flag.load(std::sync::atomic::Ordering::Acquire) { return; } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } }; tokio::select! { biased; // 优先响应 shutdown _ = shutdown_watcher => Err(ShutdownOrTimeout::Shutdown), r = timeout_fut => match r { Ok(res) => Ok(res?), Err(_) => Err(ShutdownOrTimeout::Timeout), }, } } else { match timeout_fut.await { Ok(res) => Ok(res?), Err(_) => Err(ShutdownOrTimeout::Timeout), } }; match outcome { Ok(status) => Ok(status), Err(ShutdownOrTimeout::Shutdown) => { let _ = child.start_kill(); let _ = tokio::time::timeout(std::time::Duration::from_secs(30), child.wait()).await; anyhow::bail!("节点收到退出信号,子进程已被终止"); } Err(ShutdownOrTimeout::Timeout) => { let _ = child.start_kill(); let _ = tokio::time::timeout(std::time::Duration::from_secs(30), child.wait()).await; anyhow::bail!("进程计算超时 (上限: {} 秒)", timeout_sec); } } } #[derive(Debug)] enum ShutdownOrTimeout { Shutdown, Timeout, } /// 快照 TLUSTY 最终模型的 b 因子与出射谱,防止被 SYNSPEC 覆盖丢失。 /// /// TLUSTY 在最终迭代(`LFIN=.TRUE.`)经 `OUTPRI` 写出(见 tlusty208.f): /// - `fort.12`:b 因子 / 非 LTE 偏离因子表(头 2I5 + 每深度 TEMP/ELEC/DENS/BFAC,格式 701/702/703)。 /// 随后 SYNSPEC 会复用 unit 12 写谱线证认表并覆盖它(runner 再将其存为 `.iden`), /// 故 TLUSTY 的 b 因子若不在此快照即静默丢失。 /// - `fort.14`:出射谱(波长 Å + Fλ,格式 614),同样会被 SYNSPEC 的谱线数据覆盖。 /// /// 在收敛链循环结束(链上最后一次 TLUSTY 运行即最终模型)、SYNSPEC 启动前调用, /// 快照为 `.bfac` / `.emflux`,与科学核心产物一并进入归档白名单 /// (见 `result_filter::is_result_worthy` 的 `bfac`/`emflux` 后缀)。 /// 文件不存在时静默跳过(TLUSTY 未运行/未写出);IO 错误降级为 warn,不阻断主流程。 async fn snapshot_tlusty_outputs(model_dir: &Path, name: &str) { for (src, suffix) in [("fort.12", "bfac"), ("fort.14", "emflux")] { let src_path = model_dir.join(src); if !src_path.is_file() { continue; } let dst = model_dir.join(format!("{}.{}", name, suffix)); if let Err(e) = tokio::fs::copy(&src_path, &dst).await { warn!("快照 TLUSTY {} 到 {} 失败: {}", src, dst.display(), e); } } } pub struct ExecutionRunner<'a> { pub runtime: &'a RuntimePaths, pub work_dir: PathBuf, } impl<'a> ExecutionRunner<'a> { pub fn new(runtime: &'a RuntimePaths, work_dir: PathBuf) -> Self { Self { runtime, work_dir } } #[allow(clippy::too_many_arguments)] // 透传全参给 run_model_with_timeout(后者同 allow) pub async fn run_model( &self, params: &GridPointParams, name: &str, current_strategy: &str, custom_chain: Option>, seed_atmos: Option<&Path>, seed_params: Option<&GridPointParams>, synspec_cfg: Option<&SynspecInput>, tlusty_input: Option<&TlustyInput>, energy_tolerance: Option, temp_max_factor: Option, temp_floor: Option, temp_ceiling: Option, emflux_tolerance: Option, convergence_min_ratio: Option, bfac_max: Option, bfac_min: Option, ) -> Result { self.run_model_with_timeout( params, name, current_strategy, custom_chain, seed_atmos, seed_params, synspec_cfg, true, true, 7200, None, tlusty_input, energy_tolerance, temp_max_factor, temp_floor, temp_ceiling, emflux_tolerance, convergence_min_ratio, bfac_max, bfac_min, ) .await } /// 执行收敛链中的单个 TLUSTY 阶段(2026-08-14 从 run_model_with_timeout 的 /// 阶段循环体抽取,供主链与 nl_direct 回退共用)。 /// /// 职责:写 .5/nst 输入 → 按 seed 铺 fort.8 → 运行 tlusty → fort.9 收敛诊断 /// (含假收敛排查与 STOP 留言提取)→ 快照阶段产物(.5/.6/.err/nst/_chmax*.9)。 /// /// 返回 `(StepSummary, 产出的阶段种子路径)`:rc==0 且 fort.7 存在时, /// fort.7 被快照为 `.