feat(all): 科学计算正确性修复、调度竞态消除、节点优雅退出、安全加固与收敛分析重构
科学计算正确性: - 修复 Fortran 无-E 科学记数法(指数≥100 时 E 被挤掉,如 -1.35+118)导致 发散行被静默跳过、误判收敛的 bug;扩展大气无效检测覆盖 Inf 与 *** 溢出标记 - 种子匹配改为 CNO 有向距离(富金属方向重罚 4×、贫金属方向轻罚 1×), 基于 1191 个真实种子配对回测标定,回测净改善 314 个点 - GridAxisValue 反序列化拒绝非法文本(不再静默 NaN);chmax≤0 显式报错 - ions 行宽列宽对齐真实 fort.5 格式 调度与队列竞态: - 原子选点(IMMEDIATE 事务 SELECT+UPDATE)消除并发调度重复派发 (#5) - 调度互斥锁 + 冷启动优先策略(SeedStep 仅作失败后救援,不再正常路径热启动) - 毒消息 dead_letter 标记防出队死循环;clear_queue 保留 claimed 行 (#6) - 孤儿 running 点回收兜底;stale_sec 默认 7800→21600s(3 倍超时缓冲) 节点生命周期: - SIGTERM+SIGINT 双信号监听(修复 Docker stop 发 SIGTERM 不触发优雅退出) - SlotGuard RAII 防活动 slot 泄漏;子进程超时增加二级 30s wait 防 Fortran hang - SeedStep 种子下载 fail-fast + 沙盒私有副本解耦 LRU 清理竞争 - reqwest Client 增加连接/请求超时;启动清理残留 task_* 沙盒 安全加固: - 节点注册 registration_secret 二次凭据 (H8),恒定时间比对防时序旁路 - token 缓存 generation 机制消除 reissue 后旧 token TOCTOU 复活窗口 - 新增 /api/auth/logout 服务端 session 即时撤销;fail-closed 鉴权启动策略 - 前端 token 迁移 sessionStorage;YAML 高亮改 DOM API 消除 XSS 注入面 - 备份文件权限收紧 0600;点表动态值全面 escapeHtml 前端 Dashboard: - 收敛性分析从热力图重构为 Parallel Sets 平行集合图(6 维+状态轴,手写 SVG 零依赖) - 进度曲线横轴改为真实时间(服务端 now 锚定,停滞期诚实留白);轮询指数退避 - 移除 imported 收敛途径分类,导入点按实际途径 cold_run/seed_step 归类 - 初始化时 /api/auth/check 校验 token;401 toast 提示替代静默 reload 服务端恢复与工具链: - 启动恢复 initializing 态工作流;默认工作流 INSERT-only 不覆盖 API 编辑 - body limit 分层(10MB 不再截断 256MB report);multipart 显式错误处理 - 嵌入二进制原子写(tmp+rename)防半写损坏 - import_results 判定收敛途径透传 success_method;conv.json 格式对齐本项目 - push_import_results.sh 退出码修复 + .bat UTF-8 BOM + scp 上传 - Docker USE_MIRRORS 默认关闭;移除无用 assets 挂载;删除 hosts.ini 入库
This commit is contained in:
+111
-40
@@ -98,20 +98,81 @@ pub fn default_seed_chain() -> Vec<StageConfig> {
|
||||
]
|
||||
}
|
||||
|
||||
/// 运行子进程,带超时与优雅退出(shutdown)感知。
|
||||
///
|
||||
/// 三种终止路径:
|
||||
/// 1. 子进程正常结束 → 返回 ExitStatus。
|
||||
/// 2. 超时(timeout_sec)→ SIGKILL 子进程 + 二级 30s 等待 reap,超时则放弃 Child(kill_on_drop 兜底)。
|
||||
/// 3. shutdown 信号(节点收到 SIGTERM/SIGINT)→ 立即 SIGKILL 子进程并快速返回 Err,
|
||||
/// 让上层尽快退出(在途任务的结果会丢失,由服务端 stale 重投兜底)。
|
||||
///
|
||||
/// 历史 bug:超时 kill 后 `child.wait().await` 无二级超时,Fortran 进程若卡死
|
||||
/// (OpenMP hang / ptrace)会使 wait 永久阻塞,超时机制名存实亡、slot 永久泄漏。
|
||||
async fn run_child_async_with_timeout(
|
||||
mut child: tokio::process::Child,
|
||||
timeout_sec: u64,
|
||||
shutdown: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
|
||||
) -> Result<std::process::ExitStatus> {
|
||||
match tokio::time::timeout(tokio::time::Duration::from_secs(timeout_sec), child.wait()).await {
|
||||
Ok(res) => Ok(res?),
|
||||
Err(_) => {
|
||||
let timeout_fut = tokio::time::timeout(
|
||||
tokio::time::Duration::from_secs(timeout_sec),
|
||||
child.wait(),
|
||||
);
|
||||
|
||||
// 若提供了 shutdown 标志,则与超时/正常结束三路 select;否则只等超时/正常结束。
|
||||
let outcome: Result<std::process::ExitStatus, ShutdownOrTimeout> = if let Some(flag) = shutdown {
|
||||
let shutdown_watcher = async move {
|
||||
// 轮询 shutdown 标志(10ms 粒度足够灵敏,开销可忽略)。
|
||||
loop {
|
||||
if flag.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
};
|
||||
tokio::select! {
|
||||
biased; // 优先响应 shutdown
|
||||
_ = shutdown_watcher => Err(ShutdownOrTimeout::Shutdown),
|
||||
r = timeout_fut => match r {
|
||||
Ok(res) => Ok(res?),
|
||||
Err(_) => Err(ShutdownOrTimeout::Timeout),
|
||||
},
|
||||
}
|
||||
} else {
|
||||
match timeout_fut.await {
|
||||
Ok(res) => Ok(res?),
|
||||
Err(_) => Err(ShutdownOrTimeout::Timeout),
|
||||
}
|
||||
};
|
||||
|
||||
match outcome {
|
||||
Ok(status) => Ok(status),
|
||||
Err(ShutdownOrTimeout::Shutdown) => {
|
||||
let _ = child.start_kill();
|
||||
let _ = child.wait().await;
|
||||
let _ = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
child.wait(),
|
||||
)
|
||||
.await;
|
||||
anyhow::bail!("节点收到退出信号,子进程已被终止");
|
||||
}
|
||||
Err(ShutdownOrTimeout::Timeout) => {
|
||||
let _ = child.start_kill();
|
||||
let _ = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
child.wait(),
|
||||
)
|
||||
.await;
|
||||
anyhow::bail!("进程计算超时 (上限: {} 秒)", timeout_sec);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum ShutdownOrTimeout {
|
||||
Shutdown,
|
||||
Timeout,
|
||||
}
|
||||
|
||||
pub struct ExecutionRunner<'a> {
|
||||
pub runtime: &'a RuntimePaths,
|
||||
pub work_dir: PathBuf,
|
||||
@@ -139,6 +200,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
seed_atmos,
|
||||
synspec_cfg,
|
||||
7200,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -153,6 +215,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
seed_atmos: Option<&Path>,
|
||||
synspec_cfg: Option<&SynspecConfig>,
|
||||
timeout_sec: u64,
|
||||
shutdown: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
|
||||
) -> Result<ModelSummary> {
|
||||
// `name` 取自权威的 TaskSpec.point_name(DB 的 grid_points.name 列,源精度正确),
|
||||
// 而非 params.model_name()。原因:服务端把 GridPointParams 存成 6 个 REAL 数值列,
|
||||
@@ -160,14 +223,10 @@ impl<'a> ExecutionRunner<'a> {
|
||||
// 产出错误名(g5 而非 g5.0)。point_name 走独立 TEXT 列,精度全程保留。
|
||||
// 下游(沙盒子目录、各阶段快照、conv.json.name、归档目录)全部用此 name,
|
||||
// 故只需在此处用权威 name 即可让整条链精度正确。
|
||||
let derived = params.model_name();
|
||||
if derived != name {
|
||||
warn!(
|
||||
"网格点权威名 {} 与 params 重推名 {} 不一致(DB REAL 列回读丢精度所致),\
|
||||
采用权威 point_name",
|
||||
name, derived
|
||||
);
|
||||
}
|
||||
//
|
||||
// 历史:此处曾把 params.model_name() 与 name 对比并 warn 不一致。但该不一致是
|
||||
// DB REAL 列回读丢精度的已知现象(runner 端无法修复,根治需改 DB schema 存原文),
|
||||
// 且 runner 已全程采用权威 name,对比结果不参与任何决策——故移除这段噪音 warn。
|
||||
let model_dir = self.work_dir.join(name);
|
||||
tokio::fs::create_dir_all(&model_dir).await?;
|
||||
|
||||
@@ -276,7 +335,7 @@ impl<'a> ExecutionRunner<'a> {
|
||||
.kill_on_drop(true)
|
||||
.spawn()?;
|
||||
|
||||
let status_res = run_child_async_with_timeout(child, timeout_sec).await;
|
||||
let status_res = run_child_async_with_timeout(child, timeout_sec, shutdown.clone()).await;
|
||||
let rc = match status_res {
|
||||
Ok(st) => st.code().unwrap_or(-1),
|
||||
Err(e) => {
|
||||
@@ -393,38 +452,49 @@ impl<'a> ExecutionRunner<'a> {
|
||||
|
||||
if final_7.is_file() {
|
||||
let syn_t0 = Instant::now();
|
||||
let _ = tokio::fs::copy(&final_7, model_dir.join("fort.8")).await;
|
||||
let _ = tokio::fs::remove_file(model_dir.join("fort.7")).await;
|
||||
// H10:synspec 输入文件(fort.8 大气 / fort.55 控制卡)写入失败不可静默吞掉。
|
||||
// 历史上用 `let _ =` 忽略错误,磁盘满/inode 耗尽时 synspec 会读到旧/缺失的
|
||||
// fort.8 产出垃圾光谱,却仍生成 .spec 并被归档为"成功"。现改为写入失败即记
|
||||
// synspec_err 并跳过 synspec 阶段,避免产出物理上错误的谱。
|
||||
if let Err(e) = tokio::fs::copy(&final_7, model_dir.join("fort.8")).await {
|
||||
warn!("synspec 输入 fort.8 (大气) 复制失败,跳过 synspec: {}", e);
|
||||
synspec_err = Some(format!("fort.8 copy failed: {}", e));
|
||||
} else {
|
||||
let _ = tokio::fs::remove_file(model_dir.join("fort.7")).await;
|
||||
|
||||
// Fort.55 parameter generation or symlink
|
||||
let fort55_path = model_dir.join("fort.55");
|
||||
let fort19_path = model_dir.join("fort.19");
|
||||
// Fort.55 parameter generation or symlink
|
||||
let fort55_path = model_dir.join("fort.55");
|
||||
let fort19_path = model_dir.join("fort.19");
|
||||
|
||||
let _ = tokio::fs::remove_file(&fort55_path).await;
|
||||
let _ = tokio::fs::remove_file(&fort19_path).await;
|
||||
let _ = tokio::fs::remove_file(&fort55_path).await;
|
||||
let _ = tokio::fs::remove_file(&fort19_path).await;
|
||||
|
||||
let default_cfg = SynspecConfig {
|
||||
wstart: 1400.0,
|
||||
wend: 1410.0,
|
||||
imode: 0,
|
||||
idrv: 50,
|
||||
ifreq: 1,
|
||||
rel_cutoff: 0.0001,
|
||||
abs_cutoff: 0.01,
|
||||
};
|
||||
let fort55_text = generate_fort55_content(synspec_cfg.unwrap_or(&default_cfg));
|
||||
let _ = tokio::fs::write(&fort55_path, &fort55_text).await;
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let abs_linelist = tokio::fs::canonicalize(&self.runtime.linelist)
|
||||
.await
|
||||
.unwrap_or_else(|_| self.runtime.linelist.clone());
|
||||
let _ = std::os::unix::fs::symlink(&abs_linelist, &fort19_path);
|
||||
let default_cfg = SynspecConfig {
|
||||
wstart: 1400.0,
|
||||
wend: 1410.0,
|
||||
imode: 0,
|
||||
idrv: 50,
|
||||
ifreq: 1,
|
||||
rel_cutoff: 0.0001,
|
||||
abs_cutoff: 0.01,
|
||||
};
|
||||
let fort55_text = generate_fort55_content(synspec_cfg.unwrap_or(&default_cfg));
|
||||
if let Err(e) = tokio::fs::write(&fort55_path, &fort55_text).await {
|
||||
warn!("synspec 输入 fort.55 (控制卡) 写入失败,跳过 synspec: {}", e);
|
||||
synspec_err = Some(format!("fort.55 write failed: {}", e));
|
||||
} else {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let abs_linelist = tokio::fs::canonicalize(&self.runtime.linelist)
|
||||
.await
|
||||
.unwrap_or_else(|_| self.runtime.linelist.clone());
|
||||
let _ = std::os::unix::fs::symlink(&abs_linelist, &fort19_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let input5_path = model_dir.join(format!("{}.5", name));
|
||||
if input5_path.is_file() {
|
||||
if synspec_err.is_none() && input5_path.is_file() {
|
||||
let fin = File::open(&input5_path).await?.into_std().await;
|
||||
let fout = File::create(model_dir.join(format!("{}.log", name)))
|
||||
.await?
|
||||
@@ -440,7 +510,8 @@ impl<'a> ExecutionRunner<'a> {
|
||||
.spawn()?;
|
||||
|
||||
let synspec_timeout_sec = 600_u64.min(timeout_sec);
|
||||
let status_res = run_child_async_with_timeout(child, synspec_timeout_sec).await;
|
||||
let status_res =
|
||||
run_child_async_with_timeout(child, synspec_timeout_sec, shutdown.clone()).await;
|
||||
let rc = match status_res {
|
||||
Ok(st) => st.code().unwrap_or(-1),
|
||||
Err(e) => {
|
||||
|
||||
Reference in New Issue
Block a user