418 lines
15 KiB
Rust
418 lines
15 KiB
Rust
use crate::config::{StageConfig, SynspecConfig};
|
|
use crate::conv_check::{atmosphere_has_nan, check_fort9};
|
|
use crate::embedded::RuntimePaths;
|
|
use crate::fort55_writer::generate_fort55_content;
|
|
use crate::gen_input5::make_input5;
|
|
use crate::models::{GridPointParams, ModelSummary, StageSummary, TaskType};
|
|
use crate::nst_writer::generate_nst_content;
|
|
use anyhow::Result;
|
|
use tokio::fs::File;
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::Stdio;
|
|
use tokio::process::Command as AsyncCommand;
|
|
use std::time::Instant;
|
|
use tracing::{info, warn};
|
|
|
|
pub fn default_cold_chain() -> Vec<StageConfig> {
|
|
vec![
|
|
StageConfig {
|
|
label: "lte".to_string(),
|
|
lte: "T".to_string(),
|
|
ltgray: "T".to_string(),
|
|
ilvlin: 0,
|
|
require_converged: false,
|
|
niter: 0,
|
|
chmax: None,
|
|
itek: None,
|
|
metals: Some("cno".to_string()),
|
|
ichang: None,
|
|
idlte: None,
|
|
iacc: None,
|
|
orelax: None,
|
|
},
|
|
StageConfig {
|
|
label: "nc".to_string(),
|
|
lte: "F".to_string(),
|
|
ltgray: "F".to_string(),
|
|
ilvlin: 0,
|
|
require_converged: false,
|
|
niter: 10,
|
|
chmax: None,
|
|
itek: None,
|
|
metals: Some("cno".to_string()),
|
|
ichang: None,
|
|
idlte: None,
|
|
iacc: None,
|
|
orelax: None,
|
|
},
|
|
StageConfig {
|
|
label: "nl".to_string(),
|
|
lte: "F".to_string(),
|
|
ltgray: "F".to_string(),
|
|
ilvlin: 100,
|
|
require_converged: true,
|
|
niter: 100,
|
|
chmax: None,
|
|
itek: None,
|
|
metals: Some("cno".to_string()),
|
|
ichang: None,
|
|
idlte: None,
|
|
iacc: None,
|
|
orelax: None,
|
|
},
|
|
]
|
|
}
|
|
|
|
pub fn default_seed_chain() -> Vec<StageConfig> {
|
|
vec![
|
|
StageConfig {
|
|
label: "seed_nc".to_string(),
|
|
lte: "F".to_string(),
|
|
ltgray: "F".to_string(),
|
|
ilvlin: 0,
|
|
require_converged: false,
|
|
niter: 20,
|
|
chmax: None,
|
|
itek: None,
|
|
metals: Some("cno".to_string()),
|
|
ichang: Some(0),
|
|
idlte: None,
|
|
iacc: None,
|
|
orelax: None,
|
|
},
|
|
StageConfig {
|
|
label: "nl".to_string(),
|
|
lte: "F".to_string(),
|
|
ltgray: "F".to_string(),
|
|
ilvlin: 100,
|
|
require_converged: true,
|
|
niter: 100,
|
|
chmax: None,
|
|
itek: None,
|
|
metals: Some("cno".to_string()),
|
|
ichang: Some(0),
|
|
idlte: None,
|
|
iacc: None,
|
|
orelax: None,
|
|
},
|
|
]
|
|
}
|
|
|
|
async fn run_child_async_with_timeout(
|
|
mut child: tokio::process::Child,
|
|
timeout_sec: u64,
|
|
) -> Result<std::process::ExitStatus> {
|
|
match tokio::time::timeout(tokio::time::Duration::from_secs(timeout_sec), child.wait()).await {
|
|
Ok(res) => Ok(res?),
|
|
Err(_) => {
|
|
let _ = child.start_kill();
|
|
let _ = child.wait().await;
|
|
anyhow::bail!("进程计算超时 (上限: {} 秒)", timeout_sec);
|
|
}
|
|
}
|
|
}
|
|
|
|
pub struct ExecutionRunner<'a> {
|
|
pub runtime: &'a RuntimePaths,
|
|
pub work_dir: PathBuf,
|
|
}
|
|
|
|
impl<'a> ExecutionRunner<'a> {
|
|
pub fn new(runtime: &'a RuntimePaths, work_dir: PathBuf) -> Self {
|
|
Self { runtime, work_dir }
|
|
}
|
|
|
|
pub async fn run_model(
|
|
&self,
|
|
params: &GridPointParams,
|
|
task_type: TaskType,
|
|
custom_chain: Option<Vec<StageConfig>>,
|
|
seed_atmos: Option<&Path>,
|
|
synspec_cfg: Option<&SynspecConfig>,
|
|
) -> Result<ModelSummary> {
|
|
self.run_model_with_timeout(params, task_type, custom_chain, seed_atmos, synspec_cfg, 7200).await
|
|
}
|
|
|
|
pub async fn run_model_with_timeout(
|
|
&self,
|
|
params: &GridPointParams,
|
|
task_type: TaskType,
|
|
custom_chain: Option<Vec<StageConfig>>,
|
|
seed_atmos: Option<&Path>,
|
|
synspec_cfg: Option<&SynspecConfig>,
|
|
timeout_sec: u64,
|
|
) -> Result<ModelSummary> {
|
|
let name = params.model_name();
|
|
let model_dir = self.work_dir.join(&name);
|
|
tokio::fs::create_dir_all(&model_dir).await?;
|
|
|
|
info!("开始物理计算网格模型 {} (类型: {:?})", name, task_type);
|
|
let t0 = Instant::now();
|
|
|
|
// 1. Data directory symlink setup
|
|
let link_data = model_dir.join("data");
|
|
if tokio::fs::symlink_metadata(&link_data).await.is_ok() || link_data.exists() {
|
|
let _ = tokio::fs::remove_file(&link_data).await;
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
{
|
|
let abs_data_dir = tokio::fs::canonicalize(&self.runtime.data_dir).await.unwrap_or_else(|_| self.runtime.data_dir.clone());
|
|
if let Err(e) = std::os::unix::fs::symlink(&abs_data_dir, &link_data) {
|
|
warn!("构建 data 数据集软链时发生提示性告警: {}", e);
|
|
}
|
|
}
|
|
|
|
// 2. Initial fort.8 seed setup
|
|
let fort8 = model_dir.join("fort.8");
|
|
if fort8.exists() {
|
|
let _ = tokio::fs::remove_file(&fort8).await;
|
|
}
|
|
if let Some(seed_path) = seed_atmos {
|
|
if seed_path.is_file() {
|
|
if let Err(e) = tokio::fs::copy(seed_path, &fort8).await {
|
|
warn!("向工作沙盒引导填载首期收敛模型种子 fort.8 发生复制错误: {}", e);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Clean fort.84 residue to prevent NATOMS Fortran crash
|
|
let fort84 = model_dir.join("fort.84");
|
|
if fort84.exists() {
|
|
let _ = tokio::fs::remove_file(&fort84).await;
|
|
}
|
|
|
|
let chain = custom_chain.unwrap_or_else(|| match task_type {
|
|
TaskType::ColdRun => default_cold_chain(),
|
|
TaskType::SeedStep => default_seed_chain(),
|
|
});
|
|
|
|
let mut stage_summaries = Vec::new();
|
|
let mut current_seed: Option<PathBuf> = seed_atmos.map(|p| p.to_path_buf());
|
|
let mut final_converged = false;
|
|
let mut final_chmax: Option<f64> = None;
|
|
let mut final_max_relc: Option<f64> = None;
|
|
|
|
for stage_def in &chain {
|
|
let stage_t0 = Instant::now();
|
|
let metals = stage_def.metals.as_deref().unwrap_or("cno");
|
|
let input5_text = make_input5(
|
|
params,
|
|
&stage_def.lte,
|
|
&stage_def.ltgray,
|
|
metals,
|
|
stage_def.ilvlin,
|
|
);
|
|
|
|
let input5_path = model_dir.join(format!("{}.5", name));
|
|
tokio::fs::write(&input5_path, &input5_text).await?;
|
|
|
|
// Write nst file
|
|
let nst_text = generate_nst_content(stage_def);
|
|
tokio::fs::write(model_dir.join("nst"), &nst_text).await?;
|
|
|
|
// Prepare fort.8 for this stage
|
|
if stage_def.ltgray == "T" {
|
|
if fort8.exists() {
|
|
let _ = tokio::fs::remove_file(&fort8).await;
|
|
}
|
|
} else if let Some(ref s_path) = current_seed {
|
|
if s_path.is_file() {
|
|
if let Err(e) = tokio::fs::copy(s_path, &fort8).await {
|
|
warn!("阶段 {} 重载候选近邻推算种子模型期间发生文件复制异常: {}", stage_def.label, e);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Run tlusty.exe
|
|
let fin = File::open(&input5_path).await?.into_std().await;
|
|
let fout = File::create(model_dir.join(format!("{}.6", name))).await?.into_std().await;
|
|
let ferr = File::create(model_dir.join(format!("{}.err", name))).await?.into_std().await;
|
|
|
|
let child = AsyncCommand::new(&self.runtime.tlusty_exe)
|
|
.current_dir(&model_dir)
|
|
.stdin(Stdio::from(fin))
|
|
.stdout(Stdio::from(fout))
|
|
.stderr(Stdio::from(ferr))
|
|
.kill_on_drop(true)
|
|
.spawn()?;
|
|
|
|
let status_res = run_child_async_with_timeout(child, timeout_sec).await;
|
|
let rc = match status_res {
|
|
Ok(st) => st.code().unwrap_or(-1),
|
|
Err(e) => {
|
|
warn!("tlusty 运行失败/超时: {}", e);
|
|
-1
|
|
}
|
|
};
|
|
|
|
|
|
let fort9 = model_dir.join("fort.9");
|
|
let fort7 = model_dir.join("fort.7");
|
|
|
|
let mut stage_summary = StageSummary {
|
|
label: stage_def.label.clone(),
|
|
chmax: stage_def.chmax,
|
|
lte: stage_def.lte.clone(),
|
|
converged: false,
|
|
best_max_relc: None,
|
|
elapsed_sec: stage_t0.elapsed().as_secs_f64(),
|
|
note: None,
|
|
};
|
|
|
|
if rc == 0 && fort7.is_file() {
|
|
let eff_chmax = stage_def.chmax.unwrap_or(0.001);
|
|
if fort9.is_file() {
|
|
let res = check_fort9(&fort9, eff_chmax);
|
|
stage_summary.converged = res.converged;
|
|
stage_summary.best_max_relc = Some(res.max_relc);
|
|
|
|
// Save fort.9 snapshot
|
|
let snap_name = format!("{}.{}_chmax{}.9", name, stage_def.label, eff_chmax);
|
|
let _ = tokio::fs::copy(&fort9, model_dir.join(snap_name)).await;
|
|
} else {
|
|
// NITER=0 grey start without fort.9
|
|
stage_summary.converged = true;
|
|
stage_summary.best_max_relc = Some(0.0);
|
|
stage_summary.note = Some("NITER=0 grey start".to_string());
|
|
}
|
|
|
|
// Copy fort.7 as stage seed
|
|
let stage_seed_path = model_dir.join(format!("{}.{}.7", name, stage_def.label));
|
|
let _ = tokio::fs::copy(&fort7, &stage_seed_path).await;
|
|
current_seed = Some(stage_seed_path);
|
|
} else {
|
|
stage_summary.note = Some(format!("tlusty rc={} or missing fort.7", rc));
|
|
}
|
|
|
|
final_chmax = stage_def.chmax;
|
|
final_converged = stage_summary.converged;
|
|
if let Some(r) = stage_summary.best_max_relc {
|
|
final_max_relc = Some(r);
|
|
}
|
|
stage_summaries.push(stage_summary);
|
|
|
|
if !final_converged && stage_def.require_converged {
|
|
warn!("阶段 {} 要求收敛但未达标，中止后续收敛链阶段", stage_def.label);
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Final atmosphere file .7
|
|
let final_7 = model_dir.join(format!("{}.7", name));
|
|
if let Some(ref s_path) = current_seed {
|
|
if s_path.is_file() {
|
|
let _ = tokio::fs::copy(s_path, &final_7).await;
|
|
}
|
|
} else if model_dir.join("fort.7").is_file() {
|
|
let _ = tokio::fs::copy(model_dir.join("fort.7"), &final_7).await;
|
|
}
|
|
|
|
let atmo_has_nan = atmosphere_has_nan(&final_7);
|
|
if atmo_has_nan {
|
|
final_converged = false;
|
|
}
|
|
|
|
// Run synspec if final .7 atmosphere exists
|
|
let mut synspec_rc = None;
|
|
let mut synspec_err = None;
|
|
let mut synspec_sec = None;
|
|
|
|
if final_7.is_file() {
|
|
let syn_t0 = Instant::now();
|
|
let _ = tokio::fs::copy(&final_7, model_dir.join("fort.8")).await;
|
|
let _ = tokio::fs::remove_file(model_dir.join("fort.7")).await;
|
|
|
|
// Fort.55 parameter generation or symlink
|
|
let fort55_path = model_dir.join("fort.55");
|
|
let fort19_path = model_dir.join("fort.19");
|
|
|
|
let _ = tokio::fs::remove_file(&fort55_path).await;
|
|
let _ = tokio::fs::remove_file(&fort19_path).await;
|
|
|
|
let default_cfg = SynspecConfig {
|
|
wstart: 1400.0,
|
|
wend: 1410.0,
|
|
imode: 0,
|
|
idrv: 50,
|
|
ifreq: 1,
|
|
rel_cutoff: 0.0001,
|
|
abs_cutoff: 0.01,
|
|
};
|
|
let fort55_text = generate_fort55_content(synspec_cfg.unwrap_or(&default_cfg));
|
|
let _ = tokio::fs::write(&fort55_path, &fort55_text).await;
|
|
|
|
#[cfg(unix)]
|
|
{
|
|
let abs_linelist = tokio::fs::canonicalize(&self.runtime.linelist).await.unwrap_or_else(|_| self.runtime.linelist.clone());
|
|
let _ = std::os::unix::fs::symlink(&abs_linelist, &fort19_path);
|
|
}
|
|
|
|
let input5_path = model_dir.join(format!("{}.5", name));
|
|
if input5_path.is_file() {
|
|
let fin = File::open(&input5_path).await?.into_std().await;
|
|
let fout = File::create(model_dir.join(format!("{}.log", name))).await?.into_std().await;
|
|
|
|
let child = AsyncCommand::new(&self.runtime.synspec_exe)
|
|
.current_dir(&model_dir)
|
|
.stdin(Stdio::from(fin))
|
|
.stdout(Stdio::from(fout))
|
|
.stderr(Stdio::null())
|
|
.kill_on_drop(true)
|
|
.spawn()?;
|
|
|
|
let status_res = run_child_async_with_timeout(child, timeout_sec).await;
|
|
let rc = match status_res {
|
|
Ok(st) => st.code().unwrap_or(-1),
|
|
Err(e) => {
|
|
warn!("synspec 运行失败/超时: {}", e);
|
|
-1
|
|
}
|
|
};
|
|
synspec_rc = Some(rc);
|
|
synspec_sec = Some(syn_t0.elapsed().as_secs_f64());
|
|
|
|
// Copy/move outputs: fort.7 (Synspec spectrum) -> .spec, fort.17 -> .cont, fort.12 -> .iden
|
|
if model_dir.join("fort.7").is_file() {
|
|
let _ = tokio::fs::rename(model_dir.join("fort.7"), model_dir.join(format!("{}.spec", name))).await;
|
|
}
|
|
if model_dir.join("fort.17").is_file() {
|
|
let _ = tokio::fs::copy(model_dir.join("fort.17"), model_dir.join(format!("{}.cont", name))).await;
|
|
}
|
|
if model_dir.join("fort.12").is_file() {
|
|
let _ = tokio::fs::copy(model_dir.join("fort.12"), model_dir.join(format!("{}.iden", name))).await;
|
|
}
|
|
}
|
|
} else {
|
|
synspec_err = Some("No atmosphere .7 produced".to_string());
|
|
}
|
|
|
|
let elapsed_sec = t0.elapsed().as_secs_f64();
|
|
let summary = ModelSummary {
|
|
name,
|
|
params: params.clone(),
|
|
stages: stage_summaries,
|
|
converged: final_converged,
|
|
final_max_relc,
|
|
final_chmax,
|
|
seed: seed_atmos.map(|p| p.to_string_lossy().to_string()),
|
|
atmosphere_has_nan: atmo_has_nan,
|
|
synspec_rc,
|
|
synspec_error: synspec_err,
|
|
synspec_sec,
|
|
elapsed_sec,
|
|
note: if atmo_has_nan {
|
|
Some("Invalidated: atmosphere contains >10% NaN lines".to_string())
|
|
} else {
|
|
None
|
|
},
|
|
};
|
|
|
|
// Write conv.json
|
|
let json_text = serde_json::to_string_pretty(&summary)?;
|
|
tokio::fs::write(model_dir.join("conv.json"), json_text).await?;
|
|
|
|
Ok(summary)
|
|
}
|
|
}
|