feat(all): 重炼 crates/common 核心组件、上线 Web 运维看板与 Docker 容器化部署

2026-07-28 10:31:57 +08:00
commit 4b4238d702
71 changed files with 163402 additions and 0 deletions
@@ -0,0 +1,21 @@
+[package]
+name = "node"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+common = { path = "../common" }
+mq = { path = "../mq" }
+tokio.workspace = true
+reqwest.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+tracing.workspace = true
+tracing-subscriber.workspace = true
+anyhow.workspace = true
+sysinfo.workspace = true
+gethostname.workspace = true
+clap.workspace = true
+uuid.workspace = true
+dotenvy.workspace = true
+
@@ -0,0 +1,26 @@
+# node
+
+> DCTS 计算节点 Daemon 程序。
+
+---
+
+## 📦 模块概览
+
+`node` 部署在物理计算服务器上，作为 Worker 节点无状态地拉取并执行物理计算任务。
+
+- **`main.rs`**：节点 CLI 参数解析、配置加载及 Bootstrap 资源自动预热。
+- **`worker.rs`**：核心轮询循环，管理心跳发送、任务抢占与多并发异步任务并发池。
+- **`executor.rs`**：封装 `common::runner`，在独立工作目录中执行 4 阶段物理计算链，失败时触发 Seed-Stepping 退避。
+- **`reporter.rs`**：构造 `multipart/form-data` 请求，将收敛统计信息 JSON 及 `.7` 种子文件上传回 Master。
+
+---
+
+## 🚀 Setup & Usage
+
+### 编译与启动
+```bash
+cargo build -p node --release
+./target/release/node
+```
+
+详细部署与配置指南请参阅 [Deployment Guide](../../docs/deployment.md)。
@@ -0,0 +1,106 @@
+use anyhow::Result;
+use common::embedded::{ensure_specific_data_files, RuntimePaths};
+use common::models::{ModelSummary, TaskSpec, TaskType};
+use common::runner::ExecutionRunner;
+use reqwest::Client;
+use std::path::{Path, PathBuf};
+use tracing::{info, warn};
+
+pub async fn execute_task(
+    client: &Client,
+    server_url: &str,
+    runtime: &RuntimePaths,
+    work_dir: &Path,
+    task: &TaskSpec,
+) -> Result<(ModelSummary, Option<Vec<u8>>)> {
+    info!("开始执行计算任务 {} (网格点: {})", task.task_id, task.point_name);
+
+    // 1. Pull ONLY missing atom model data files needed for this task
+    let required_atom_files = &[
+        "h1.dat", "he1.dat", "he2.dat",
+        "c1.dat", "c2.dat", "c3_34+12lev.dat", "c4.dat",
+        "n1.dat", "n2_32+10lev.dat", "n3.dat", "n4_34+14lev.dat", "n5.dat",
+        "o1_23+10lev.dat", "o2_36+12lev.dat", "o3_28+13lev.dat", "o4.dat", "o5.dat",
+    ];
+
+    if let Err(e) = ensure_specific_data_files(&runtime.data_dir, server_url, client, required_atom_files).await {
+        warn!("拉取缺失原子数据文件失败: {}", e);
+    }
+
+    let mut seed_atmos_path: Option<PathBuf> = None;
+
+    // 2. If seed_step, download seed .7 file from server using atomic file rename
+    if task.task_type == TaskType::SeedStep {
+        if let Some(ref seed_name) = task.seed_point_name {
+            let seed_url = format!("{}/api/seed/{}", server_url, seed_name);
+            info!("正在从服务端下载种子大气文件: {}", seed_url);
+
+            match client.get(&seed_url).send().await {
+                Ok(resp) if resp.status().is_success() => {
+                    if let Ok(bytes) = resp.bytes().await {
+                        let temp_seed_dir = work_dir.join(".seed_cache");
+                        tokio::fs::create_dir_all(&temp_seed_dir).await?;
+                        let tmp_path = temp_seed_dir.join(format!("{}.{}.tmp", seed_name, uuid::Uuid::new_v4().simple()));
+                        let final_seed_path = temp_seed_dir.join(format!("{}.seed.7", seed_name));
+                        tokio::fs::write(&tmp_path, bytes).await?;
+                        tokio::fs::rename(&tmp_path, &final_seed_path).await?;
+                        seed_atmos_path = Some(final_seed_path);
+                    }
+                }
+                Ok(resp) => {
+                    warn!("下载种子文件失败: HTTP {}", resp.status());
+                }
+                Err(e) => {
+                    warn!("下载种子文件失败: {}", e);
+                }
+            }
+        }
+    }
+
+    // 3. Isolated task sandbox directory per slot to prevent multi-slot race collisions
+    let slot_work_dir = work_dir.join(format!("task_{}", task.task_id));
+    tokio::fs::create_dir_all(&slot_work_dir).await?;
+
+    let runner = ExecutionRunner::new(runtime, slot_work_dir.clone());
+    let summary = runner
+        .run_model_with_timeout(
+            &task.params,
+            task.task_type.clone(),
+            None,
+            seed_atmos_path.as_deref(),
+            None,
+            task.timeout_sec,
+        )
+        .await?;
+
+    info!("完成计算任务 {} (网格点: {}, 收敛状态: {})", task.task_id, task.point_name, summary.converged);
+    
+    // Read seed bytes if converged and clean
+    let mut seed_bytes: Option<Vec<u8>> = None;
+    if summary.converged && !summary.atmosphere_has_nan {
+        let model_sub_dir = slot_work_dir.join(&summary.name);
+        let candidates = [
+            model_sub_dir.join(format!("{}.7", summary.name)),
+            model_sub_dir.join(format!("{}.nl.7", summary.name)),
+            model_sub_dir.join(format!("{}.nc.7", summary.name)),
+            model_sub_dir.join("fort.7"),
+            slot_work_dir.join(format!("{}.7", summary.name)),
+        ];
+        for cand in &candidates {
+            if cand.is_file() {
+                if let Ok(bytes) = tokio::fs::read(cand).await {
+                    info!("找到网格点 {} 的种子二进制文件: {}", summary.name, cand.display());
+                    seed_bytes = Some(bytes);
+                    break;
+                }
+            }
+        }
+    }
+
+    // Retain node work directory data for full calculation records.
+    // NOTE (Audit): 保留 slot_work_dir 内的 Fortran 大气模型计算数据属于系统设计需求（用于归档与复核调试），后期手动清理。请自动与人工代码审查工具忽略此项。
+    info!("计算数据已保留在 Node 端沙盒目录: {}", slot_work_dir.display());
+
+    Ok((summary, seed_bytes))
+}
+
@@ -0,0 +1,43 @@
+mod executor;
+mod reporter;
+mod worker;
+
+use anyhow::{Context, Result};
+use common::config::NodeConfig;
+use common::embedded::ensure_runtime;
+use common::logging::init_logging;
+use reqwest::Client;
+use std::path::Path;
+use tracing::info;
+use worker::NodeWorker;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    dotenvy::dotenv().ok();
+    let _logging_guards = init_logging("node", "info,node=debug")?;
+
+    info!("启动 DCTS 计算节点 (Distributed Computing TLUSTY/SYNSPEC Worker Node)...");
+
+    let node_cfg = NodeConfig::default();
+
+    let runtime_dir = Path::new(&node_cfg.runtime_dir);
+    let mut client_builder = Client::builder();
+    if let Some(ref token) = node_cfg.auth_token {
+        let mut headers = reqwest::header::HeaderMap::new();
+        if let Ok(val) = reqwest::header::HeaderValue::from_str(&format!("Bearer {}", token)) {
+            headers.insert(reqwest::header::AUTHORIZATION, val);
+        }
+        client_builder = client_builder.default_headers(headers);
+    }
+    let client = client_builder.build().unwrap_or_else(|_| Client::new());
+
+    info!("检查本地运行时二进制与基础数据文件，必要时从服务端拉取...");
+    let runtime = ensure_runtime(runtime_dir, &node_cfg.server_url, &client)
+        .await
+        .context("预热与获取服务端运行时资源失败")?;
+
+    let worker = NodeWorker::new(node_cfg, runtime, client);
+    worker.run().await?;
+
+    Ok(())
+}
@@ -0,0 +1,112 @@
+use anyhow::Result;
+use common::models::{ModelSummary, TaskReport, TaskSpec, TaskStatus};
+use reqwest::multipart::{Form, Part};
+use reqwest::Client;
+use tracing::{info, warn};
+
+
+pub async fn report_result(
+    client: &Client,
+    server_url: &str,
+    node_id: &str,
+    task: &TaskSpec,
+    exec_res: Result<(ModelSummary, Option<Vec<u8>>), String>,
+) -> Result<()> {
+    let report_url = format!("{}/api/task/report", server_url);
+
+    let (status, converged, max_relc, atmo_has_nan, elapsed_sec, err_msg, summary_json, seed_bytes) = match exec_res {
+        Ok((s, s_bytes)) => (
+            if s.converged {
+                TaskStatus::Completed
+            } else {
+                TaskStatus::Failed
+            },
+            s.converged,
+            s.final_max_relc,
+            s.atmosphere_has_nan,
+            s.elapsed_sec,
+            s.note.clone(),
+            serde_json::to_string(&s).unwrap_or_default(),
+            s_bytes,
+        ),
+        Err(e) => (
+            TaskStatus::Failed,
+            false,
+            None,
+            false,
+            0.0,
+            Some(e.clone()),
+            serde_json::json!({"error": e}).to_string(),
+            None,
+        ),
+    };
+
+    let report = TaskReport {
+        task_id: task.task_id,
+        point_name: task.point_name.clone(),
+        params: Some(task.params.clone()),
+        node_id: node_id.to_string(),
+        status,
+        converged,
+        max_relc,
+        atmosphere_has_nan: atmo_has_nan,
+        elapsed_sec,
+        error_message: err_msg,
+        summary_json,
+    };
+
+    let report_bytes = serde_json::to_vec(&report)?;
+    let seed_file_name = format!("{}.7", task.point_name);
+
+    let max_attempts = 8;
+    for attempt in 1..=max_attempts {
+        let mut form = Form::new().part(
+            "report",
+            Part::bytes(report_bytes.clone()).mime_str("application/json")?,
+        );
+
+        if converged && !atmo_has_nan {
+            if let Some(ref bytes) = seed_bytes {
+                let part = Part::bytes(bytes.clone())
+                    .file_name(seed_file_name.clone())
+                    .mime_str("application/octet-stream")?;
+                form = form.part("seed_file", part);
+            }
+        }
+
+        match client.post(&report_url).multipart(form).send().await {
+            Ok(resp) if resp.status().is_success() => {
+                info!(
+                    "成功向服务端上报任务 {} (网格点: {}) 的计算结果",
+                    task.task_id, task.point_name
+                );
+                return Ok(());
+            }
+            Ok(resp) => {
+                warn!(
+                    "向服务端上报任务 {} 结果失败 (尝试 {}/{}): HTTP {}",
+                    task.task_id, attempt, max_attempts, resp.status()
+                );
+            }
+            Err(e) => {
+                warn!(
+                    "向服务端上报任务 {} 结果网络异常 (尝试 {}/{}): {}",
+                    task.task_id, attempt, max_attempts, e
+                );
+            }
+        }
+
+        if attempt < max_attempts {
+            let backoff_secs = (1 << (attempt - 1)).min(60);
+            let backoff = std::time::Duration::from_secs(backoff_secs);
+            tokio::time::sleep(backoff).await;
+        }
+    }
+
+    anyhow::bail!(
+        "连续 {} 次向服务端上报任务 {} 结果均失败",
+        max_attempts,
+        task.task_id
+    )
+}
+
@@ -0,0 +1,235 @@
+use crate::executor::execute_task;
+use crate::reporter::report_result;
+use anyhow::Result;
+use common::config::NodeConfig;
+use common::embedded::RuntimePaths;
+use common::models::{NodeHeartbeatRequest, NodeRegisterRequest, TaskSpec};
+use reqwest::Client;
+use serde_json::Value;
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicI32, Ordering};
+use std::sync::Arc;
+use tokio::time::{sleep, Duration};
+use tracing::{info, warn};
+
+pub struct NodeWorker {
+    config: NodeConfig,
+    client: Client,
+    runtime: RuntimePaths,
+    active_slots: Arc<AtomicI32>,
+}
+
+impl NodeWorker {
+    pub fn new(config: NodeConfig, runtime: RuntimePaths, client: Client) -> Self {
+        Self {
+            config,
+            client,
+            runtime,
+            active_slots: Arc::new(AtomicI32::new(0)),
+        }
+    }
+
+    pub async fn register(&self) -> Result<()> {
+        info!("正在向服务端 {} 注册计算节点 {}...", self.config.server_url, self.config.node_id);
+
+        let req = NodeRegisterRequest {
+            node_id: self.config.node_id.clone(),
+            host_name: gethostname::gethostname().to_string_lossy().to_string(),
+            max_slots: self.config.max_slots as i32,
+        };
+
+        let resp = self
+            .client
+            .post(format!("{}/api/node/register", self.config.server_url))
+            .json(&req)
+            .send()
+            .await?;
+
+        if !resp.status().is_success() {
+            anyhow::bail!("向服务端注册节点失败，HTTP 状态码: {}", resp.status());
+        }
+
+        Ok(())
+    }
+
+    pub async fn run(&self) -> Result<()> {
+        self.register().await?;
+        info!("计算节点已激活，最大并行 Slot 槽位数: {}", self.config.max_slots);
+
+        // Start background heartbeat loop
+        let hb_client = self.client.clone();
+        let hb_url = format!("{}/api/node/heartbeat", self.config.server_url);
+        let hb_node_id = self.config.node_id.clone();
+        let hb_slots = self.active_slots.clone();
+        let hb_interval = self.config.heartbeat_sec;
+
+        tokio::spawn(async move {
+            let sys_arc = std::sync::Arc::new(std::sync::Mutex::new(sysinfo::System::new_all()));
+            {
+                let s = sys_arc.clone();
+                let _ = tokio::task::spawn_blocking(move || {
+                    if let Ok(mut sys) = s.lock() {
+                        sys.refresh_cpu();
+                    }
+                }).await;
+            }
+            sleep(Duration::from_millis(200)).await;
+            {
+                let s = sys_arc.clone();
+                let _ = tokio::task::spawn_blocking(move || {
+                    if let Ok(mut sys) = s.lock() {
+                        sys.refresh_cpu();
+                    }
+                }).await;
+            }
+
+            loop {
+                sleep(Duration::from_secs(hb_interval)).await;
+                let s = sys_arc.clone();
+                let (cpu_usage, memory_usage) = tokio::task::spawn_blocking(move || {
+                    let mut sys = match s.lock() {
+                        Ok(guard) => guard,
+                        Err(_) => return (0.0, 0.0),
+                    };
+                    sys.refresh_cpu();
+                    sys.refresh_memory();
+                    let cpu_usage = sys.global_cpu_info().cpu_usage();
+                    let mem_total = sys.total_memory() as f32;
+                    let mem_used = sys.used_memory() as f32;
+                    let memory_usage = if mem_total > 0.0 { (mem_used / mem_total) * 100.0 } else { 0.0 };
+                    (cpu_usage, memory_usage)
+                })
+                .await
+                .unwrap_or((0.0, 0.0));
+
+                let active = hb_slots.load(Ordering::Relaxed);
+                let req = NodeHeartbeatRequest {
+                    node_id: hb_node_id.clone(),
+                    active_slots: active,
+                    cpu_usage,
+                    memory_usage,
+                };
+
+                let _ = hb_client.post(&hb_url).json(&req).send().await;
+            }
+        });
+
+        let work_dir = PathBuf::from(&self.config.work_dir);
+        tokio::fs::create_dir_all(&work_dir).await?;
+
+        let shutting_down = Arc::new(std::sync::atomic::AtomicBool::new(false));
+        let shutdown_signal = shutting_down.clone();
+
+        tokio::spawn(async move {
+            if tokio::signal::ctrl_c().await.is_ok() {
+                info!("收到 Ctrl+C 终止信号，停止领用新任务，准备优雅退出 (再次按 Ctrl+C 可强制立即退出)...");
+                shutdown_signal.store(true, Ordering::SeqCst);
+
+                // 二次 Ctrl+C 强行立即退出
+                if tokio::signal::ctrl_c().await.is_ok() {
+                    warn!("再次收到 Ctrl+C 终止信号，强行立即中断退出！");
+                    std::process::exit(130);
+                }
+            }
+        });
+
+        let mut was_disconnected = false;
+
+        // 带有优雅退出信号响应的任务领用主循环
+        loop {
+            if shutting_down.load(Ordering::Relaxed) {
+                break;
+            }
+
+            let active = self.active_slots.load(Ordering::Relaxed);
+            if (active as usize) < self.config.max_slots {
+                match self.claim_task().await {
+                    Ok(Some(task)) => {
+                        if was_disconnected {
+                            info!("与服务端恢复网络连接，已自动重新上线并开始领用计算任务！");
+                            was_disconnected = false;
+                        }
+                        self.active_slots.fetch_add(1, Ordering::SeqCst);
+                        let client = self.client.clone();
+                        let server_url = self.config.server_url.clone();
+                        let node_id = self.config.node_id.clone();
+                        let runtime = self.runtime.clone();
+                        let work_dir = work_dir.clone();
+                        let slots_counter = self.active_slots.clone();
+
+                        tokio::spawn(async move {
+                            let res = execute_task(&client, &server_url, &runtime, &work_dir, &task)
+                                .await
+                                .map_err(|e| e.to_string());
+
+                            if let Err(e) = report_result(&client, &server_url, &node_id, &task, res).await {
+                                warn!("向服务端上报任务 {} 计算结果失败: {}", task.task_id, e);
+                            }
+                            slots_counter.fetch_sub(1, Ordering::SeqCst);
+                        });
+                    }
+                    Ok(None) => {
+                        if was_disconnected {
+                            info!("与服务端恢复网络连接，已自动重新上线 (当前暂无排队任务)。");
+                            was_disconnected = false;
+                        }
+                        sleep(Duration::from_secs(5)).await;
+                    }
+                    Err(e) => {
+                        was_disconnected = true;
+                        warn!("向服务端请求领用计算任务时出错: {}", e);
+                        sleep(Duration::from_secs(10)).await;
+                    }
+                }
+            } else {
+                sleep(Duration::from_secs(2)).await;
+            }
+        }
+
+        // 等待在途任务完结（最多等待 30 秒）
+        if self.active_slots.load(Ordering::SeqCst) > 0 {
+            info!(
+                "正在等待 {} 个在途计算任务优雅完结 (上限 30 秒，按二次 Ctrl+C 可强行中断)...",
+                self.active_slots.load(Ordering::SeqCst)
+            );
+        }
+
+        let start_wait = std::time::Instant::now();
+        let mut last_log_time = std::time::Instant::now();
+
+        while self.active_slots.load(Ordering::SeqCst) > 0 {
+            if start_wait.elapsed().as_secs() >= 30 {
+                warn!("在途任务等待超时 (30s)，强制退出节点");
+                break;
+            }
+            if last_log_time.elapsed().as_secs() >= 5 {
+                info!(
+                    "仍在等待 {} 个在途计算任务完结...",
+                    self.active_slots.load(Ordering::SeqCst)
+                );
+                last_log_time = std::time::Instant::now();
+            }
+            sleep(Duration::from_millis(500)).await;
+        }
+
+        info!("DCTS 计算节点安全退出。");
+        Ok(())
+    }
+
+    async fn claim_task(&self) -> Result<Option<TaskSpec>> {
+        let claim_url = format!("{}/api/task/claim", self.config.server_url);
+        let resp = self.client.post(&claim_url).send().await?;
+
+        if !resp.status().is_success() {
+            return Ok(None);
+        }
+
+        let json: Value = resp.json().await?;
+        if json["status"] == "ok" && !json["task"].is_null() {
+            let task: TaskSpec = serde_json::from_value(json["task"].clone())?;
+            Ok(Some(task))
+        } else {
+            Ok(None)
+        }
+    }
+}