feat(server,dashboard): 引入多工作流数据隔离、安全中间件与前端 ESM 模块化重构

- server: 实现按 workflow_name 的多工作流数据隔离与旧数据库平滑迁移机制
- server: 新增 API Key 认证(auth)、限流中间件(rate_limit)与运维备份接口(admin)
- server: 统一 AppError 错误处理体系,重构调度器 scheduler 支持工作流级重置与抢占
- node: 节点 ID 缺失时自动生成随机 UUID,原生支持 `docker compose --scale node=N` 动态扩容
- dashboard: 前端模块化重构(state/api/components),升级 CSS 变量设计系统与 Toast 通知
- docker/docs: 更新 /healthz 健康检查、部署脚本 IP 配置及数据库设计文档
This commit is contained in:
fmq
2026-07-28 21:54:02 +08:00
parent 4b4238d702
commit b91f1e4fa5
59 changed files with 7697 additions and 1490 deletions
+125 -54
View File
@@ -1,6 +1,6 @@
use super::AppState;
use super::{AppState, AuthenticatedNode};
use axum::{
extract::{Multipart, State},
extract::{Extension, Multipart, State},
response::IntoResponse,
Json,
};
@@ -12,27 +12,38 @@ use tracing::{info, warn};
use axum::http::StatusCode;
pub async fn claim_task(State(state): State<AppState>) -> impl IntoResponse {
match state.queue.pop_task().await {
pub async fn claim_task(
State(state): State<AppState>,
Extension(auth_node): Extension<AuthenticatedNode>,
) -> Result<impl IntoResponse, crate::api::AppError> {
// 领用时记录任务归属:pop_task 写入 claimed_by_node_id
// report 阶段据此校验「上报者确为领用者」,杜绝跨节点伪造结果。
match state.queue.pop_task(&auth_node.node_id).await {
Ok(Some(task)) => {
if let Err(e) = state.db.mark_grid_point_running(&task.point_name).await {
// 多工作流分区:mark_grid_point_running 须带 workflow_name,避免按 name 全局更新
// 误改其他工作流的同名点。TaskSpec.workflow_name 在调度时已绑定。
let wf = task.workflow_name.as_deref().unwrap_or("");
if let Err(e) = state.db.mark_grid_point_running(&task.point_name, wf).await {
warn!("领用任务 {} 后同步变更为 running 状态遇到异常: {}. 后置 stale 定时自取检索引索将介入修复维护", task.task_id, e);
}
(StatusCode::OK, Json(json!({"status": "ok", "task": task}))).into_response()
Ok((StatusCode::OK, Json(json!({"status": "ok", "task": task}))))
}
Ok(None) => Ok((
StatusCode::OK,
Json(json!({"status": "empty", "task": null})),
)),
Err(e) => {
tracing::error!("领用任务数据库异常: {}", e);
Err(crate::api::AppError::Internal(e))
}
Ok(None) => (StatusCode::OK, Json(json!({"status": "empty", "task": null}))).into_response(),
Err(e) => (
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"status": "error", "message": format!("领用任务失败: {}", e)})),
)
.into_response(),
}
}
pub async fn report_task(
State(state): State<AppState>,
Extension(auth_node): Extension<AuthenticatedNode>,
mut multipart: Multipart,
) -> impl IntoResponse {
) -> Result<impl IntoResponse, crate::api::AppError> {
let mut report_json: Option<TaskReport> = None;
let mut seed_file_data: Option<Vec<u8>> = None;
@@ -51,53 +62,98 @@ pub async fn report_task(
}
}
let report = match report_json {
let mut report = match report_json {
Some(r) => r,
None => {
return (
StatusCode::BAD_REQUEST,
Json(json!({"status": "error", "message": "请求中缺少 report 字段"})),
)
.into_response();
return Err(crate::api::AppError::BadRequest(
"请求中缺少 report 字段".to_string(),
));
}
};
// ── 任务归属校验(S1 核心,防跨节点伪造结果投毒)──
// 1. 该 task_id 必须由当前鉴权 node 领用(claim 时记录的 claimed_by_node_id 匹配)。
// 2. 上报的 point_name 必须与该 task 绑定的 point_name 一致(防跨点上报)。
// 3. 忽略 body 里声称的 node_id,统一以鉴权 node_id 写库(修复审计归因断裂)。
// 4. 取 task 绑定的 workflow_name,用于定向更新该工作流的 grid_points(多工作流分区)。
let (claimed_point, claimed_workflow) = match state
.queue
.verify_task_claim(&report.task_id.to_string(), &auth_node.node_id)
.await
{
Ok(Some((p, w))) => (p, w),
Ok(None) => {
warn!(
"任务归属校验失败:node={} 上报 task_id={} 但未领用或已被清理",
auth_node.node_id, report.task_id
);
return Err(crate::api::AppError::Forbidden(
"任务未由本节点领用或已上报过".to_string(),
));
}
Err(e) => {
tracing::error!("校验任务归属数据库异常: {}", e);
return Err(crate::api::AppError::Internal(e));
}
};
if claimed_point != report.point_name {
warn!(
"任务点名校验失败:task_id={} 领用 point={} 但上报 point={}",
report.task_id, claimed_point, report.point_name
);
return Err(crate::api::AppError::Forbidden(
"上报的网格点与领用任务不匹配".to_string(),
));
}
// 统一以鉴权 node_id 覆盖 body 里的 node_id,保证归因可信
report.node_id = auth_node.node_id.clone();
// workflow_name 以领用记录为准(claim 时从 TaskSpec 落库),body 无权声称。
let workflow_name = claimed_workflow.unwrap_or_default();
let name = report.point_name.clone();
if name.is_empty() || name.starts_with('.') || !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '_' || c == '-' || c == '+' || c == '@') {
warn!("拒绝可能包含路径穿越或特殊非常规编码号攻击的网格点名称请求: {}", name);
return (
StatusCode::BAD_REQUEST,
Json(json!({"status": "error", "message": "非法的网格点名称参数"})),
)
.into_response();
if name.is_empty()
|| name.starts_with('.')
|| !name.chars().all(|c| {
c.is_ascii_alphanumeric() || c == '.' || c == '_' || c == '-' || c == '+' || c == '@'
})
{
warn!(
"拒绝可能包含路径穿越或特殊非常规编码号攻击的网格点名称请求: {}",
name
);
return Err(crate::api::AppError::BadRequest(
"非法的网格点名称参数".to_string(),
));
}
let params = match extract_params(&report) {
Some(p) => p,
None => {
warn!("网格点 {} 汇报数据解析失败: 无法解析 params 或 summary_json", name);
return (
StatusCode::BAD_REQUEST,
Json(json!({"status": "error", "message": "无法解析 params 或 summary_json"})),
)
.into_response();
warn!(
"网格点 {} 汇报数据解析失败: 无法解析 params 或 summary_json",
name
);
return Err(crate::api::AppError::BadRequest(
"无法解析 params 或 summary_json".to_string(),
));
}
};
// Record in DB
if let Err(e) = state.db.record_task_report(&report).await {
warn!("记录网格点 {} 任务结果到数据库失败: {}", name, e);
return (
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"status": "error", "message": format!("记录数据库失败: {}", e)})),
)
.into_response();
// Record in DB(带 workflow_name 定向更新该工作流的 grid_points
if let Err(e) = state.db.record_task_report(&report, &workflow_name).await {
// DB 错误细节进日志,对客户端只返回通用消息(避免泄露表结构/内部错误给未授权方)
tracing::error!("记录网格点 {} 任务结果到数据库失败: {}", name, e);
return Err(crate::api::AppError::Internal(e));
}
// Clean up task from task_queue table to prevent queue DB bloat
if let Err(e) = state.queue.remove_task(&report.task_id.to_string()).await {
tracing::warn!("从任务队列中清理已上报任务记录 {} 失败: {}", report.task_id, e);
tracing::warn!(
"从任务队列中清理已上报任务记录 {} 失败: {}",
report.task_id,
e
);
}
// 采用原子写入模式保持 conv.json 与核心二进制数据完整落地后才揭晓真实文件名
@@ -112,30 +168,46 @@ pub async fn report_task(
// Save seed file .7 using atomic temporary writing strategy
if report.converged && !report.atmosphere_has_nan {
if let Some(bytes) = seed_file_data {
let seed_tmp = model_dir.join(format!("{}.7.{}.tmp", name, uuid::Uuid::new_v4().simple()));
let seed_tmp =
model_dir.join(format!("{}.7.{}.tmp", name, uuid::Uuid::new_v4().simple()));
let seed_path = model_dir.join(format!("{}.7", name));
if fs::write(&seed_tmp, bytes).await.is_ok() {
if fs::rename(&seed_tmp, &seed_path).await.is_ok() {
info!("成功保持原子写入落地并保存网格点 {} 的收敛种子文件: {}", name, seed_path.display());
let _ = state
.db
.insert_seed(&params, &seed_path.to_string_lossy())
.await;
}
if fs::write(&seed_tmp, bytes).await.is_ok()
&& fs::rename(&seed_tmp, &seed_path).await.is_ok()
{
info!(
"成功保持原子写入落地并保存网格点 {} 的收敛种子文件: {}",
name,
seed_path.display()
);
let _ = state
.db
.insert_seed(&params, &seed_path.to_string_lossy())
.await;
}
}
}
}
if report.status == TaskStatus::Failed || report.status == TaskStatus::Timeout || report.atmosphere_has_nan {
if !report.converged
|| report.atmosphere_has_nan
|| report.status == TaskStatus::Failed
|| report.status == TaskStatus::Timeout
{
// Task did not succeed -> check if seed_step fallback should be triggered
info!("网格点 {} 计算未成功完成,检查种子回退机制...", name);
if let Err(e) = state.scheduler.trigger_seed_step_fallback(&params).await {
if let Err(e) = state
.scheduler
.trigger_seed_step_fallback(&params, &workflow_name)
.await
{
warn!("网格点 {} 触发种子回退机制失败: {}", name, e);
}
}
(StatusCode::OK, Json(json!({"status": "ok", "message": "上报成功"}))).into_response()
Ok((
StatusCode::OK,
Json(json!({"status": "ok", "message": "上报成功"})),
))
}
fn extract_params(report: &TaskReport) -> Option<GridPointParams> {
@@ -146,4 +218,3 @@ fn extract_params(report: &TaskReport) -> Option<GridPointParams> {
.ok()
.map(|summary| summary.params)
}