feat(server,dashboard): 引入多工作流数据隔离、安全中间件与前端 ESM 模块化重构
- server: 实现按 workflow_name 的多工作流数据隔离与旧数据库平滑迁移机制 - server: 新增 API Key 认证(auth)、限流中间件(rate_limit)与运维备份接口(admin) - server: 统一 AppError 错误处理体系,重构调度器 scheduler 支持工作流级重置与抢占 - node: 节点 ID 缺失时自动生成随机 UUID,原生支持 `docker compose --scale node=N` 动态扩容 - dashboard: 前端模块化重构(state/api/components),升级 CSS 变量设计系统与 Toast 通知 - docker/docs: 更新 /healthz 健康检查、部署脚本 IP 配置及数据库设计文档
This commit is contained in:
+125
-54
@@ -1,6 +1,6 @@
|
||||
use super::AppState;
|
||||
use super::{AppState, AuthenticatedNode};
|
||||
use axum::{
|
||||
extract::{Multipart, State},
|
||||
extract::{Extension, Multipart, State},
|
||||
response::IntoResponse,
|
||||
Json,
|
||||
};
|
||||
@@ -12,27 +12,38 @@ use tracing::{info, warn};
|
||||
|
||||
use axum::http::StatusCode;
|
||||
|
||||
pub async fn claim_task(State(state): State<AppState>) -> impl IntoResponse {
|
||||
match state.queue.pop_task().await {
|
||||
pub async fn claim_task(
|
||||
State(state): State<AppState>,
|
||||
Extension(auth_node): Extension<AuthenticatedNode>,
|
||||
) -> Result<impl IntoResponse, crate::api::AppError> {
|
||||
// 领用时记录任务归属:pop_task 写入 claimed_by_node_id,
|
||||
// report 阶段据此校验「上报者确为领用者」,杜绝跨节点伪造结果。
|
||||
match state.queue.pop_task(&auth_node.node_id).await {
|
||||
Ok(Some(task)) => {
|
||||
if let Err(e) = state.db.mark_grid_point_running(&task.point_name).await {
|
||||
// 多工作流分区:mark_grid_point_running 须带 workflow_name,避免按 name 全局更新
|
||||
// 误改其他工作流的同名点。TaskSpec.workflow_name 在调度时已绑定。
|
||||
let wf = task.workflow_name.as_deref().unwrap_or("");
|
||||
if let Err(e) = state.db.mark_grid_point_running(&task.point_name, wf).await {
|
||||
warn!("领用任务 {} 后同步变更为 running 状态遇到异常: {}. 后置 stale 定时自取检索引索将介入修复维护", task.task_id, e);
|
||||
}
|
||||
(StatusCode::OK, Json(json!({"status": "ok", "task": task}))).into_response()
|
||||
Ok((StatusCode::OK, Json(json!({"status": "ok", "task": task}))))
|
||||
}
|
||||
Ok(None) => Ok((
|
||||
StatusCode::OK,
|
||||
Json(json!({"status": "empty", "task": null})),
|
||||
)),
|
||||
Err(e) => {
|
||||
tracing::error!("领用任务数据库异常: {}", e);
|
||||
Err(crate::api::AppError::Internal(e))
|
||||
}
|
||||
Ok(None) => (StatusCode::OK, Json(json!({"status": "empty", "task": null}))).into_response(),
|
||||
Err(e) => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(json!({"status": "error", "message": format!("领用任务失败: {}", e)})),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn report_task(
|
||||
State(state): State<AppState>,
|
||||
Extension(auth_node): Extension<AuthenticatedNode>,
|
||||
mut multipart: Multipart,
|
||||
) -> impl IntoResponse {
|
||||
) -> Result<impl IntoResponse, crate::api::AppError> {
|
||||
let mut report_json: Option<TaskReport> = None;
|
||||
let mut seed_file_data: Option<Vec<u8>> = None;
|
||||
|
||||
@@ -51,53 +62,98 @@ pub async fn report_task(
|
||||
}
|
||||
}
|
||||
|
||||
let report = match report_json {
|
||||
let mut report = match report_json {
|
||||
Some(r) => r,
|
||||
None => {
|
||||
return (
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(json!({"status": "error", "message": "请求中缺少 report 字段"})),
|
||||
)
|
||||
.into_response();
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"请求中缺少 report 字段".to_string(),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// ── 任务归属校验(S1 核心,防跨节点伪造结果投毒)──
|
||||
// 1. 该 task_id 必须由当前鉴权 node 领用(claim 时记录的 claimed_by_node_id 匹配)。
|
||||
// 2. 上报的 point_name 必须与该 task 绑定的 point_name 一致(防跨点上报)。
|
||||
// 3. 忽略 body 里声称的 node_id,统一以鉴权 node_id 写库(修复审计归因断裂)。
|
||||
// 4. 取 task 绑定的 workflow_name,用于定向更新该工作流的 grid_points(多工作流分区)。
|
||||
let (claimed_point, claimed_workflow) = match state
|
||||
.queue
|
||||
.verify_task_claim(&report.task_id.to_string(), &auth_node.node_id)
|
||||
.await
|
||||
{
|
||||
Ok(Some((p, w))) => (p, w),
|
||||
Ok(None) => {
|
||||
warn!(
|
||||
"任务归属校验失败:node={} 上报 task_id={} 但未领用或已被清理",
|
||||
auth_node.node_id, report.task_id
|
||||
);
|
||||
return Err(crate::api::AppError::Forbidden(
|
||||
"任务未由本节点领用或已上报过".to_string(),
|
||||
));
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("校验任务归属数据库异常: {}", e);
|
||||
return Err(crate::api::AppError::Internal(e));
|
||||
}
|
||||
};
|
||||
if claimed_point != report.point_name {
|
||||
warn!(
|
||||
"任务点名校验失败:task_id={} 领用 point={} 但上报 point={}",
|
||||
report.task_id, claimed_point, report.point_name
|
||||
);
|
||||
return Err(crate::api::AppError::Forbidden(
|
||||
"上报的网格点与领用任务不匹配".to_string(),
|
||||
));
|
||||
}
|
||||
// 统一以鉴权 node_id 覆盖 body 里的 node_id,保证归因可信
|
||||
report.node_id = auth_node.node_id.clone();
|
||||
// workflow_name 以领用记录为准(claim 时从 TaskSpec 落库),body 无权声称。
|
||||
let workflow_name = claimed_workflow.unwrap_or_default();
|
||||
|
||||
let name = report.point_name.clone();
|
||||
|
||||
if name.is_empty() || name.starts_with('.') || !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '_' || c == '-' || c == '+' || c == '@') {
|
||||
warn!("拒绝可能包含路径穿越或特殊非常规编码号攻击的网格点名称请求: {}", name);
|
||||
return (
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(json!({"status": "error", "message": "非法的网格点名称参数"})),
|
||||
)
|
||||
.into_response();
|
||||
if name.is_empty()
|
||||
|| name.starts_with('.')
|
||||
|| !name.chars().all(|c| {
|
||||
c.is_ascii_alphanumeric() || c == '.' || c == '_' || c == '-' || c == '+' || c == '@'
|
||||
})
|
||||
{
|
||||
warn!(
|
||||
"拒绝可能包含路径穿越或特殊非常规编码号攻击的网格点名称请求: {}",
|
||||
name
|
||||
);
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"非法的网格点名称参数".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let params = match extract_params(&report) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
warn!("网格点 {} 汇报数据解析失败: 无法解析 params 或 summary_json", name);
|
||||
return (
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(json!({"status": "error", "message": "无法解析 params 或 summary_json"})),
|
||||
)
|
||||
.into_response();
|
||||
warn!(
|
||||
"网格点 {} 汇报数据解析失败: 无法解析 params 或 summary_json",
|
||||
name
|
||||
);
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"无法解析 params 或 summary_json".to_string(),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Record in DB
|
||||
if let Err(e) = state.db.record_task_report(&report).await {
|
||||
warn!("记录网格点 {} 任务结果到数据库失败: {}", name, e);
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(json!({"status": "error", "message": format!("记录数据库失败: {}", e)})),
|
||||
)
|
||||
.into_response();
|
||||
// Record in DB(带 workflow_name 定向更新该工作流的 grid_points)
|
||||
if let Err(e) = state.db.record_task_report(&report, &workflow_name).await {
|
||||
// DB 错误细节进日志,对客户端只返回通用消息(避免泄露表结构/内部错误给未授权方)
|
||||
tracing::error!("记录网格点 {} 任务结果到数据库失败: {}", name, e);
|
||||
return Err(crate::api::AppError::Internal(e));
|
||||
}
|
||||
|
||||
// Clean up task from task_queue table to prevent queue DB bloat
|
||||
if let Err(e) = state.queue.remove_task(&report.task_id.to_string()).await {
|
||||
tracing::warn!("从任务队列中清理已上报任务记录 {} 失败: {}", report.task_id, e);
|
||||
tracing::warn!(
|
||||
"从任务队列中清理已上报任务记录 {} 失败: {}",
|
||||
report.task_id,
|
||||
e
|
||||
);
|
||||
}
|
||||
|
||||
// 采用原子写入模式保持 conv.json 与核心二进制数据完整落地后才揭晓真实文件名
|
||||
@@ -112,30 +168,46 @@ pub async fn report_task(
|
||||
// Save seed file .7 using atomic temporary writing strategy
|
||||
if report.converged && !report.atmosphere_has_nan {
|
||||
if let Some(bytes) = seed_file_data {
|
||||
let seed_tmp = model_dir.join(format!("{}.7.{}.tmp", name, uuid::Uuid::new_v4().simple()));
|
||||
let seed_tmp =
|
||||
model_dir.join(format!("{}.7.{}.tmp", name, uuid::Uuid::new_v4().simple()));
|
||||
let seed_path = model_dir.join(format!("{}.7", name));
|
||||
if fs::write(&seed_tmp, bytes).await.is_ok() {
|
||||
if fs::rename(&seed_tmp, &seed_path).await.is_ok() {
|
||||
info!("成功保持原子写入落地并保存网格点 {} 的收敛种子文件: {}", name, seed_path.display());
|
||||
let _ = state
|
||||
.db
|
||||
.insert_seed(¶ms, &seed_path.to_string_lossy())
|
||||
.await;
|
||||
}
|
||||
if fs::write(&seed_tmp, bytes).await.is_ok()
|
||||
&& fs::rename(&seed_tmp, &seed_path).await.is_ok()
|
||||
{
|
||||
info!(
|
||||
"成功保持原子写入落地并保存网格点 {} 的收敛种子文件: {}",
|
||||
name,
|
||||
seed_path.display()
|
||||
);
|
||||
let _ = state
|
||||
.db
|
||||
.insert_seed(¶ms, &seed_path.to_string_lossy())
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if report.status == TaskStatus::Failed || report.status == TaskStatus::Timeout || report.atmosphere_has_nan {
|
||||
if !report.converged
|
||||
|| report.atmosphere_has_nan
|
||||
|| report.status == TaskStatus::Failed
|
||||
|| report.status == TaskStatus::Timeout
|
||||
{
|
||||
// Task did not succeed -> check if seed_step fallback should be triggered
|
||||
info!("网格点 {} 计算未成功完成,检查种子回退机制...", name);
|
||||
if let Err(e) = state.scheduler.trigger_seed_step_fallback(¶ms).await {
|
||||
if let Err(e) = state
|
||||
.scheduler
|
||||
.trigger_seed_step_fallback(¶ms, &workflow_name)
|
||||
.await
|
||||
{
|
||||
warn!("网格点 {} 触发种子回退机制失败: {}", name, e);
|
||||
}
|
||||
}
|
||||
|
||||
(StatusCode::OK, Json(json!({"status": "ok", "message": "上报成功"}))).into_response()
|
||||
Ok((
|
||||
StatusCode::OK,
|
||||
Json(json!({"status": "ok", "message": "上报成功"})),
|
||||
))
|
||||
}
|
||||
|
||||
fn extract_params(report: &TaskReport) -> Option<GridPointParams> {
|
||||
@@ -146,4 +218,3 @@ fn extract_params(report: &TaskReport) -> Option<GridPointParams> {
|
||||
.ok()
|
||||
.map(|summary| summary.params)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user