feat(all): 任务引擎双阶段解耦、僵尸涡旋修复、动态 CPU 配额与前端详情页重构
将 TLUSTY/SYNSPEC 拆为各自独立的 enabled/policy/strategies 阶段,
以策略链自动弹栈取代单级 seed_step 布尔回退;定向修复 2026-08-02
僵尸任务涡旋事故;新增节点并发配额热调;前端详情页从 1412 行巨型
视图拆为薄控制器 + detail 子模块,并补齐工具层与单测。
引擎与调度(task_engine_decoupling_design.md)
- models.rs: 新增 StagePolicy / EngineStageConfig / TaskSpec 阶段字段、
normalize_compat() 校正旧版在途消息策略链、failed_stage 归因
- scheduler.rs: resolve_dispatchable_chain 派发门控、
trigger_strategy_fallback 按 failed_stage 精确弹栈;启动期
force_recompute/skip_converged(默认)/skip_failed 三策略
- db.rs: tasks 表 +7 列持久化阶段配置;终态守卫
(mark_grid_point_running 仅 pending/queued→running;
record_task_report 拒绝迟到失败翻黑 converged);策略弹栈快照
僵尸涡旋修复(runbook-20260802-zombie-vortex-fix.md)
- 全链路跨库活性交叉校验:派发/claim/孤儿回收/回退统一查 MQ 队列活性,
活则放行、死则清僵尸,结构性消除"每点重复派发"
- stop/重启卫生:清队列同步 delete_tasks_by_ids,杜绝遗留 pending 行
- report_task: 幂等吸收 + 409 区分迟到冗余结果,仅 state_changed 时回退
- MQ: NULL workflow_name 回填 __legacy__、requeue 后迟到上报被 403 竞态修复
动态 CPU 配额(dynamic_cpu_slots_design.md)
- admin.rs: POST /admin/nodes/:id/quota(Option<Option<i32>> 区分
缺字段/显式 null);nodes 表 +admin_max_slots
- worker.rs: effective_max_slots = min(admin, physical),心跳下发原子生效
科学产物保全(tlusty_result_artifacts.md)
- runner.rs: SYNSPEC 启动前快照 fort.12/fort.14 → .bfac/.emflux 防覆盖
- 半失败点(大气收敛+光谱失败)改判 Failed 并写入 note;仅 SYNSPEC
场景不再恒判失败;撤销归档 LRU 200 上限改为永久保留
- executor.rs: 透传 synspec_params 数值参数(此前固定 None)
前端(dashboard/)
- workflowDetail.js 1412→328 行,拆出 views/detail/{ctx,overview,
pointsTable,parSets,pointPanel}.js,AbortController 治理监听/请求生命周期
- 删除 wfActions.js,新增 wfEnginePanel.js(双阶段三维配置编辑面板)
- 新增 utils/{errors,format,icons,polling,yamlStage}.js 纯函数模块
- 路由级动态 import 代码分割;节点配额三点菜单 + Modal 管理
- 首次引入 node:test 单测(format/polling/yamlStage/psCache,644 行)
- 系统性补齐 a11y:skip-link、ARIA、Tab 键盘漫游、toast 关闭、退出动画
文档与工具
- 新增 6 篇设计/调研:引擎解耦、动态配额、涡旋 runbook、
光谱正确性分析、收敛判断、产物归档
- PIPELINE/design/api/database 等协同重写为分布式 C/S 架构口径
- scripts/fetch_results.sh 跨节点产物备份;import_results 按 cno 升序导入
- workflows/sdB_cno.yaml: 新增 tlusty/synspec_stage 配置块,修正 wstart 笔误
This commit is contained in:
@@ -191,3 +191,103 @@ pub async fn enable_node(
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// POST /api/admin/nodes/:node_id/quota — 设置节点并发槽位配额(动态调整 CPU 核数)。
|
||||
///
|
||||
/// 见 `docs/dynamic_cpu_slots_design.md`:管理员通过本接口下发配额上限,服务端写入
|
||||
/// `nodes.admin_max_slots`,Worker 在下一次心跳响应里取回并据此调整本地领用并发数。
|
||||
/// 不中断正在运行的任务,实现平滑降级。
|
||||
///
|
||||
/// 请求体:`{"admin_max_slots": 4}`(`null` 表示解除限制,恢复物理 `max_slots`;`0` 表示
|
||||
/// 暂停接新任务)。配额合法后立即落库,Worker 下次心跳(`heartbeat_sec` 秒内)即生效。
|
||||
#[derive(serde::Deserialize)]
|
||||
pub struct SetNodeQuotaRequest {
|
||||
/// 双层 Option 区分「字段缺失」与「显式 null」:
|
||||
/// - 外层 `None`(JSON 字段缺失)→ 400(防止畸形输入被静默当作清除配额)
|
||||
/// - 外层 `Some(None)`(JSON `null`)→ 清除限制
|
||||
/// - 外层 `Some(Some(n))`(JSON 数字)→ 设为 n(n >= 0)
|
||||
///
|
||||
/// 注:serde 对嵌套 `Option<Option<T>>` 默认把 `null` 也映射为外层 `None`
|
||||
/// (与缺失字段不可区分),必须用 `deserialize_with` 显式区分(审查修复 M1)。
|
||||
#[serde(default, deserialize_with = "deserialize_quota")]
|
||||
pub admin_max_slots: Option<Option<i32>>,
|
||||
}
|
||||
|
||||
/// 自定义反序列化:字段**存在**时把 `null` 映射为 `Some(None)`(清除),数字映射为
|
||||
/// `Some(Some(n))`(设值)。字段**缺失**时 serde 走 `#[serde(default)]`(外层 None),
|
||||
/// 与显式 null 语义区分开。
|
||||
fn deserialize_quota<'de, D>(deserializer: D) -> Result<Option<Option<i32>>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let inner: Option<i32> = serde::Deserialize::deserialize(deserializer)?;
|
||||
Ok(Some(inner))
|
||||
}
|
||||
|
||||
pub async fn set_node_quota(
|
||||
State(state): State<AppState>,
|
||||
AxumPath(node_id): AxumPath<String>,
|
||||
Json(req): Json<SetNodeQuotaRequest>,
|
||||
) -> Result<impl IntoResponse, crate::api::AppError> {
|
||||
if !is_valid_node_id(&node_id) {
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"非法的节点 ID 参数".to_string(),
|
||||
));
|
||||
}
|
||||
// 字段缺失(`{}`、字段名拼错等)是畸形输入而非「清除」:显式拒绝,避免破坏性默认值。
|
||||
// 审查修复(M1):旧实现 `Option<i32>` 在缺字段时 serde 默认 None,会把任何畸形请求
|
||||
// 静默当成「清除配额」执行,客户端只见 200「已清除限制」。
|
||||
let admin_max_slots = match req.admin_max_slots {
|
||||
None => {
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"请求体缺少 admin_max_slots 字段(传 null 清除限制,传非负整数设置配额)"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
Some(v) => v,
|
||||
};
|
||||
// 配额非负校验:0 合法(暂停接新任务),仅拒绝负数。
|
||||
if let Some(n) = admin_max_slots {
|
||||
if n < 0 {
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"admin_max_slots 不能为负数(0 = 暂停接新任务,null = 清除限制)".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
// 节点须存在(防幽灵 node_id 写入空更新)
|
||||
match state.db.get_node_exists(&node_id).await {
|
||||
Ok(false) => {
|
||||
return Err(crate::api::AppError::NotFound(format!(
|
||||
"节点 '{}' 不存在",
|
||||
node_id
|
||||
)));
|
||||
}
|
||||
Ok(true) => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
match state
|
||||
.db
|
||||
.set_node_admin_max_slots(&node_id, admin_max_slots)
|
||||
.await
|
||||
{
|
||||
Ok(true) => {
|
||||
let desc = match admin_max_slots {
|
||||
Some(n) => format!("配额上限设为 {}(下一次心跳后生效)", n),
|
||||
None => "已清除配额限制(恢复物理 max_slots,下一次心跳后生效)".to_string(),
|
||||
};
|
||||
info!("管理员已调整节点 {} 的并发槽位配额:{}", node_id, desc);
|
||||
Ok((
|
||||
StatusCode::OK,
|
||||
Json(json!({
|
||||
"success": true,
|
||||
"message": format!("节点 '{}' {}", node_id, desc),
|
||||
})),
|
||||
))
|
||||
}
|
||||
Ok(false) => Err(crate::api::AppError::NotFound(format!(
|
||||
"节点 '{}' 不存在",
|
||||
node_id
|
||||
))),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,10 +127,7 @@ pub async fn check_auth() -> impl IntoResponse {
|
||||
/// 一致的 `extract_token_from_headers`,同时支持 Authorization: Bearer 与 X-API-Key、
|
||||
/// 拒绝空值)并从 `admin_sessions` 中移除,使该 token 在服务端立即失效(而非等 24h 过期)。
|
||||
/// 这样即便 token 已被窃取,登出操作也能立即阻断重放。
|
||||
pub async fn logout(
|
||||
State(state): State<AppState>,
|
||||
headers: HeaderMap,
|
||||
) -> impl IntoResponse {
|
||||
pub async fn logout(State(state): State<AppState>, headers: HeaderMap) -> impl IntoResponse {
|
||||
// 与 auth_middleware 口径一致地提取 token(支持 X-API-Key、拒绝空值)。
|
||||
let token = crate::api::extract_token_from_headers(&headers);
|
||||
|
||||
|
||||
@@ -181,7 +181,10 @@ pub async fn heartbeat_node(
|
||||
));
|
||||
}
|
||||
match state.db.heartbeat_node(&req).await {
|
||||
Ok(_) => Ok(Json(json!({"status": "ok"}))),
|
||||
Ok(admin_max_slots) => Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"admin_max_slots": admin_max_slots,
|
||||
}))),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
+109
-32
@@ -40,9 +40,23 @@ pub async fn claim_task(
|
||||
Ok(Some(task)) => {
|
||||
// 多工作流分区:mark_grid_point_running 须带 workflow_name,避免按 name 全局更新
|
||||
// 误改其他工作流的同名点。TaskSpec.workflow_name 在调度时已绑定。
|
||||
let wf = task.workflow_name.as_deref().unwrap_or("");
|
||||
if let Err(e) = state.db.mark_grid_point_running(&task.point_name, wf).await {
|
||||
warn!("领用任务 {} 后同步变更为 running 状态遇到异常: {}. 后置 stale 定时自取检索引索将介入修复维护", task.task_id, e);
|
||||
// 终态守卫(2026-08-02 涡旋事故修复):仅 pending/queued → running;返回
|
||||
// false 表示点已在 running 或终态(迟到/重复领用),记录后照常下发任务
|
||||
// (队列凭证有效,计算结果仍会被 record_task_report 的终态守卫正确吸收)。
|
||||
// 旧版在途任务 payload 无 workflow_name(None)→ 归一到主库迁移回填的
|
||||
// '__legacy__' 标记,使 mark_grid_point_running 能命中 legacy 网格点(H1 修复)。
|
||||
let wf = crate::db::normalize_workflow_name(task.workflow_name.as_deref());
|
||||
match state.db.mark_grid_point_running(&task.point_name, &wf).await {
|
||||
Ok(false) => {
|
||||
info!(
|
||||
"领用任务 {}(网格点 {})时点已非 pending/queued 态,跳过 running 标记(迟到/重复领用)",
|
||||
task.task_id, task.point_name
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("领用任务 {} 后同步变更为 running 状态遇到异常: {}. 后置 stale 定时自取检索引索将介入修复维护", task.task_id, e);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok((StatusCode::OK, Json(json!({"status": "ok", "task": task}))))
|
||||
}
|
||||
@@ -127,6 +141,11 @@ pub async fn report_task(
|
||||
// 2. 上报的 point_name 必须与该 task 绑定的 point_name 一致(防跨点上报)。
|
||||
// 3. 忽略 body 里声称的 node_id,统一以鉴权 node_id 写库(修复审计归因断裂)。
|
||||
// 4. 取 task 绑定的 workflow_name,用于定向更新该工作流的 grid_points(多工作流分区)。
|
||||
// 幂等吸收:verify_task_claim 落空有两种可能——(a) 首轮上报已完成且 MQ 领用行已被
|
||||
// remove_task 清除,但响应在链路上丢失,节点重试;(b) 真·伪造。凭 tasks 表已结算记录
|
||||
// 区分:若该任务确已由本节点结算(find_settled_task_claim),按幂等重放处理(补写种子、
|
||||
// 返回 200),避免节点误判「token 失效」而中止并导致种子/结算丢失。
|
||||
let mut idempotent = false;
|
||||
let (claimed_point, claimed_workflow) = match state
|
||||
.queue
|
||||
.verify_task_claim(&report.task_id.to_string(), &auth_node.node_id)
|
||||
@@ -134,13 +153,37 @@ pub async fn report_task(
|
||||
{
|
||||
Ok(Some((p, w))) => (p, w),
|
||||
Ok(None) => {
|
||||
warn!(
|
||||
"任务归属校验失败:node={} 上报 task_id={} 但未领用或已被清理",
|
||||
auth_node.node_id, report.task_id
|
||||
);
|
||||
return Err(crate::api::AppError::Forbidden(
|
||||
"任务未由本节点领用或已上报过".to_string(),
|
||||
));
|
||||
match state
|
||||
.db
|
||||
.find_settled_task_claim(&report.task_id.to_string(), &auth_node.node_id)
|
||||
.await
|
||||
{
|
||||
Ok(Some((p, w))) => {
|
||||
info!(
|
||||
"任务 {} 已由本节点结算(幂等重放上报),吸收处理并补写种子",
|
||||
report.task_id
|
||||
);
|
||||
idempotent = true;
|
||||
(p, w)
|
||||
}
|
||||
Ok(None) => {
|
||||
// M3 修复:归属校验落空 ≠ token 失效。verify 落空且非本节点已结算,
|
||||
// 通常是因为任务被 requeue_stale_tasks 重投后被**其他节点**重新领用
|
||||
// (或已由他节点结算)——迟到冗余结果应被识别为"结果被弃",而非引导
|
||||
// 运维去换 token。返回 409 Conflict 与 401/403(真·鉴权失败)区分开。
|
||||
warn!(
|
||||
"任务归属校验失败:node={} 上报 task_id={} 未由本节点领用/结算(可能已被其他节点重新领用或已结算)",
|
||||
auth_node.node_id, report.task_id
|
||||
);
|
||||
return Err(crate::api::AppError::Conflict(
|
||||
"任务未由本节点领用:可能已被其他节点重新领用或已结算".to_string(),
|
||||
));
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("校验任务已结算状态数据库异常: {}", e);
|
||||
return Err(crate::api::AppError::Internal(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("校验任务归属数据库异常: {}", e);
|
||||
@@ -159,7 +202,9 @@ pub async fn report_task(
|
||||
// 统一以鉴权 node_id 覆盖 body 里的 node_id,保证归因可信
|
||||
report.node_id = auth_node.node_id.clone();
|
||||
// workflow_name 以领用记录为准(claim 时从 TaskSpec 落库),body 无权声称。
|
||||
let workflow_name = claimed_workflow.unwrap_or_default();
|
||||
// 旧版任务(payload 无 workflow_name)归一到 '__legacy__',定向更新主库 legacy 网格点
|
||||
// (H1 修复:否则 record_task_report 按 workflow_name='' 更新 0 行,点永久卡死)。
|
||||
let workflow_name = crate::db::normalize_workflow_name(claimed_workflow.as_deref());
|
||||
|
||||
let name = report.point_name.clone();
|
||||
|
||||
@@ -187,19 +232,31 @@ pub async fn report_task(
|
||||
};
|
||||
|
||||
// Record in DB(带 workflow_name 定向更新该工作流的 grid_points)
|
||||
if let Err(e) = state.db.record_task_report(&report, &workflow_name).await {
|
||||
// DB 错误细节进日志,对客户端只返回通用消息(避免泄露表结构/内部错误给未授权方)
|
||||
tracing::error!("记录网格点 {} 任务结果到数据库失败: {}", name, e);
|
||||
return Err(crate::api::AppError::Internal(e));
|
||||
}
|
||||
// state_changed:网格点状态是否实际迁移。被终态守卫吸收的重复报告返回 false,
|
||||
// 下方种子回退据此跳过,杜绝重复失败报告触发多余 seed_step(2026-08-02 事故修复)。
|
||||
// 幂等重放(idempotent=true):首轮已结算,跳过结算与队列清理(终态守卫已吸收)。
|
||||
let state_changed = if idempotent {
|
||||
false
|
||||
} else {
|
||||
match state.db.record_task_report(&report, &workflow_name).await {
|
||||
Ok(changed) => changed,
|
||||
Err(e) => {
|
||||
// DB 错误细节进日志,对客户端只返回通用消息(避免泄露表结构/内部错误给未授权方)
|
||||
tracing::error!("记录网格点 {} 任务结果到数据库失败: {}", name, e);
|
||||
return Err(crate::api::AppError::Internal(e));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Clean up task from task_queue table to prevent queue DB bloat
|
||||
if let Err(e) = state.queue.remove_task(&report.task_id.to_string()).await {
|
||||
tracing::warn!(
|
||||
"从任务队列中清理已上报任务记录 {} 失败: {}",
|
||||
report.task_id,
|
||||
e
|
||||
);
|
||||
if !idempotent {
|
||||
if let Err(e) = state.queue.remove_task(&report.task_id.to_string()).await {
|
||||
tracing::warn!(
|
||||
"从任务队列中清理已上报任务记录 {} 失败: {}",
|
||||
report.task_id,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 采用原子写入模式保持 conv.json 与核心二进制数据完整落地后才揭晓真实文件名
|
||||
@@ -240,19 +297,29 @@ pub async fn report_task(
|
||||
}
|
||||
}
|
||||
|
||||
if !report.converged
|
||||
|| report.atmosphere_has_nan
|
||||
|| report.status == TaskStatus::Failed
|
||||
|| report.status == TaskStatus::Timeout
|
||||
if state_changed
|
||||
&& (!report.converged
|
||||
|| report.atmosphere_has_nan
|
||||
|| report.status == TaskStatus::Failed
|
||||
|| report.status == TaskStatus::Timeout)
|
||||
{
|
||||
// Task did not succeed -> check if seed_step fallback should be triggered
|
||||
info!("网格点 {} 计算未成功完成,检查种子回退机制...", name);
|
||||
// Task did not succeed -> check if strategy chain fallback should be triggered.
|
||||
// 仅在网格点状态实际迁移时才检查回退:被终态守卫吸收的迟到/重复失败报告
|
||||
// (state_changed=false)不再触发,避免重复派发(2026-08-02 涡旋事故修复)。
|
||||
// 失败阶段归因(docs/task_engine_decoupling_design.md §4.2):节点据 summary 推断
|
||||
// 失败发生在 TLUSTY 还是 SYNSPEC,服务端弹对应策略链;旧节点不携带该字段 →
|
||||
// 兜底 "tlusty"(行为与旧版一致)。
|
||||
let failed_stage = report.failed_stage.as_deref().unwrap_or("tlusty");
|
||||
info!(
|
||||
"网格点 {} 计算未成功完成(失败阶段: {}),检查策略链回退机制...",
|
||||
name, failed_stage
|
||||
);
|
||||
if let Err(e) = state
|
||||
.scheduler
|
||||
.trigger_seed_step_fallback(¶ms, &name, &workflow_name)
|
||||
.trigger_strategy_fallback(¶ms, &name, &workflow_name, failed_stage)
|
||||
.await
|
||||
{
|
||||
warn!("网格点 {} 触发种子回退机制失败: {}", name, e);
|
||||
warn!("网格点 {} 触发策略链回退机制失败: {}", name, e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -402,9 +469,17 @@ pub async fn import_seed(
|
||||
// 1. 幂等写入 grid_points(ON CONFLICT DO NOTHING):无需事先 start 工作流。
|
||||
// 用权威 name(旧 conv.json 的源精度真名),而非从 params 重推——导入路径的
|
||||
// params 来自旧 JSON(无源文本,model_name() 会失真)。
|
||||
// wave 修复(2026-08-04):此前硬编码 wave=0,导入点全部被错误归入第一波,
|
||||
// 前端难度波次推进显示错误。现按 initialize_grid 同口径计算波次(该工作流内
|
||||
// cno_sum 严格小于本点的去重值个数),新导入点落库即归入正确波次。
|
||||
let wave = state
|
||||
.db
|
||||
.compute_wave_for_cno_sum(&workflow_name, params.cno_sum())
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
if let Err(e) = state
|
||||
.db
|
||||
.upsert_grid_point_named(&name, ¶ms, 0, &workflow_name)
|
||||
.upsert_grid_point_named(&name, ¶ms, wave, &workflow_name)
|
||||
.await
|
||||
{
|
||||
tracing::error!("历史种子导入:upsert grid_points {} 失败: {}", name, e);
|
||||
@@ -469,7 +544,9 @@ pub async fn import_seed(
|
||||
|
||||
info!(
|
||||
"历史种子导入完成:网格点 {} (workflow={}, converged={}, success_method={}, max_relc={:?})",
|
||||
name, workflow_name, converged,
|
||||
name,
|
||||
workflow_name,
|
||||
converged,
|
||||
success_method.as_deref().unwrap_or("(default cold_run)"),
|
||||
max_relc
|
||||
);
|
||||
|
||||
@@ -96,11 +96,36 @@ pub async fn save_workflow(
|
||||
}
|
||||
|
||||
// Validate YAML config string(用源精度解析,校验 + 保留 grid 轴书写小数位)
|
||||
if let Err(e) = GridConfig::from_yaml_str(&req.config_yaml) {
|
||||
return Err(crate::api::AppError::BadRequest(format!(
|
||||
"无效的 YAML 配置: {}",
|
||||
e
|
||||
)));
|
||||
let cfg = match GridConfig::from_yaml_str(&req.config_yaml) {
|
||||
Ok(cfg) => cfg,
|
||||
Err(e) => {
|
||||
return Err(crate::api::AppError::BadRequest(format!(
|
||||
"无效的 YAML 配置: {}",
|
||||
e
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
// 阶段配置合法性校验(见 docs/task_engine_decoupling_design.md §3,修复审查 #4/#5):
|
||||
// - 启用的阶段必须配置 ≥1 个策略(空链 → 调度无顺位可派,任务必失败);
|
||||
// - 至少一个阶段启用(双关 → 无任何计算可执行,任务必失败)。
|
||||
// 校验基于 resolve_* 的最终生效配置(含旧字段推断),口径与调度器一致。
|
||||
let tlusty_cfg = cfg.resolve_tlusty_config();
|
||||
let synspec_cfg = cfg.resolve_synspec_config();
|
||||
if !tlusty_cfg.enabled && !synspec_cfg.enabled {
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"TLUSTY 与 SYNSPEC 阶段均被禁用:至少应启用一个计算阶段".to_string(),
|
||||
));
|
||||
}
|
||||
if tlusty_cfg.enabled && tlusty_cfg.strategies.is_empty() {
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"TLUSTY 阶段已启用但策略链为空:至少需要 1 个策略(如 cold_run)".to_string(),
|
||||
));
|
||||
}
|
||||
if synspec_cfg.enabled && synspec_cfg.strategies.is_empty() {
|
||||
return Err(crate::api::AppError::BadRequest(
|
||||
"SYNSPEC 阶段已启用但策略链为空:至少需要 1 个策略(如 standard)".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// 检查被编辑的工作流是否正处于激活运行中
|
||||
@@ -304,7 +329,13 @@ pub struct ProgressQuery {
|
||||
///
|
||||
/// - `series`:窗口内的计数快照(超 300 点自动降采样,首末点保留);
|
||||
/// - `now`:服务端当前 UTC 时刻(同 ts 格式),供前端把曲线右缘锚定为“现在”、横轴按真实时间铺开;
|
||||
/// - `rate_per_hour`:窗口首末 converged 增量 ÷ 时长(快照 <2 条或时长 ≤0 为 null);
|
||||
/// - `rate_per_hour`:**最近 2 小时**的平均收敛速率(点/小时)。取末快照往前 2h 子窗口的
|
||||
/// 首末 converged 增量 ÷ 实际跨度;子窗口不足 2 个快照时回退整窗。只看近 2h 是因为速率
|
||||
/// 首要用于 ETA,应由当前吞吐驱动,而非被整窗(最长 24h)的早期快慢/停滞抹平;
|
||||
/// - `done_rate_per_hour`:终态完成(converged+failed)的同口径近 2h 平均速率,即队列实际
|
||||
/// 清空速率,供前端 ETA 使用(剩余点数已同时扣除 converged 与 failed,口径须一致);
|
||||
/// - `rate_span_hours`:速率统计的实际时间跨度(近 2h 子窗口首末间隔,≤2h;回退整窗时为
|
||||
/// 整窗跨度)。前端须按实际跨度展示,避免把短数据跨度误读为固定 2h 或整窗平均;
|
||||
/// - `stalled_minutes`:终态数(converged+failed)最后一次增长到窗口末端的分钟数
|
||||
/// (用于"进度停滞"预警;快照 <2 条为 null)。
|
||||
pub async fn get_workflow_progress(
|
||||
@@ -346,21 +377,72 @@ pub async fn get_workflow_progress(
|
||||
// SQLite datetime('now') 为 UTC 'YYYY-MM-DD HH:MM:SS'
|
||||
let parse_ts = |ts: &str| chrono::NaiveDateTime::parse_from_str(ts, "%Y-%m-%d %H:%M:%S").ok();
|
||||
|
||||
let rate_per_hour: Option<f64> = match (series.first(), series.last()) {
|
||||
(Some(first), Some(last)) if series.len() >= 2 => {
|
||||
match (parse_ts(&first.ts), parse_ts(&last.ts)) {
|
||||
(Some(t0), Some(t1)) => {
|
||||
let dh = (t1 - t0).num_seconds() as f64 / 3600.0;
|
||||
if dh > 0.0 {
|
||||
Some((last.converged - first.converged) as f64 / dh)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
// 经验速率 = 最近 2 小时的平均速率(点/小时)。
|
||||
// 为什么只看近 2h 而非整窗:速率的首要用途是 ETA(“剩余点还要多久”),应由**当前
|
||||
// 吞吐**驱动。整窗(最长 24h)平均会把早期的快/慢、中途停滞一并抹平,对“接下来”的
|
||||
// 预测失真;近 2h 平均更贴近当下的真实处理速度。
|
||||
const RATE_WINDOW_SEC: i64 = 2 * 3600;
|
||||
|
||||
// 近 2h 子窗口起点下标:升序序列中首个「距末快照 ≤2h」的快照。用末快照(而非 now)
|
||||
// 锚定,使停滞期不被算进窗口尾部。子窗口不足 2 个快照(极慢/刚启动/长期停滞)时
|
||||
// 回退整窗,尽量仍给出数值。
|
||||
let recent_start: usize = if series.len() >= 2 {
|
||||
let s = match parse_ts(&series[series.len() - 1].ts) {
|
||||
Some(t_last) => series
|
||||
.iter()
|
||||
.position(|p| {
|
||||
parse_ts(&p.ts)
|
||||
.map(|t| (t_last - t).num_seconds() <= RATE_WINDOW_SEC)
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(0),
|
||||
None => 0,
|
||||
};
|
||||
if series.len() - s < 2 {
|
||||
0
|
||||
} else {
|
||||
s
|
||||
}
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
// 子窗口首末增量 ÷ 实际跨度 = 时段平均速率(弦斜率)。跨度即 rate_span_hours(≤2h,
|
||||
// 回退整窗时为整窗跨度),前端据此展示“近 X 小时平均”,不暗示固定 2h 或整窗。
|
||||
let avg_per_hour = |count: fn(&crate::db::ProgressPoint) -> i64| -> Option<f64> {
|
||||
let sub = &series[recent_start..];
|
||||
if sub.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
let first = sub.first()?;
|
||||
let last = sub.last()?;
|
||||
let t0 = parse_ts(&first.ts)?;
|
||||
let t1 = parse_ts(&last.ts)?;
|
||||
let dh = (t1 - t0).num_seconds() as f64 / 3600.0;
|
||||
if dh > 0.0 {
|
||||
Some((count(last) - count(first)) as f64 / dh)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
let rate_per_hour = avg_per_hour(|p| p.converged);
|
||||
let done_rate_per_hour = avg_per_hour(|p| p.converged + p.failed);
|
||||
|
||||
// 速率统计的实际时间跨度(近 2h 子窗口首末间隔,≤2h;回退整窗时为整窗跨度)。
|
||||
let rate_span_hours: Option<f64> = {
|
||||
let sub = &series[recent_start..];
|
||||
match (sub.first(), sub.last()) {
|
||||
(Some(first), Some(last)) if sub.len() >= 2 => {
|
||||
match (parse_ts(&first.ts), parse_ts(&last.ts)) {
|
||||
(Some(t0), Some(t1)) => {
|
||||
Some(((t1 - t0).num_seconds() as f64 / 3600.0).max(0.0))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let stalled_minutes: Option<f64> = if series.len() >= 2 {
|
||||
@@ -396,10 +478,12 @@ pub async fn get_workflow_progress(
|
||||
message: "成功获取进度时间序列".to_string(),
|
||||
data: Some(serde_json::json!({
|
||||
"hours": hours,
|
||||
"now": now,
|
||||
"series": series,
|
||||
"rate_per_hour": rate_per_hour,
|
||||
"stalled_minutes": stalled_minutes,
|
||||
"now": now,
|
||||
"series": series,
|
||||
"rate_per_hour": rate_per_hour,
|
||||
"done_rate_per_hour": done_rate_per_hour,
|
||||
"rate_span_hours": rate_span_hours,
|
||||
"stalled_minutes": stalled_minutes,
|
||||
})),
|
||||
}),
|
||||
))
|
||||
@@ -481,6 +565,13 @@ pub async fn get_workflow_points(
|
||||
"teff" => format!("gp.teff {dir}, gp.wave ASC, gp.cno_sum ASC"),
|
||||
"max_relc" => format!("t.max_relc IS NULL ASC, t.max_relc {dir}, gp.wave ASC"),
|
||||
"attempts" => format!("gp.attempt_count {dir}, gp.wave ASC, gp.cno_sum ASC"),
|
||||
// 耗时取最近一次尝试的真实墙钟(与列表展示同口径 COALESCE),NULL(从未派发)靠后。
|
||||
"elapsed" => {
|
||||
format!(
|
||||
"COALESCE(t.elapsed_sec, gp.last_elapsed_sec) IS NULL ASC, \
|
||||
COALESCE(t.elapsed_sec, gp.last_elapsed_sec) {dir}, gp.wave ASC"
|
||||
)
|
||||
}
|
||||
"last_completed_at" => {
|
||||
format!("t.completed_at IS NULL ASC, t.completed_at {dir}, gp.wave ASC")
|
||||
}
|
||||
@@ -594,9 +685,26 @@ pub async fn stop_workflow(
|
||||
match state.db.update_workflow_status(&name, "paused").await {
|
||||
Ok(_) => {
|
||||
// 多工作流分区:清理与重置都限定在本工作流内,避免误伤其他并发运行的工作流。
|
||||
// - clear_queue_by_workflow:只删本工作流的排队任务。
|
||||
// - clear_queue_by_workflow:只删本工作流的排队任务,返回被删 task_id。
|
||||
// - delete_tasks_by_ids:同步清理主库 tasks 表对应 pending 行(2026-08-02
|
||||
// 涡旋事故修复:此前只删队列行遗留僵尸 pending 行,成为重复派发燃料)。
|
||||
// - reset_queued_grid_points_to_pending(&name):只把本工作流的 queued 点打回 pending。
|
||||
let _ = state.queue.clear_queue_by_workflow(&name).await;
|
||||
match state.queue.clear_queue_by_workflow(&name).await {
|
||||
Ok(cleared_ids) if !cleared_ids.is_empty() => {
|
||||
if let Err(e) = state.db.delete_tasks_by_ids(&cleared_ids).await {
|
||||
tracing::warn!(
|
||||
"暂停工作流 {} 时同步清理 {} 条被删队列任务的 tasks 历史行失败: {}",
|
||||
name,
|
||||
cleared_ids.len(),
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("暂停工作流 {} 时清理排队任务失败: {}", name, e);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
let _ = state.db.reset_queued_grid_points_to_pending(&name).await;
|
||||
Ok((
|
||||
StatusCode::OK,
|
||||
|
||||
+1767
-77
File diff suppressed because it is too large
Load Diff
+37
-25
@@ -106,22 +106,23 @@ async fn main() -> Result<()> {
|
||||
);
|
||||
for (wf_name, wf_yaml) in &stuck {
|
||||
match GridConfig::from_yaml_str(wf_yaml) {
|
||||
Ok(cfg) => {
|
||||
match scheduler.initialize_grid(&cfg, wf_name).await {
|
||||
Ok(_) => {
|
||||
let _ = db.update_workflow_status(wf_name, "running").await;
|
||||
info!("启动恢复:工作流 {} 已完成重新初始化并切回 running", wf_name);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"启动恢复:工作流 {} 重新初始化失败,回退为 idle: {}",
|
||||
wf_name,
|
||||
e
|
||||
);
|
||||
let _ = db.update_workflow_status(wf_name, "idle").await;
|
||||
}
|
||||
Ok(cfg) => match scheduler.initialize_grid(&cfg, wf_name).await {
|
||||
Ok(_) => {
|
||||
let _ = db.update_workflow_status(wf_name, "running").await;
|
||||
info!(
|
||||
"启动恢复:工作流 {} 已完成重新初始化并切回 running",
|
||||
wf_name
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"启动恢复:工作流 {} 重新初始化失败,回退为 idle: {}",
|
||||
wf_name,
|
||||
e
|
||||
);
|
||||
let _ = db.update_workflow_status(wf_name, "idle").await;
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"启动恢复:工作流 {} 的 YAML 配置解析失败,回退为 idle: {}",
|
||||
@@ -188,8 +189,11 @@ async fn main() -> Result<()> {
|
||||
let mut by_wf: std::collections::HashMap<String, Vec<String>> =
|
||||
std::collections::HashMap::new();
|
||||
for (point, wf) in &requeued {
|
||||
// 旧版任务(payload 无 workflow_name)归一到 '__legacy__',
|
||||
// 使 reset_specific_grid_points_to_pending 命中 legacy 网格点
|
||||
// (H1 修复:否则按 '' 更新 0 行,重投后的 legacy 点永远重置不回 pending)。
|
||||
by_wf
|
||||
.entry(wf.clone().unwrap_or_default())
|
||||
.entry(server::db::normalize_workflow_name(wf.as_deref()))
|
||||
.or_default()
|
||||
.push(point.clone());
|
||||
}
|
||||
@@ -218,17 +222,23 @@ async fn main() -> Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
// 孤儿 running 点回收(#6 修复兜底):queue 行已消失(误删/崩溃丢队列)
|
||||
// 但 grid_points 仍卡在 running 的点,requeue_stale_tasks 找不到它们,
|
||||
// 在此按 tasks 表的 stale pending 记录兜底重置为 pending,让调度器重新派发。
|
||||
match bg_db_clone.reset_orphaned_running_points(stale_sec).await {
|
||||
// 孤儿点回收(#6 修复兜底,2026-08-02 涡旋事故重构):queue 凭证已消失
|
||||
// (误删/崩溃丢队列/insert 后 push 前崩溃)但 grid_points 仍卡在
|
||||
// running/queued 的点,requeue_stale_tasks 找不到它们。经 MQ 活性交叉
|
||||
// 校验确认真孤儿后重置为 pending 让调度器重新派发,并清除作为判据的
|
||||
// 僵尸 tasks 行(旧实现仅凭 tasks 表 stale pending 行判定,僵尸行使
|
||||
// 判据恒真 → 重复派发涡旋,已废弃)。
|
||||
match bg_scheduler_clone.reclaim_orphaned_points(stale_sec).await {
|
||||
Ok(reset) => {
|
||||
if reset > 0 {
|
||||
info!("已回收 {} 个孤儿 running 网格点(领用凭证丢失,重置为 pending)", reset);
|
||||
info!(
|
||||
"已回收 {} 个孤儿网格点(领用凭证丢失,重置为 pending)",
|
||||
reset
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("回收孤儿 running 网格点失败: {}", e);
|
||||
tracing::warn!("回收孤儿网格点失败: {}", e);
|
||||
has_error = true;
|
||||
}
|
||||
}
|
||||
@@ -402,6 +412,10 @@ async fn main() -> Result<()> {
|
||||
"/admin/nodes/:node_id/enable",
|
||||
post(api::admin::enable_node),
|
||||
)
|
||||
.route(
|
||||
"/admin/nodes/:node_id/quota",
|
||||
post(api::admin::set_node_quota),
|
||||
)
|
||||
.layer(DefaultBodyLimit::max(DEFAULT_BODY_LIMIT));
|
||||
|
||||
// 合并两个子 router:各自携带自己的 body limit,互不覆盖。
|
||||
@@ -431,9 +445,7 @@ async fn main() -> Result<()> {
|
||||
let auth_layer = axum::middleware::from_fn_with_state(state.clone(), api::auth_middleware);
|
||||
api_router.layer(auth_layer).layer(rate_limit_layer)
|
||||
} else {
|
||||
info!(
|
||||
"DCTS_AUTH_DISABLE=1 已生效:服务端运行在无鉴权模式(仅限本地调试,切勿用于生产)。"
|
||||
);
|
||||
info!("DCTS_AUTH_DISABLE=1 已生效:服务端运行在无鉴权模式(仅限本地调试,切勿用于生产)。");
|
||||
api_router
|
||||
};
|
||||
|
||||
|
||||
+2121
-59
File diff suppressed because it is too large
Load Diff
@@ -1466,7 +1466,8 @@ async fn test_import_seed_admin_endpoint() {
|
||||
|
||||
// 4. 未收敛点 → 200,但不写 .7、grid_points 维持 pending(未建 converged)。
|
||||
let conv_fail = make_legacy_conv_json("t20000_g5.0_he-2_c-4_n-4_o-4_fail", false);
|
||||
let body_bytes = make_import_multipart("boundary4", &conv_fail, b"WONT_BE_USED", "x.7", "cold_run");
|
||||
let body_bytes =
|
||||
make_import_multipart("boundary4", &conv_fail, b"WONT_BE_USED", "x.7", "cold_run");
|
||||
let res = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
@@ -1538,8 +1539,13 @@ async fn test_import_seed_python_legacy_conv_json() {
|
||||
|
||||
let name = "t20000_g5.0_he-2_c-4_n-4_o-4";
|
||||
let conv = make_python_legacy_conv_json(name);
|
||||
let body_bytes =
|
||||
make_import_multipart("boundaryL", &conv, b"FAKE_ATMOS_7", &format!("{name}.7"), "cold_run");
|
||||
let body_bytes = make_import_multipart(
|
||||
"boundaryL",
|
||||
&conv,
|
||||
b"FAKE_ATMOS_7",
|
||||
&format!("{name}.7"),
|
||||
"cold_run",
|
||||
);
|
||||
let res = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
@@ -1667,6 +1673,7 @@ async fn test_node_disable_enable_flow() {
|
||||
timeout_sec: 60,
|
||||
workflow_name: Some("wf_test".to_string()),
|
||||
wave: 0,
|
||||
..Default::default()
|
||||
};
|
||||
queue.push_task(&task).await.unwrap();
|
||||
|
||||
@@ -1883,6 +1890,232 @@ async fn test_node_disable_enable_flow() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_admin_set_node_quota_and_heartbeat_sync() {
|
||||
// 验证动态 CPU 槽位配额下发链路(见 docs/dynamic_cpu_slots_design.md):
|
||||
// admin POST /quota 设 admin_max_slots → 落库 nodes.admin_max_slots →
|
||||
// list_nodes_with_credentials 返回配额 → 心跳响应体透传 admin_max_slots。
|
||||
// 覆盖:设值 / 清除(null) / 负数 400 / 不存在节点 404。
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let db_path = temp_dir.path().join("quota_db.db");
|
||||
let queue_db_path = temp_dir.path().join("quota_queue.db");
|
||||
let seeds_dir = temp_dir.path().join("results");
|
||||
std::fs::create_dir_all(&seeds_dir).unwrap();
|
||||
|
||||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||||
let queue = Arc::new(
|
||||
SqliteTaskQueue::new(&queue_db_path.to_string_lossy())
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let scheduler = Arc::new(GridScheduler::new(db.clone(), queue.clone()));
|
||||
|
||||
// 注册并审批 node-quota-test,颁发 token
|
||||
let reg = common::models::NodeRegisterRequest {
|
||||
node_id: "node-quota-test".to_string(),
|
||||
max_slots: 4,
|
||||
};
|
||||
db.register_node(®).await.unwrap();
|
||||
let token = db.approve_node("node-quota-test").await.unwrap();
|
||||
|
||||
let state = AppState {
|
||||
db: db.clone(),
|
||||
queue: queue.clone(),
|
||||
scheduler,
|
||||
seeds_dir: seeds_dir.to_string_lossy().to_string(),
|
||||
rate_limiter: server::api::rate_limit::RateLimiter::new(
|
||||
5,
|
||||
std::time::Duration::from_secs(300),
|
||||
),
|
||||
admin_token: Some("admin-secret".to_string()),
|
||||
auth_disabled: false,
|
||||
admin_sessions: std::sync::Arc::new(tokio::sync::RwLock::new(
|
||||
std::collections::HashMap::new(),
|
||||
)),
|
||||
};
|
||||
|
||||
let api_router = axum::Router::new()
|
||||
.route(
|
||||
"/node/heartbeat",
|
||||
axum::routing::post(server::api::node::heartbeat_node),
|
||||
)
|
||||
.route(
|
||||
"/admin/nodes/:node_id/quota",
|
||||
axum::routing::post(server::api::admin::set_node_quota),
|
||||
);
|
||||
let auth_layer =
|
||||
axum::middleware::from_fn_with_state(state.clone(), server::api::auth_middleware);
|
||||
let app = axum::Router::new()
|
||||
.nest("/api", api_router.layer(auth_layer))
|
||||
.with_state(state);
|
||||
|
||||
// 1. 基线心跳:admin_max_slots 应为 null(无配额限制)
|
||||
let hb = serde_json::json!({"node_id":"node-quota-test","active_slots":0,"cpu_usage":10.0,"memory_usage":20.0});
|
||||
let res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/node/heartbeat")
|
||||
.header("authorization", format!("Bearer {}", token))
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(serde_json::to_vec(&hb).unwrap()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
let body: serde_json::Value = serde_json::from_slice(
|
||||
&axum::body::to_bytes(res.into_body(), usize::MAX)
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(body["status"], "ok");
|
||||
assert!(
|
||||
body["admin_max_slots"].is_null(),
|
||||
"未设配额时心跳响应 admin_max_slots 应为 null"
|
||||
);
|
||||
|
||||
// 2. admin 设配额 = 2 → 200
|
||||
let res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/admin/nodes/node-quota-test/quota")
|
||||
.header("authorization", "Bearer admin-secret")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
serde_json::to_vec(&serde_json::json!({"admin_max_slots": 2})).unwrap(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
|
||||
// 2a. 配额落库(list_nodes_with_credentials)
|
||||
let nodes = db.list_nodes_with_credentials().await.unwrap();
|
||||
let me = nodes
|
||||
.iter()
|
||||
.find(|n| n.node_id == "node-quota-test")
|
||||
.unwrap();
|
||||
assert_eq!(me.admin_max_slots, Some(2), "配额应已落库为 Some(2)");
|
||||
|
||||
// 3. 心跳响应透传 admin_max_slots = 2
|
||||
let res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/node/heartbeat")
|
||||
.header("authorization", format!("Bearer {}", token))
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(serde_json::to_vec(&hb).unwrap()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
let body: serde_json::Value = serde_json::from_slice(
|
||||
&axum::body::to_bytes(res.into_body(), usize::MAX)
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(body["admin_max_slots"], 2, "心跳响应应透传配额 2");
|
||||
|
||||
// 3a. 请求体缺 admin_max_slots 字段(畸形输入)→ 400,绝不能静默当作「清除配额」
|
||||
// (审查修复 M1:Option<i32> 缺字段默认 None 会把畸形请求误判为清除限制)。
|
||||
let res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/admin/nodes/node-quota-test/quota")
|
||||
.header("authorization", "Bearer admin-secret")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
serde_json::to_vec(&serde_json::json!({})).unwrap(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
res.status(),
|
||||
StatusCode::BAD_REQUEST,
|
||||
"缺字段应 400 而非静默清除配额"
|
||||
);
|
||||
// 畸形请求无副作用:既有配额仍为 Some(2),未被静默清除
|
||||
let nodes = db.list_nodes_with_credentials().await.unwrap();
|
||||
let me = nodes
|
||||
.iter()
|
||||
.find(|n| n.node_id == "node-quota-test")
|
||||
.unwrap();
|
||||
assert_eq!(me.admin_max_slots, Some(2), "畸形请求不应改动既有配额");
|
||||
|
||||
// 4. 设 null 清除配额 → 心跳响应 admin_max_slots = null
|
||||
let res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/admin/nodes/node-quota-test/quota")
|
||||
.header("authorization", "Bearer admin-secret")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
serde_json::to_vec(&serde_json::json!({"admin_max_slots": null})).unwrap(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
let nodes = db.list_nodes_with_credentials().await.unwrap();
|
||||
let me = nodes
|
||||
.iter()
|
||||
.find(|n| n.node_id == "node-quota-test")
|
||||
.unwrap();
|
||||
assert_eq!(me.admin_max_slots, None, "清除后配额应为 None");
|
||||
|
||||
// 5. 负数 → 400
|
||||
let res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/admin/nodes/node-quota-test/quota")
|
||||
.header("authorization", "Bearer admin-secret")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
serde_json::to_vec(&serde_json::json!({"admin_max_slots": -1})).unwrap(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::BAD_REQUEST, "负数配额应 400");
|
||||
|
||||
// 6. 不存在节点 → 404
|
||||
let res = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/admin/nodes/ghost-node/quota")
|
||||
.header("authorization", "Bearer admin-secret")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
serde_json::to_vec(&serde_json::json!({"admin_max_slots": 1})).unwrap(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::NOT_FOUND, "不存在节点应 404");
|
||||
}
|
||||
|
||||
// ===== 工作流执行观测端点测试(stats / points / point detail) =====
|
||||
|
||||
/// 测试专用:走真实写路径派发并回报一个网格点任务。
|
||||
@@ -1907,6 +2140,7 @@ async fn dispatch_and_report(
|
||||
timeout_sec: 7200,
|
||||
workflow_name: Some(wf.to_string()),
|
||||
wave: 0,
|
||||
..Default::default()
|
||||
};
|
||||
db.insert_task(&spec).await.unwrap();
|
||||
let report = common::models::TaskReport {
|
||||
@@ -1929,6 +2163,7 @@ async fn dispatch_and_report(
|
||||
Some("nl stage diverged".to_string())
|
||||
},
|
||||
summary_json: "{}".to_string(),
|
||||
failed_stage: None,
|
||||
};
|
||||
db.record_task_report(&report, wf).await.unwrap();
|
||||
}
|
||||
@@ -2844,3 +3079,340 @@ async fn test_wf_progress_endpoint() {
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
/// 构造仅含 report 字段的 multipart body(模拟节点上报,不带 seed_file)。
|
||||
fn make_report_only_multipart(boundary: &str, report_json: &str) -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes());
|
||||
body.extend_from_slice(b"Content-Disposition: form-data; name=\"report\"\r\n");
|
||||
body.extend_from_slice(b"Content-Type: application/json\r\n\r\n");
|
||||
body.extend_from_slice(report_json.as_bytes());
|
||||
body.extend_from_slice(b"\r\n");
|
||||
body.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes());
|
||||
body
|
||||
}
|
||||
|
||||
/// 端到端回归(2026-08-02 涡旋事故):已收敛点收到迟到的重复失败报告时,
|
||||
/// 走完整 HTTP 链路(claim → report)后状态保持 converged,且不触发种子回退
|
||||
/// (state_changed=false 跳过 fallback;纵有可用种子也不产生 seed_step 任务)。
|
||||
#[tokio::test]
|
||||
async fn test_duplicate_failure_report_cannot_flip_converged() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let db_path = temp_dir.path().join("flip_db.db");
|
||||
let queue_db_path = temp_dir.path().join("flip_queue.db");
|
||||
let seeds_dir = temp_dir.path().join("results");
|
||||
std::fs::create_dir_all(&seeds_dir).unwrap();
|
||||
|
||||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||||
let queue = Arc::new(
|
||||
SqliteTaskQueue::new(&queue_db_path.to_string_lossy())
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let scheduler = Arc::new(GridScheduler::new(db.clone(), queue.clone()));
|
||||
|
||||
// 预置节点与 token。
|
||||
let reg = common::models::NodeRegisterRequest {
|
||||
node_id: "node-flip".to_string(),
|
||||
max_slots: 2,
|
||||
};
|
||||
db.register_node(®).await.unwrap();
|
||||
let token = db.issue_node_token("node-flip").await.unwrap();
|
||||
|
||||
let state = AppState {
|
||||
db: db.clone(),
|
||||
queue: queue.clone(),
|
||||
scheduler,
|
||||
seeds_dir: seeds_dir.to_string_lossy().to_string(),
|
||||
rate_limiter: server::api::rate_limit::RateLimiter::new(
|
||||
100,
|
||||
std::time::Duration::from_secs(300),
|
||||
),
|
||||
admin_token: None,
|
||||
auth_disabled: false,
|
||||
admin_sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
|
||||
};
|
||||
let api_router = axum::Router::new()
|
||||
.route(
|
||||
"/task/claim",
|
||||
axum::routing::post(server::api::task::claim_task),
|
||||
)
|
||||
.route(
|
||||
"/task/report",
|
||||
axum::routing::post(server::api::task::report_task),
|
||||
);
|
||||
let auth_layer =
|
||||
axum::middleware::from_fn_with_state(state.clone(), server::api::auth_middleware);
|
||||
let app = axum::Router::new()
|
||||
.nest("/api", api_router.layer(auth_layer))
|
||||
.with_state(state);
|
||||
|
||||
// 工作流 running + 网格点 + 可用种子(若回退被错误触发,必然能匹配到种子并派发任务,
|
||||
// 使"无 seed_step 任务"断言成为强证据)。
|
||||
db.upsert_workflow("wf_flip", None, "", "running")
|
||||
.await
|
||||
.unwrap();
|
||||
let params = common::models::GridPointParams {
|
||||
teff: 35000.0.into(),
|
||||
logg: 5.5.into(),
|
||||
loghe: (-1.0).into(),
|
||||
logc: (-2.0).into(),
|
||||
logn: (-2.0).into(),
|
||||
logo: (-2.0).into(),
|
||||
};
|
||||
let name = params.model_name();
|
||||
db.upsert_grid_point(¶ms, 0, "wf_flip").await.unwrap();
|
||||
db.insert_seed_named(&name, ¶ms, "/tmp/flip_seed.7")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// ---- 任务 A:正常收敛 ----
|
||||
let task_a = common::models::TaskSpec {
|
||||
task_id: uuid::Uuid::new_v4(),
|
||||
point_name: name.clone(),
|
||||
params: params.clone(),
|
||||
task_type: common::models::TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 7200,
|
||||
workflow_name: Some("wf_flip".to_string()),
|
||||
wave: 0,
|
||||
..Default::default()
|
||||
};
|
||||
db.insert_task(&task_a).await.unwrap();
|
||||
queue.push_task(&task_a).await.unwrap();
|
||||
|
||||
let claim_res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/task/claim")
|
||||
.header("authorization", format!("Bearer {}", token))
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(claim_res.status(), StatusCode::OK);
|
||||
|
||||
let report_a = common::models::TaskReport {
|
||||
task_id: task_a.task_id,
|
||||
point_name: name.clone(),
|
||||
params: Some(params.clone()),
|
||||
node_id: "node-flip".to_string(),
|
||||
status: common::models::TaskStatus::Completed,
|
||||
converged: true,
|
||||
max_relc: Some(0.0005),
|
||||
atmosphere_has_nan: false,
|
||||
elapsed_sec: 120.0,
|
||||
error_message: None,
|
||||
summary_json: "{}".to_string(),
|
||||
failed_stage: None,
|
||||
};
|
||||
let body_a =
|
||||
make_report_only_multipart("flipbound1", &serde_json::to_string(&report_a).unwrap());
|
||||
let report_res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/task/report")
|
||||
.header("authorization", format!("Bearer {}", token))
|
||||
.header("content-type", "multipart/form-data; boundary=flipbound1")
|
||||
.body(Body::from(body_a))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(report_res.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
db.get_grid_point_status(&name, "wf_flip")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.0,
|
||||
"converged"
|
||||
);
|
||||
|
||||
// ---- 任务 B:迟到的重复失败报告(涡旋残留任务的典型行为)----
|
||||
let task_b = common::models::TaskSpec {
|
||||
task_id: uuid::Uuid::new_v4(),
|
||||
point_name: name.clone(),
|
||||
params: params.clone(),
|
||||
task_type: common::models::TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 7200,
|
||||
workflow_name: Some("wf_flip".to_string()),
|
||||
wave: 0,
|
||||
..Default::default()
|
||||
};
|
||||
db.insert_task(&task_b).await.unwrap();
|
||||
queue.push_task(&task_b).await.unwrap();
|
||||
|
||||
let claim_b = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/task/claim")
|
||||
.header("authorization", format!("Bearer {}", token))
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(claim_b.status(), StatusCode::OK);
|
||||
|
||||
let report_b = common::models::TaskReport {
|
||||
task_id: task_b.task_id,
|
||||
point_name: name.clone(),
|
||||
params: Some(params.clone()),
|
||||
node_id: "node-flip".to_string(),
|
||||
status: common::models::TaskStatus::Failed,
|
||||
converged: false,
|
||||
max_relc: Some(9.5e5),
|
||||
atmosphere_has_nan: false,
|
||||
elapsed_sec: 130.0,
|
||||
error_message: Some("nl stage diverged".to_string()),
|
||||
summary_json: "{}".to_string(),
|
||||
failed_stage: None,
|
||||
};
|
||||
let body_b =
|
||||
make_report_only_multipart("flipbound2", &serde_json::to_string(&report_b).unwrap());
|
||||
let report_b_res = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/task/report")
|
||||
.header("authorization", format!("Bearer {}", token))
|
||||
.header("content-type", "multipart/form-data; boundary=flipbound2")
|
||||
.body(Body::from(body_b))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
report_b_res.status(),
|
||||
StatusCode::OK,
|
||||
"上报本身应成功(吸收)"
|
||||
);
|
||||
|
||||
// 核心断言:converged 不被翻黑,且未触发任何种子回退任务。
|
||||
assert_eq!(
|
||||
db.get_grid_point_status(&name, "wf_flip")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.0,
|
||||
"converged",
|
||||
"迟到失败报告不得翻黑 converged 点"
|
||||
);
|
||||
assert!(
|
||||
db.has_pending_tasks_for_point(&name, "wf_flip", Some("seed_step"))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"不得因迟到失败报告派发 seed_step"
|
||||
);
|
||||
assert!(
|
||||
!db.has_seed_step_attempt(&name, "wf_flip").await.unwrap(),
|
||||
"全程不应产生任何 seed_step 行"
|
||||
);
|
||||
}
|
||||
|
||||
/// save_workflow 阶段配置合法性校验(修复审查 #4/#5):
|
||||
/// - 双阶段全关 → 400(无任何计算可执行);
|
||||
/// - 启用阶段空策略链 → 400(调度无顺位可派,任务必失败);
|
||||
/// - 合法配置(synspec-only 场景 B / 默认双开)→ 200。
|
||||
#[tokio::test]
|
||||
async fn test_save_workflow_validates_stage_configs() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let db_path = temp_dir.path().join("wfval_db.db");
|
||||
let queue_db_path = temp_dir.path().join("wfval_queue.db");
|
||||
let seeds_dir = temp_dir.path().join("results");
|
||||
std::fs::create_dir_all(&seeds_dir).unwrap();
|
||||
|
||||
let db = Database::new(&db_path.to_string_lossy()).await.unwrap();
|
||||
let queue = Arc::new(
|
||||
SqliteTaskQueue::new(&queue_db_path.to_string_lossy())
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let scheduler = Arc::new(GridScheduler::new(db.clone(), queue.clone()));
|
||||
let state = AppState {
|
||||
db: db.clone(),
|
||||
queue,
|
||||
scheduler,
|
||||
seeds_dir: seeds_dir.to_string_lossy().to_string(),
|
||||
rate_limiter: server::api::rate_limit::RateLimiter::new(
|
||||
5,
|
||||
std::time::Duration::from_secs(300),
|
||||
),
|
||||
admin_token: Some("admin-secret".to_string()),
|
||||
auth_disabled: false,
|
||||
admin_sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
|
||||
};
|
||||
let app = axum::Router::new()
|
||||
.route(
|
||||
"/api/workflows",
|
||||
axum::routing::post(server::api::workflow::save_workflow),
|
||||
)
|
||||
.with_state(state);
|
||||
|
||||
let base_yaml = "grid:\n teff: [35000]\n logg: [5.5]\n loghe: [-1]\n logc: [-2]\n logn: [-2]\n logo: [-2]";
|
||||
let post_save = |name: &str, yaml: &str| {
|
||||
let app = app.clone();
|
||||
let body = serde_json::json!({
|
||||
"name": name,
|
||||
"description": null,
|
||||
"config_yaml": yaml,
|
||||
});
|
||||
async move {
|
||||
app.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/workflows")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(serde_json::to_vec(&body).unwrap()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
};
|
||||
|
||||
// 1. 双阶段全关 → 400
|
||||
let both_off = format!(
|
||||
"{}\ntlusty:\n enabled: false\n policy: skip_converged\n strategies: [cold_run, seed_step]\n\
|
||||
synspec_stage:\n enabled: false\n policy: skip_converged\n strategies: [standard]\n",
|
||||
base_yaml
|
||||
);
|
||||
let res = post_save("wf_val_a", &both_off).await;
|
||||
assert_eq!(res.status(), StatusCode::BAD_REQUEST, "双阶段全关应 400");
|
||||
|
||||
// 2. 启用阶段空策略链 → 400
|
||||
let empty_chain = format!(
|
||||
"{}\ntlusty:\n enabled: true\n policy: skip_converged\n strategies: []\n",
|
||||
base_yaml
|
||||
);
|
||||
let res = post_save("wf_val_b", &empty_chain).await;
|
||||
assert_eq!(
|
||||
res.status(),
|
||||
StatusCode::BAD_REQUEST,
|
||||
"启用阶段空策略链应 400"
|
||||
);
|
||||
|
||||
// 3. 合法:TLUSTY 关 + SYNSPEC 启(设计 §2.2 场景 B:仅更新光谱)→ 200
|
||||
let syn_only = format!(
|
||||
"{}\ntlusty:\n enabled: false\n policy: skip_converged\n strategies: [cold_run, seed_step]\n\
|
||||
synspec_stage:\n enabled: true\n policy: force_recompute\n strategies: [standard]\n",
|
||||
base_yaml
|
||||
);
|
||||
let res = post_save("wf_val_c", &syn_only).await;
|
||||
assert_eq!(res.status(), StatusCode::OK, "synspec-only 合法配置应 200");
|
||||
|
||||
// 4. 合法:无阶段块(默认双开)→ 200
|
||||
let res = post_save("wf_val_d", base_yaml).await;
|
||||
assert_eq!(res.status(), StatusCode::OK, "默认配置应 200");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user