feat(all): 数据库模块化拆分与版本化迁移、任务引擎命名体系收敛、物理输出校验加固与用户配置接通
- server/db: 拆 4929 行 db.rs 单体为 db/ 目录，migrations.rs 引入 PRAGMA user_version
版本化迁移运行器（M1~M13）
- 任务引擎 Phase 6/7b/7c 改名收敛：EngineStageConfig→PhaseConfig、StagePolicy→ResumePolicy、
Converged→Completed、删除 task_type 列、success_method 拆 tlusty_/synspec_ 双列、
新增 tlusty_status/synspec_status 半失败阶段守卫
- 科学正确性加固：conv_check 任意行 NaN/Inf/溢出判无效（0 行容忍）、新增 spec_is_valid
校验 SYNSPEC 脏谱、itek_history 逐次迭代全量保真、fmt_abn powf 溢出饱和
- 用户配置真正接通：tlusty_chain/tlusty_input 由死字段经 调度器→TaskSpec→executor→runner
透传生效；config 加载期 validate + deny_unknown_fields + 解析失败记 warn
- 调度修复：H1 活锁（pending_strategies 跳过已失败策略）、种子查找错误不再静默降级冷启动
- dashboard: 阶段配置面板 tlusty_stage/synspec_stage、"已完成"标签、迭代诊断展示
- docs: 新增 database_refactor_design.md，同步 database/api/PIPELINE/workflow_detail
This commit is contained in:
@@ -13,7 +13,16 @@ pub async fn healthz() -> Result<impl IntoResponse, crate::api::AppError> {
|
||||
pub async fn get_status(
|
||||
State(state): State<AppState>,
|
||||
) -> Result<impl IntoResponse, crate::api::AppError> {
|
||||
let nodes = state.db.get_active_nodes().await.unwrap_or_default();
|
||||
// 审查修复 #N6：原 unwrap_or_default 吞掉 DB 错误，前端显示「0 节点在线」掩盖真实故障。
|
||||
// 现至少记 warn，让运维能在日志里定位「dashboard 为何显示无节点」。仍降级为空列表
|
||||
// （返回 500 会让 dashboard 整个 status 面板不可用，空列表是更安全的退化）。
|
||||
let nodes = match state.db.get_active_nodes().await {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
tracing::warn!("get_status 查询活跃节点失败，降级为空列表: {}", e);
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
let total_active_slots: i32 = nodes.iter().map(|n| n.active_slots).sum();
|
||||
let total_max_slots: i32 = nodes.iter().map(|n| n.max_slots).sum();
|
||||
|
||||
@@ -24,8 +33,8 @@ pub async fn get_status(
|
||||
.get_grid_summary_stats(None)
|
||||
.await
|
||||
.unwrap_or(serde_json::json!({
|
||||
"total": 0, "pending": 0, "queued": 0, "running": 0, "converged": 0, "failed": 0,
|
||||
"cold_run_converged": 0, "seed_step_converged": 0
|
||||
"total": 0, "pending": 0, "queued": 0, "running": 0, "completed": 0, "failed": 0,
|
||||
"cold_run_converged": 0, "seed_step_converged": 0, "synspec_converged": 0
|
||||
}));
|
||||
|
||||
Ok(Json(json!({
|
||||
|
||||
@@ -46,7 +46,11 @@ pub async fn claim_task(
|
||||
// 旧版在途任务 payload 无 workflow_name（None）→ 归一到主库迁移回填的
|
||||
// '__legacy__' 标记，使 mark_grid_point_running 能命中 legacy 网格点（H1 修复）。
|
||||
let wf = crate::db::normalize_workflow_name(task.workflow_name.as_deref());
|
||||
match state.db.mark_grid_point_running(&task.point_name, &wf).await {
|
||||
match state
|
||||
.db
|
||||
.mark_grid_point_running(&task.point_name, &wf)
|
||||
.await
|
||||
{
|
||||
Ok(false) => {
|
||||
info!(
|
||||
"领用任务 {}（网格点 {}）时点已非 pending/queued 态，跳过 running 标记（迟到/重复领用）",
|
||||
@@ -269,7 +273,7 @@ pub async fn report_task(
|
||||
}
|
||||
|
||||
// Save seed file .7 using atomic temporary writing strategy
|
||||
if report.converged && !report.atmosphere_has_nan {
|
||||
if report.result_valid && !report.atmosphere_has_nan {
|
||||
if let Some(bytes) = seed_file_data {
|
||||
let seed_tmp =
|
||||
model_dir.join(format!("{}.7.{}.tmp", name, uuid::Uuid::new_v4().simple()));
|
||||
@@ -298,7 +302,7 @@ pub async fn report_task(
|
||||
}
|
||||
|
||||
if state_changed
|
||||
&& (!report.converged
|
||||
&& (!report.result_valid
|
||||
|| report.atmosphere_has_nan
|
||||
|| report.status == TaskStatus::Failed
|
||||
|| report.status == TaskStatus::Timeout)
|
||||
@@ -369,8 +373,9 @@ pub async fn import_seed(
|
||||
) -> Result<impl IntoResponse, crate::api::AppError> {
|
||||
let mut summary_json: Option<String> = None;
|
||||
let mut seed_file_data: Option<Vec<u8>> = None;
|
||||
// 收敛途径（cold_run/seed_step）：由 import_results 工具依据旧 conv.json 的 stages 是否
|
||||
// 大气收敛途径（cold_run/seed_step）：由 import_results 工具依据旧 conv.json 的 stages 是否
|
||||
// 含 seed_nc 判定后透传。缺失或非法时兜底 cold_run（容错旧版工具 / 防注入）。
|
||||
// 语义为 TLUSTY 阶段策略，写入 grid_points.tlusty_success_method。
|
||||
let mut success_method: Option<String> = None;
|
||||
let mut multipart_error = false;
|
||||
|
||||
@@ -400,13 +405,13 @@ pub async fn import_seed(
|
||||
multipart_error = true;
|
||||
}
|
||||
}
|
||||
} else if field_name == "success_method" {
|
||||
} else if field_name == "tlusty_success_method" {
|
||||
match field.text().await {
|
||||
Ok(text) => {
|
||||
success_method = Some(text);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("历史种子导入：读取 success_method 字段失败: {}", e);
|
||||
warn!("历史种子导入：读取 tlusty_success_method 字段失败: {}", e);
|
||||
multipart_error = true;
|
||||
}
|
||||
}
|
||||
@@ -463,7 +468,7 @@ pub async fn import_seed(
|
||||
|
||||
let workflow_name = query.workflow;
|
||||
let params = summary.params.clone();
|
||||
let converged = summary.converged && !summary.atmosphere_has_nan;
|
||||
let converged = summary.result_valid && !summary.atmosphere_has_nan;
|
||||
let max_relc = summary.final_max_relc;
|
||||
|
||||
// 1. 幂等写入 grid_points（ON CONFLICT DO NOTHING）：无需事先 start 工作流。
|
||||
@@ -524,7 +529,7 @@ pub async fn import_seed(
|
||||
}
|
||||
}
|
||||
|
||||
// 4. 更新 grid_points 状态：收敛→converged(success_method=工具判定的途径)；否则维持 pending
|
||||
// 4. 更新 grid_points 状态：收敛→converged(大气归因 tlusty_success_method=工具判定的途径)；否则维持 pending
|
||||
// 让正常调度处理（导入未收敛点无意义，但记录其尝试）。
|
||||
// 途径缺失或非法时兜底 cold_run（容错旧版工具 / 防注入），由 db 层再次白名单校验。
|
||||
if converged {
|
||||
@@ -556,6 +561,8 @@ pub async fn import_seed(
|
||||
Json(json!({
|
||||
"status": "ok",
|
||||
"point_name": name,
|
||||
// 注：此 "converged" 键是大气收敛标志（由 summary.result_valid 派生），与 grid_points.status
|
||||
// 的 completed 重命名无关，勿改（改键名会破坏历史种子导入客户端）。
|
||||
"converged": converged,
|
||||
"max_relc": max_relc,
|
||||
})),
|
||||
|
||||
@@ -426,8 +426,8 @@ pub async fn get_workflow_progress(
|
||||
}
|
||||
};
|
||||
|
||||
let rate_per_hour = avg_per_hour(|p| p.converged);
|
||||
let done_rate_per_hour = avg_per_hour(|p| p.converged + p.failed);
|
||||
let rate_per_hour = avg_per_hour(|p| p.completed);
|
||||
let done_rate_per_hour = avg_per_hour(|p| p.completed + p.failed);
|
||||
|
||||
// 速率统计的实际时间跨度（近 2h 子窗口首末间隔，≤2h；回退整窗时为整窗跨度）。
|
||||
let rate_span_hours: Option<f64> = {
|
||||
@@ -448,8 +448,8 @@ pub async fn get_workflow_progress(
|
||||
let stalled_minutes: Option<f64> = if series.len() >= 2 {
|
||||
let mut last_progress_idx = None;
|
||||
for i in 1..series.len() {
|
||||
let prev = series[i - 1].converged + series[i - 1].failed;
|
||||
let cur = series[i].converged + series[i].failed;
|
||||
let prev = series[i - 1].completed + series[i - 1].failed;
|
||||
let cur = series[i].completed + series[i].failed;
|
||||
if cur > prev {
|
||||
last_progress_idx = Some(i);
|
||||
}
|
||||
@@ -529,9 +529,11 @@ pub async fn get_workflow_points(
|
||||
}
|
||||
|
||||
if let Some(s) = &pq.status {
|
||||
// "converged" 保留作旧客户端兼容值（7c 由 converged 改名 completed），
|
||||
// 通过后于下方归一化为 "completed" 再绑定 SQL。
|
||||
if !matches!(
|
||||
s.as_str(),
|
||||
"pending" | "queued" | "running" | "converged" | "failed"
|
||||
"pending" | "queued" | "running" | "completed" | "converged" | "failed"
|
||||
) {
|
||||
return Err(crate::api::AppError::BadRequest(format!(
|
||||
"非法的 status 参数: {}",
|
||||
@@ -540,7 +542,10 @@ pub async fn get_workflow_points(
|
||||
}
|
||||
}
|
||||
if let Some(m) = &pq.method {
|
||||
if !matches!(m.as_str(), "cold_run" | "seed_step") {
|
||||
// 值域：TLUSTY 阶段策略名（cold_run/seed_step，映射 tlusty_success_method）∪
|
||||
// sentinel "synspec_only"（光谱专用点，映射 tlusty IS NULL AND synspec IS NOT NULL）。
|
||||
// 前缀化避免与潜在的同名 TLUSTY 策略碰撞（P9/E 语义拆分）。
|
||||
if !matches!(m.as_str(), "cold_run" | "seed_step" | "synspec_only") {
|
||||
return Err(crate::api::AppError::BadRequest(format!(
|
||||
"非法的 method 参数: {}",
|
||||
m
|
||||
@@ -560,21 +565,16 @@ pub async fn get_workflow_points(
|
||||
} else {
|
||||
"ASC"
|
||||
};
|
||||
// P3 窗口重写后，ORDER BY 作用于外层子查询（列名裸露，无 gp./t. 前缀）；
|
||||
// 耗时排序引用内层 COALESCE 的输出别名 eff_elapsed。列名均为编译期白名单。
|
||||
let order_by = match sort {
|
||||
"wave" => format!("gp.wave {dir}, gp.cno_sum ASC, gp.teff ASC"),
|
||||
"teff" => format!("gp.teff {dir}, gp.wave ASC, gp.cno_sum ASC"),
|
||||
"max_relc" => format!("t.max_relc IS NULL ASC, t.max_relc {dir}, gp.wave ASC"),
|
||||
"attempts" => format!("gp.attempt_count {dir}, gp.wave ASC, gp.cno_sum ASC"),
|
||||
// 耗时取最近一次尝试的真实墙钟（与列表展示同口径 COALESCE），NULL（从未派发）靠后。
|
||||
"elapsed" => {
|
||||
format!(
|
||||
"COALESCE(t.elapsed_sec, gp.last_elapsed_sec) IS NULL ASC, \
|
||||
COALESCE(t.elapsed_sec, gp.last_elapsed_sec) {dir}, gp.wave ASC"
|
||||
)
|
||||
}
|
||||
"last_completed_at" => {
|
||||
format!("t.completed_at IS NULL ASC, t.completed_at {dir}, gp.wave ASC")
|
||||
}
|
||||
"wave" => format!("wave {dir}, cno_sum ASC, teff ASC"),
|
||||
"teff" => format!("teff {dir}, wave ASC, cno_sum ASC"),
|
||||
"max_relc" => format!("max_relc IS NULL ASC, max_relc {dir}, wave ASC"),
|
||||
"attempts" => format!("attempt_count {dir}, wave ASC, cno_sum ASC"),
|
||||
// 耗时取最近一次尝试的真实墙钟（与列表展示同口径 eff_elapsed），NULL（从未派发）靠后。
|
||||
"elapsed" => format!("eff_elapsed IS NULL ASC, eff_elapsed {dir}, wave ASC"),
|
||||
"last_completed_at" => format!("completed_at IS NULL ASC, completed_at {dir}, wave ASC"),
|
||||
_ => {
|
||||
return Err(crate::api::AppError::BadRequest(format!(
|
||||
"非法的 sort 参数: {}",
|
||||
@@ -584,7 +584,14 @@ pub async fn get_workflow_points(
|
||||
};
|
||||
|
||||
let filter = crate::db::PointFilter {
|
||||
status: pq.status.clone(),
|
||||
// 7c 改名：旧值 "converged" 归一化为 "completed"（DB 实际存储值）。
|
||||
status: pq.status.as_deref().map(|s| {
|
||||
if s == "converged" {
|
||||
"completed".to_string()
|
||||
} else {
|
||||
s.to_string()
|
||||
}
|
||||
}),
|
||||
method: pq.method.clone(),
|
||||
wave: pq.wave,
|
||||
q: pq.q.clone(),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,937 @@
|
||||
//! 网格点（从原 db.rs 拆分，2026-08-06）。
|
||||
//! `impl Database` 的 网格点 域方法。共享基础设施（Database struct、连接管理、类型、helper）见父模块 `super`（crate::db）。
|
||||
use super::*;
|
||||
|
||||
impl Database {
|
||||
// --- Grid Point & Task operations ---
|
||||
pub async fn upsert_grid_point(
|
||||
&self,
|
||||
params_in: &GridPointParams,
|
||||
wave: i32,
|
||||
workflow_name: &str,
|
||||
) -> Result<()> {
|
||||
let name = params_in.model_name();
|
||||
self.upsert_grid_point_named(&name, params_in, wave, workflow_name)
|
||||
.await
|
||||
}
|
||||
|
||||
/// 与 `upsert_grid_point` 相同，但使用调用方提供的权威 `name`（而非从 params 重推）。
|
||||
///
|
||||
/// 历史种子导入专用：旧版 conv.json 的 `name` 是源精度真名（如 `t20000_g5.0_...`），
|
||||
/// 而导入路径的 params 来自旧 JSON（数值，无源文本，`model_name()` 会失真），故必须
|
||||
/// 显式传入旧名以保证 DB `name` 列与旧数据逐字符一致。
|
||||
pub async fn upsert_grid_point_named(
|
||||
&self,
|
||||
name: &str,
|
||||
params_in: &GridPointParams,
|
||||
wave: i32,
|
||||
workflow_name: &str,
|
||||
) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let p = params_in.clone();
|
||||
let name = name.to_string();
|
||||
let cno_sum = p.cno_sum();
|
||||
let wf = workflow_name.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"INSERT INTO grid_points (name, workflow_name, teff, logg, loghe, logc, logn, logo, cno_sum, wave)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
|
||||
ON CONFLICT(workflow_name, name) DO NOTHING",
|
||||
params![name, wf, p.teff.value(), p.logg.value(), p.loghe.value(), p.logc.value(), p.logn.value(), p.logo.value(), cno_sum, wave],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 计算某网格点的难度波次（wave）：该工作流内所有 `cno_sum` 严格小于本点（1e-5 容差）
|
||||
/// 的去重值个数，即 `cno_sum` 升序中的桶序号——与 `initialize_grid` 的波次分组口径一致
|
||||
/// （`scheduler.rs` 沿排序序扫描，cno_sum 变化 >1e-5 则 wave+1）。
|
||||
///
|
||||
/// 历史种子导入（`import_seed`）用它替代此前硬编码的 `wave=0`：导入点按现有规则归入
|
||||
/// 正确波次，前端难度波次推进不再把全部导入点挤在第一波。对已完全填充的工作流（所有
|
||||
/// cno_sum 等级都在库内），任意导入顺序的 rank 都正确；对全新工作流，配合 import_results
|
||||
/// 工具按 cno_sum 升序导入即可保证最终波次一致。
|
||||
pub async fn compute_wave_for_cno_sum(&self, workflow_name: &str, cno_sum: f64) -> Result<i32> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = workflow_name.to_string();
|
||||
let cno = cno_sum;
|
||||
let wave = tokio::task::spawn_blocking(move || -> Result<i32> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let n: i64 = conn.query_row(
|
||||
"SELECT COUNT(DISTINCT cno_sum) FROM grid_points WHERE workflow_name = ?1 AND cno_sum < ?2 - 1e-5",
|
||||
params![wf, cno],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
Ok(n as i32)
|
||||
})
|
||||
.await??;
|
||||
Ok(wave)
|
||||
}
|
||||
|
||||
/// 仅用于测试/诊断：取指定工作流的全部 pending 点（无 LIMIT）。生产调度走 _limit 版本。
|
||||
pub async fn get_pending_grid_points(
|
||||
&self,
|
||||
workflow_name: &str,
|
||||
) -> Result<Vec<(String, GridPointParams, i32)>> {
|
||||
self.get_pending_grid_points_limit(usize::MAX, workflow_name)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn get_pending_grid_points_limit(
|
||||
&self,
|
||||
limit: usize,
|
||||
workflow_name: &str,
|
||||
) -> Result<Vec<(String, GridPointParams, i32)>> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = workflow_name.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<(String, GridPointParams, i32)>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT name, teff, logg, loghe, logc, logn, logo, wave FROM grid_points
|
||||
WHERE status = 'pending' AND workflow_name = ?1
|
||||
ORDER BY wave ASC, cno_sum ASC, teff ASC LIMIT ?2",
|
||||
)?;
|
||||
|
||||
let limit_param = if limit == usize::MAX {
|
||||
-1i64
|
||||
} else {
|
||||
limit as i64
|
||||
};
|
||||
let rows_iter = stmt.query_map(params![wf, limit_param], |r| {
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
GridPointParams {
|
||||
teff: GridAxisValue::from_value(r.get::<_, f64>(1)?),
|
||||
logg: GridAxisValue::from_value(r.get::<_, f64>(2)?),
|
||||
loghe: GridAxisValue::from_value(r.get::<_, f64>(3)?),
|
||||
logc: GridAxisValue::from_value(r.get::<_, f64>(4)?),
|
||||
logn: GridAxisValue::from_value(r.get::<_, f64>(5)?),
|
||||
logo: GridAxisValue::from_value(r.get::<_, f64>(6)?),
|
||||
},
|
||||
r.get::<_, i32>(7)?,
|
||||
))
|
||||
})?;
|
||||
|
||||
let mut list = Vec::new();
|
||||
for r in rows_iter {
|
||||
list.push(r?);
|
||||
}
|
||||
Ok(list)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 原子选点：在 IMMEDIATE 事务内将 pending 点标记为 queued 并返回。
|
||||
///
|
||||
/// 解决 `get_pending_grid_points_limit`（SELECT）与 `update_grid_status`（UPDATE）
|
||||
/// 分离导致的 TOCTOU 竞态：两个并发调度调用可能 SELECT 到同一批 pending 点，
|
||||
/// 各自创建任务，产生重复派发（#5 修复）。
|
||||
///
|
||||
/// 与 `pop_task`（sqlite_queue.rs）和 `take_pending_node_token` 同口径：
|
||||
/// IMMEDIATE 事务在 BEGIN 时即获取写锁，SELECT 与 UPDATE 之间不会被其它
|
||||
/// 调用方插入，从而只有一个调用方能 claiming 到某批点。
|
||||
pub async fn claim_pending_grid_points(
|
||||
&self,
|
||||
limit: usize,
|
||||
workflow_name: &str,
|
||||
) -> Result<Vec<(String, GridPointParams, i32)>> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = workflow_name.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<(String, GridPointParams, i32)>> {
|
||||
let mut conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
|
||||
|
||||
let limit_param = if limit == usize::MAX {
|
||||
-1i64
|
||||
} else {
|
||||
limit as i64
|
||||
};
|
||||
|
||||
// P3 附带优化（§5.3）：先按调度优先级 SELECT 有序取行（ORDER BY wave/cno_sum/teff），
|
||||
// 再在同一 IMMEDIATE 事务内按 rowid 原子标记 queued——UPDATE...RETURNING 不保序，
|
||||
// 旧实现依赖 Rust 侧 sort_by 重排；改为 SELECT 排序贯穿后删除 Rust 排序。
|
||||
let mut select_stmt = tx.prepare(
|
||||
"SELECT rowid, name, teff, logg, loghe, logc, logn, logo, wave
|
||||
FROM grid_points
|
||||
WHERE status = 'pending' AND workflow_name = ?1
|
||||
ORDER BY wave ASC, cno_sum ASC, teff ASC
|
||||
LIMIT ?2",
|
||||
)?;
|
||||
|
||||
let mut list = Vec::new();
|
||||
let mut ids = Vec::new();
|
||||
{
|
||||
let rows = select_stmt.query_map(params![wf, limit_param], |r| {
|
||||
Ok((
|
||||
r.get::<_, i64>(0)?,
|
||||
(
|
||||
r.get::<_, String>(1)?,
|
||||
GridPointParams {
|
||||
teff: GridAxisValue::from_value(r.get::<_, f64>(2)?),
|
||||
logg: GridAxisValue::from_value(r.get::<_, f64>(3)?),
|
||||
loghe: GridAxisValue::from_value(r.get::<_, f64>(4)?),
|
||||
logc: GridAxisValue::from_value(r.get::<_, f64>(5)?),
|
||||
logn: GridAxisValue::from_value(r.get::<_, f64>(6)?),
|
||||
logo: GridAxisValue::from_value(r.get::<_, f64>(7)?),
|
||||
},
|
||||
r.get::<_, i32>(8)?,
|
||||
),
|
||||
))
|
||||
})?;
|
||||
for r in rows {
|
||||
let (id, item) = r?;
|
||||
ids.push(id);
|
||||
list.push(item);
|
||||
}
|
||||
}
|
||||
drop(select_stmt);
|
||||
|
||||
// 同事务原子标记 queued（IMMEDIATE 已持有写锁，SELECT→UPDATE 间无竞态窗口）。
|
||||
// 阶段列（5b）同步：tlusty_status 守卫保留既有终态（半失败重试时 tlusty_status 已
|
||||
// converged 不被覆盖）；synspec_status 自由流转为 queued（重试进行中可见——设计
|
||||
// §7.3 打开项 #2 "仅 synspec 侧流转"，审查修正：原实现连 synspec 'failed' 也保留，
|
||||
// 与设计意图不符）。
|
||||
if !ids.is_empty() {
|
||||
// 审查修复 #M6：SQLite 单语句参数上限 999，limit=usize::MAX（映射为 LIMIT -1）
|
||||
// 且 pending 点极多时会撑爆 IN(?,?,...)。按 500 一批分次 UPDATE（与
|
||||
// delete_tasks_by_ids 同口径），每批独立语句、同一事务，原子性与原实现等价。
|
||||
for chunk in ids.chunks(500) {
|
||||
let placeholders = vec!["?"; chunk.len()].join(", ");
|
||||
let sql = format!(
|
||||
"UPDATE grid_points SET status = 'queued',
|
||||
tlusty_status = CASE WHEN tlusty_status NOT IN ('converged','failed') THEN 'queued' ELSE tlusty_status END,
|
||||
synspec_status = 'queued'
|
||||
WHERE rowid IN ({placeholders})"
|
||||
);
|
||||
tx.execute(&sql, rusqlite::params_from_iter(chunk.iter()))?;
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
|
||||
Ok(list)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 重置指定工作流的 queued 点为 pending（系统重启/工作流启动时使用）。
|
||||
/// 按 workflow 隔离，避免误伤其他工作流（多工作流分区修复点）。
|
||||
pub async fn reset_queued_grid_points_to_pending(&self, workflow_name: &str) -> Result<usize> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<usize> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 只重置 queued 状态的任务为 pending。对于 running (正由 Worker 处理的项目)，不可在系统重启或初始化时粗暴清零，让 Worker 正常完成汇报或触发心跳/超时自动逐回
|
||||
let rows = conn.execute(
|
||||
"UPDATE grid_points SET status = 'pending' WHERE status = 'queued' AND workflow_name = ?1",
|
||||
params![wf],
|
||||
)?;
|
||||
Ok(rows)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// ForceRecompute 策略的终态重置（见 docs/task_engine_decoupling_design.md §2.1）：
|
||||
/// 把工作流内**已收敛 / 已失败**的终态点全部打回 pending，使调度器无视历史状态与
|
||||
/// 产物强制重算。在 workflow start 时由 initialize_grid 据策略调用。
|
||||
///
|
||||
/// 不触碰 queued/running（在途任务正常结算）与 pending（本就待派发）。
|
||||
/// 按 workflow 隔离，避免跨工作流误改同名点（多工作流分区修复点）。
|
||||
pub async fn reset_terminal_points_for_recompute(&self, workflow_name: &str) -> Result<usize> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<usize> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let rows = conn.execute(
|
||||
"UPDATE grid_points SET status = 'pending'
|
||||
WHERE workflow_name = ?1 AND status IN ('completed', 'failed')",
|
||||
params![wf],
|
||||
)?;
|
||||
Ok(rows)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// SkipConverged 策略的启动时重置（2026-08-04 语义修正）：仅把**已失败**的点打回
|
||||
/// pending 重试，**已收敛**点保留（增量语义）。与 ForceRecompute（收敛+失败全量
|
||||
/// 重置）和 SkipFailed（收敛/失败都保留）区别开——"跳过收敛、重试失败"是默认策略
|
||||
/// 应有的行为，此前 SkipConverged 与 SkipFailed 在启动时行为相同，无法表达它。
|
||||
///
|
||||
/// 不触碰 queued/running（在途任务正常结算）与 pending（本就待派发）。
|
||||
/// 按 workflow 隔离，避免跨工作流误改同名点。
|
||||
pub async fn reset_failed_points_for_retry(&self, workflow_name: &str) -> Result<usize> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<usize> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let rows = conn.execute(
|
||||
"UPDATE grid_points SET status = 'pending'
|
||||
WHERE workflow_name = ?1 AND status = 'failed'",
|
||||
params![wf],
|
||||
)?;
|
||||
Ok(rows)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 重置指定工作流内一批点（按 name）为 pending。
|
||||
/// 按 workflow 隔离，避免跨工作流误改同名点（多工作流分区修复点）。
|
||||
pub async fn reset_specific_grid_points_to_pending(
|
||||
&self,
|
||||
names: &[String],
|
||||
workflow_name: &str,
|
||||
) -> Result<usize> {
|
||||
if names.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let pool = self.pool.clone();
|
||||
let names_owned = names.to_vec();
|
||||
let wf = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<usize> {
|
||||
let mut conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let tx = conn.transaction()?;
|
||||
let mut count = 0;
|
||||
for name in &names_owned {
|
||||
count += tx.execute(
|
||||
"UPDATE grid_points SET status = 'pending' WHERE name = ?1 AND workflow_name = ?2 AND status IN ('queued', 'running')",
|
||||
params![name, wf],
|
||||
)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(count)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 找出疑似孤儿的网格点及其 stale pending 任务行（#6 修复兜底，2026-08-02 重构）。
|
||||
///
|
||||
/// 候选条件：网格点处于 `running`/`queued` 态，且 `tasks` 表中存在该点创建时间
|
||||
/// 老于 stale_sec 的 `pending` 行。候选**不等于**孤儿——老 pending 行可能是正常
|
||||
/// 在途任务（requeue_stale_tasks 按同 task_id 重投不更新 tasks.created_at，长任务
|
||||
/// 会天然变"老"）。调用方（GridScheduler::reclaim_orphaned_points）必须对每个
|
||||
/// task_id 做 MQ 活性交叉校验（task_row_exists）：队列行仍 pending/claimed 即真
|
||||
/// 在途，放行；全部无队列行才是凭证丢失的真孤儿。
|
||||
///
|
||||
/// 历史教训（2026-08-02 涡旋事故）：旧实现 reset_orphaned_running_points 仅凭
|
||||
/// "存在老 pending 行"即重置点，而 stop/重启遗留的僵尸 pending 行使该判据对每个
|
||||
/// 在跑的点恒真 → 每 30s 巡检重置 + 同轮再派发 → 单点被重复计算上百次。
|
||||
///
|
||||
/// 返回 (name, workflow_name, task_id) 行列表，由调用方按点分组。
|
||||
pub async fn find_stale_pending_points(
|
||||
&self,
|
||||
stale_sec: u64,
|
||||
) -> Result<Vec<(String, String, String)>> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<(String, String, String)>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let stale_offset = format!("-{} seconds", stale_sec);
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT gp.name, gp.workflow_name, t.task_id
|
||||
FROM grid_points gp
|
||||
JOIN tasks t
|
||||
ON t.point_name = gp.name
|
||||
AND t.workflow_name = gp.workflow_name
|
||||
WHERE gp.status IN ('running', 'queued')
|
||||
AND t.status = 'pending'
|
||||
AND t.created_at < datetime('now', ?1)",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![stale_offset], |r| {
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
r.get::<_, String>(1)?,
|
||||
r.get::<_, String>(2)?,
|
||||
))
|
||||
})?;
|
||||
let mut out = Vec::new();
|
||||
for r in rows {
|
||||
out.push(r?);
|
||||
}
|
||||
Ok(out)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 把孤儿网格点救回 `pending`（仅当点仍处于 running/queued 时生效）。
|
||||
///
|
||||
/// 与 `find_stale_pending_points` + MQ 活性校验 + `delete_tasks_by_ids` 组合使用：
|
||||
/// 调用方确认该点所有 stale pending 任务行均无队列凭证（真孤儿）并清除这些行后，
|
||||
/// 调本方法让调度器重新派发。WHERE 的 status 条件是并发防护：若校验期间恰好有
|
||||
/// 迟到上报把点置为终态（converged/failed），本 UPDATE 命中 0 行，不覆盖终态。
|
||||
///
|
||||
/// 返回是否实际重置（false = 点已不在 running/queued，无需处理）。
|
||||
pub async fn rescue_orphaned_point(&self, name: &str, workflow_name: &str) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
let changed = tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let n = conn.execute(
|
||||
"UPDATE grid_points SET status = 'pending'
|
||||
WHERE name = ?1 AND workflow_name = ?2 AND status IN ('running', 'queued')",
|
||||
params![name_owned, wf],
|
||||
)?;
|
||||
Ok(n > 0)
|
||||
})
|
||||
.await??;
|
||||
Ok(changed)
|
||||
}
|
||||
|
||||
/// 更新指定工作流内某点的状态。按 workflow 隔离，防跨工作流误改同名点。
|
||||
pub async fn update_grid_status(
|
||||
&self,
|
||||
name: &str,
|
||||
status: GridPointStatus,
|
||||
workflow_name: &str,
|
||||
) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let status_str = status.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"UPDATE grid_points SET status = ?1 WHERE name = ?2 AND workflow_name = ?3",
|
||||
params![status_str, name_owned, wf],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// H1 活锁修复辅助：记录网格点当前「剩余策略链」（JSON 数组）。运行时回退
|
||||
/// （trigger_strategy_fallback）把点打回 pending 等种子时调用，标记该点已失败过部分
|
||||
/// 策略（如 cold_run），供调度路径重派时用剩余链而非完整 YAML 链，避免重跑已失败策略。
|
||||
pub async fn set_pending_strategies(
|
||||
&self,
|
||||
name: &str,
|
||||
workflow_name: &str,
|
||||
strategies_json: &str,
|
||||
) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
let s = strategies_json.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"UPDATE grid_points SET pending_strategies = ?1 WHERE name = ?2 AND workflow_name = ?3",
|
||||
params![s, name_owned, wf],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 读取网格点的剩余策略链标记（JSON 数组字符串）。无标记返回 None。
|
||||
pub async fn get_pending_strategies(
|
||||
&self,
|
||||
name: &str,
|
||||
workflow_name: &str,
|
||||
) -> Result<Option<String>> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
let v = tokio::task::spawn_blocking(move || -> Result<Option<String>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT pending_strategies FROM grid_points WHERE name = ?1 AND workflow_name = ?2",
|
||||
)?;
|
||||
let v = stmt
|
||||
.query_row(params![name_owned, wf], |r| r.get::<_, Option<String>>(0))
|
||||
.ok()
|
||||
.flatten();
|
||||
Ok(v)
|
||||
})
|
||||
.await??;
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// 清除网格点的剩余策略链标记（调度路径消费后调用）。
|
||||
pub async fn clear_pending_strategies(&self, name: &str, workflow_name: &str) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"UPDATE grid_points SET pending_strategies = NULL WHERE name = ?1 AND workflow_name = ?2",
|
||||
params![name_owned, wf],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 领用任务时把网格点标记为 running（2026-08-02 涡旋事故修复：自带终态守卫）。
|
||||
///
|
||||
/// 仅允许 `pending`/`queued` → `running`：迟到/重复领用（对应点已被上报置为
|
||||
/// converged/failed）不得复活终态点。源态集含 `pending` 是为兼容 requeue 路径
|
||||
/// （requeue_stale_tasks 把点重置为 pending 后，节点可能先于调度器领用重投行）。
|
||||
///
|
||||
/// 不委托 `update_grid_status`：其本体须保持无守卫——种子回退的 failed→Queued
|
||||
/// 复活（scheduler.rs）依赖它覆盖终态。
|
||||
///
|
||||
/// 返回是否实际变更（false = 点已在 running 或终态，本次领用属迟到/重复）。
|
||||
pub async fn mark_grid_point_running(&self, name: &str, workflow_name: &str) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
let changed = tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 阶段列（5b）同步置 running：tlusty_status 守卫保留终态（同 claim），synspec 自由流转。
|
||||
let n = conn.execute(
|
||||
"UPDATE grid_points SET status = 'running',
|
||||
tlusty_status = CASE WHEN tlusty_status NOT IN ('converged','failed') THEN 'running' ELSE tlusty_status END,
|
||||
synspec_status = 'running'
|
||||
WHERE name = ?1 AND workflow_name = ?2 AND status IN ('pending', 'queued')",
|
||||
params![name_owned, wf],
|
||||
)?;
|
||||
Ok(n > 0)
|
||||
})
|
||||
.await??;
|
||||
Ok(changed)
|
||||
}
|
||||
|
||||
/// 历史种子导入专用：把网格点标记为 converged 并记录大气收敛途径 `tlusty_success_method`。
|
||||
///
|
||||
/// 与正常 `record_task_report` 路径的区别：导入不走 task 队列，无 task_type 可取，
|
||||
/// 故由导入工具（import_results）依据旧 conv.json 的 stages 是否含 seed_nc 判定该点
|
||||
/// 当初是冷启动收敛（cold_run）还是种子步进收敛（seed_step），经 multipart 字段透传至此。
|
||||
/// 导入点因此融入冷启动/种子步进统计，而非独立为 imported 分类。
|
||||
///
|
||||
/// - `success_method`：须为 "cold_run" 或 "seed_step"，非法值兜底为 "cold_run"（防注入）。
|
||||
/// 语义为 TLUSTY 阶段策略，写入 `tlusty_success_method` 列。
|
||||
/// - `elapsed_sec`：旧版 conv.json 的单点墙钟耗时（`summary.elapsed_sec`），落入
|
||||
/// `last_elapsed_sec` 列使迁移点在详情页/点表保留真实耗时；无此数据传 None。
|
||||
pub async fn mark_grid_point_imported(
|
||||
&self,
|
||||
name: &str,
|
||||
workflow_name: &str,
|
||||
elapsed_sec: Option<f64>,
|
||||
success_method: &str,
|
||||
) -> Result<()> {
|
||||
// 白名单校验：仅接受两种合法途径，非法值兜底 cold_run（避免拼接 SQL 注入风险）。
|
||||
let method = match success_method {
|
||||
"seed_step" => "seed_step",
|
||||
_ => "cold_run",
|
||||
};
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 阶段列（5b）：导入点是历史 TLUSTY 大气收敛 → tlusty_status='converged'，
|
||||
// synspec_status 保持 NULL（历史结果仅大气，光谱未运行/未记录）。与正常结算路径的
|
||||
// 状态一致性（审查 #2 修正：原实现不设阶段列，导入点与正常点状态口径不一致）。
|
||||
conn.execute(
|
||||
"UPDATE grid_points SET status = 'completed', tlusty_success_method = ?1, last_elapsed_sec = ?2, tlusty_status = 'converged' \
|
||||
WHERE name = ?3 AND workflow_name = ?4",
|
||||
params![method, elapsed_sec, name_owned, wf],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_grid_point_status(
|
||||
&self,
|
||||
name: &str,
|
||||
workflow_name: &str,
|
||||
) -> Result<Option<(String, i32)>> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<Option<(String, i32)>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare("SELECT status, attempt_count FROM grid_points WHERE name = ?1 AND workflow_name = ?2")?;
|
||||
let res = stmt.query_row(params![name_owned, wf], |r| Ok((r.get(0)?, r.get(1)?)));
|
||||
match res {
|
||||
Ok(tuple) => Ok(Some(tuple)),
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 网格汇总统计。
|
||||
///
|
||||
/// `workflow_filter`：
|
||||
/// - `None`：聚合全部工作流的 grid_points（dashboard 全局概览用）。
|
||||
/// - `Some(wf)`：仅聚合指定工作流（按工作流隔离的进度统计）。
|
||||
///
|
||||
/// 口径说明：`pending` 与 `queued` **分开**计数（详情页需要区分"未入队"与"排队中"）；
|
||||
/// 旧版前端若需合并口径，自行相加（见 dashboard state.js）。导入的历史点按其实际
|
||||
/// 收敛途径（cold_run/seed_step）归类，与正常计算点一并统计——不再有独立 imported 分类。
|
||||
pub async fn get_grid_summary_stats(
|
||||
&self,
|
||||
workflow_filter: Option<&str>,
|
||||
) -> Result<serde_json::Value> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = workflow_filter.map(|s| s.to_string());
|
||||
tokio::task::spawn_blocking(move || -> Result<serde_json::Value> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 合并原先 7 条独立 COUNT 查询为单次扫描，用 SUM(CASE WHEN ...) 一次性聚合所有口径，
|
||||
// 显著降低 status API 的数据库往返与锁竞争开销。
|
||||
let row = match &wf {
|
||||
Some(name) => conn.query_row(
|
||||
"SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) AS pending,
|
||||
SUM(CASE WHEN status = 'queued' THEN 1 ELSE 0 END) AS queued,
|
||||
SUM(CASE WHEN status = 'running' THEN 1 ELSE 0 END) AS running,
|
||||
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) AS completed,
|
||||
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed,
|
||||
SUM(CASE WHEN status = 'completed' AND tlusty_success_method = 'cold_run' THEN 1 ELSE 0 END) AS cold_run_converged,
|
||||
SUM(CASE WHEN status = 'completed' AND tlusty_success_method = 'seed_step' THEN 1 ELSE 0 END) AS seed_step_converged,
|
||||
SUM(CASE WHEN status = 'completed' AND synspec_success_method IS NOT NULL THEN 1 ELSE 0 END) AS synspec_converged
|
||||
FROM grid_points WHERE workflow_name = ?1",
|
||||
params![name],
|
||||
|r| {
|
||||
let n = |i: usize| -> i64 { r.get::<_, Option<i64>>(i).unwrap_or(None).unwrap_or(0) };
|
||||
Ok((n(0), n(1), n(2), n(3), n(4), n(5), n(6), n(7), n(8)))
|
||||
},
|
||||
),
|
||||
None => conn.query_row(
|
||||
"SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) AS pending,
|
||||
SUM(CASE WHEN status = 'queued' THEN 1 ELSE 0 END) AS queued,
|
||||
SUM(CASE WHEN status = 'running' THEN 1 ELSE 0 END) AS running,
|
||||
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) AS completed,
|
||||
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed,
|
||||
SUM(CASE WHEN status = 'completed' AND tlusty_success_method = 'cold_run' THEN 1 ELSE 0 END) AS cold_run_converged,
|
||||
SUM(CASE WHEN status = 'completed' AND tlusty_success_method = 'seed_step' THEN 1 ELSE 0 END) AS seed_step_converged,
|
||||
SUM(CASE WHEN status = 'completed' AND synspec_success_method IS NOT NULL THEN 1 ELSE 0 END) AS synspec_converged
|
||||
FROM grid_points",
|
||||
[],
|
||||
|r| {
|
||||
let n = |i: usize| -> i64 { r.get::<_, Option<i64>>(i).unwrap_or(None).unwrap_or(0) };
|
||||
Ok((n(0), n(1), n(2), n(3), n(4), n(5), n(6), n(7), n(8)))
|
||||
},
|
||||
),
|
||||
}?;
|
||||
let (
|
||||
total,
|
||||
pending,
|
||||
queued,
|
||||
running,
|
||||
completed,
|
||||
failed,
|
||||
cold_run_converged,
|
||||
seed_step_converged,
|
||||
synspec_converged,
|
||||
) = row;
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"total": total,
|
||||
"pending": pending,
|
||||
"queued": queued,
|
||||
"running": running,
|
||||
"completed": completed,
|
||||
"failed": failed,
|
||||
"cold_run_converged": cold_run_converged,
|
||||
"seed_step_converged": seed_step_converged,
|
||||
"synspec_converged": synspec_converged,
|
||||
}))
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 单工作流执行统计（详情页数据源）。
|
||||
///
|
||||
/// 在 `get_grid_summary_stats(Some(name))` 之上追加：
|
||||
/// - `waves`：难度波次分布（调度时按 cno_sum 分组的批次进度）；
|
||||
/// - `avg_point_sec`：单点平均真实耗时 = `AVG(COALESCE(tasks.elapsed_sec, 时间戳差))`——
|
||||
/// 优先用 Worker 回报的精确墙钟（P3 落库），历史无 elapsed_sec 的行回退时间戳差近似；
|
||||
/// - `eta_sec`：`avg_point_sec × 剩余点数 ÷ total_slots`（并发感知；slots 为在线节点
|
||||
/// 总槽位，由 handler 传入，≤0 时按串行兜底），无历史数据为 None。
|
||||
///
|
||||
/// `status` 由调用方传入（工作流当前状态），避免重复查询。
|
||||
pub async fn get_workflow_detail_stats(
|
||||
&self,
|
||||
name: &str,
|
||||
status: &str,
|
||||
total_slots: i64,
|
||||
) -> Result<WorkflowStats> {
|
||||
let base = self.get_grid_summary_stats(Some(name)).await?;
|
||||
let g = |k: &str| base.get(k).and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
let total = g("total");
|
||||
let completed = g("completed");
|
||||
let failed = g("failed");
|
||||
|
||||
let pool = self.pool.clone();
|
||||
let wf = name.to_string();
|
||||
let (waves, avg_point_sec) =
|
||||
tokio::task::spawn_blocking(move || -> Result<(Vec<WaveStats>, Option<f64>)> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT wave,
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) AS completed,
|
||||
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed
|
||||
FROM grid_points WHERE workflow_name = ?1
|
||||
GROUP BY wave ORDER BY wave ASC",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![wf], |r| {
|
||||
Ok(WaveStats {
|
||||
wave: r.get(0)?,
|
||||
total: r.get(1)?,
|
||||
completed: r.get::<_, Option<i64>>(2)?.unwrap_or(0),
|
||||
failed: r.get::<_, Option<i64>>(3)?.unwrap_or(0),
|
||||
})
|
||||
})?;
|
||||
let mut waves = Vec::new();
|
||||
for r in rows {
|
||||
waves.push(r?);
|
||||
}
|
||||
|
||||
// AVG 在无匹配行时返回 NULL；过滤掉非正/非有限值，保持 ETA 估算合理。
|
||||
// COALESCE：精确耗时（elapsed_sec）优先，旧数据回退 created→completed 时间戳差。
|
||||
let avg: Option<f64> = conn
|
||||
.query_row(
|
||||
"SELECT AVG(COALESCE(elapsed_sec, (julianday(completed_at) - julianday(created_at)) * 86400.0))
|
||||
FROM tasks
|
||||
WHERE workflow_name = ?1
|
||||
AND completed_at IS NOT NULL
|
||||
AND status IN ('completed', 'failed', 'timeout')",
|
||||
params![wf],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|v: &f64| v.is_finite() && *v > 0.0);
|
||||
|
||||
Ok((waves, avg))
|
||||
})
|
||||
.await??;
|
||||
|
||||
let remaining = total - completed - failed;
|
||||
let slots = (total_slots.max(1)) as f64;
|
||||
let eta_sec = match avg_point_sec {
|
||||
Some(avg) if remaining > 0 => Some(avg * remaining as f64 / slots),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
Ok(WorkflowStats {
|
||||
name: name.to_string(),
|
||||
status: status.to_string(),
|
||||
total,
|
||||
pending: g("pending"),
|
||||
queued: g("queued"),
|
||||
running: g("running"),
|
||||
completed,
|
||||
failed,
|
||||
cold_run_converged: g("cold_run_converged"),
|
||||
seed_step_converged: g("seed_step_converged"),
|
||||
waves,
|
||||
avg_point_sec,
|
||||
eta_sec,
|
||||
})
|
||||
}
|
||||
|
||||
/// 工作流逐点列表（带最近一次尝试信息），分页返回 `(符合过滤的总数, 当前页行)`。
|
||||
///
|
||||
/// 最近尝试用关联子查询取 tasks 最新行（走 idx_tasks_point_wf_time）；
|
||||
/// 所有过滤值参数化绑定，`q` 的 LIKE 通配符先转义；ORDER BY 片段来自白名单。
|
||||
pub async fn list_workflow_points(
|
||||
&self,
|
||||
wf: &str,
|
||||
f: &PointFilter,
|
||||
) -> Result<(i64, Vec<PointRow>)> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = wf.to_string();
|
||||
let f = f.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<(i64, Vec<PointRow>)> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
|
||||
// 动态 WHERE：子句与占位符同步增长，值全部走绑定参数（SQLite 动态类型，
|
||||
// wave 以字符串绑定由列亲和性转回 INTEGER 比较）。
|
||||
// 列名**不带 gp. 前缀**：窗口重写后筛选作用于外层子查询（列名裸露）。
|
||||
let mut clauses: Vec<String> = vec!["workflow_name = ?1".to_string()];
|
||||
let mut binds: Vec<Box<dyn rusqlite::types::ToSql>> = vec![Box::new(wf.clone())];
|
||||
if let Some(s) = &f.status {
|
||||
binds.push(Box::new(s.clone()));
|
||||
clauses.push(format!("status = ?{}", binds.len()));
|
||||
}
|
||||
if let Some(m) = &f.method {
|
||||
if m == "synspec_only" {
|
||||
// 光谱专用收敛点（tlusty 禁用）：仅命中 synspec-only 点（tlusty 归因 NULL），
|
||||
// 与徽章/parSets 的 SYNSPEC 档同口径。双阶段点归因在大气侧，不含在这里。
|
||||
clauses.push(
|
||||
"tlusty_success_method IS NULL AND synspec_success_method IS NOT NULL"
|
||||
.to_string(),
|
||||
);
|
||||
} else {
|
||||
// cold_run/seed_step 等 TLUSTY 阶段策略 → 过滤 tlusty_success_method。
|
||||
binds.push(Box::new(m.clone()));
|
||||
clauses.push(format!("tlusty_success_method = ?{}", binds.len()));
|
||||
}
|
||||
}
|
||||
if let Some(w) = f.wave {
|
||||
binds.push(Box::new(w.to_string()));
|
||||
clauses.push(format!("wave = ?{}", binds.len()));
|
||||
}
|
||||
if let Some(q) = &f.q {
|
||||
// 转义 LIKE 通配符（\ % _），仅影响匹配语义，不构成注入面（值仍绑定）。
|
||||
let escaped = q
|
||||
.replace('\\', "\\\\")
|
||||
.replace('%', "\\%")
|
||||
.replace('_', "\\_");
|
||||
binds.push(Box::new(format!("%{}%", escaped)));
|
||||
clauses.push(format!("name LIKE ?{} ESCAPE '\\'", binds.len()));
|
||||
}
|
||||
let where_sql = clauses.join(" AND ");
|
||||
let bind_refs = || binds.iter().map(|b| b.as_ref());
|
||||
|
||||
// 总数（同过滤条件，作用于 grid_points，无需 JOIN）
|
||||
let total: i64 = conn.query_row(
|
||||
&format!("SELECT COUNT(*) FROM grid_points WHERE {}", where_sql),
|
||||
rusqlite::params_from_iter(bind_refs()),
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
|
||||
// 数据行：ROW_NUMBER() 窗口单遍取每点最新任务，替代逐行相关子查询
|
||||
// （P3，数千点 → 数千次子查询消除）。窗口 PARTITION BY (name, workflow_name)
|
||||
// 内按「未完成靠后 → completed_at DESC → created_at DESC」选最新行，与旧
|
||||
// 相关子查询 ORDER BY 语义逐行等价；外层 rn=1 过滤 + 用户排序/分页。
|
||||
let mut all_binds = binds;
|
||||
// limit=None 时不拼 LIMIT 子句（联合分析需全量，截断会让分析失真）。
|
||||
let limit_clause = match f.limit {
|
||||
Some(lim) => {
|
||||
let limit_idx = all_binds.len() + 1;
|
||||
all_binds.push(Box::new(lim));
|
||||
format!(" LIMIT ?{}", limit_idx)
|
||||
}
|
||||
None => String::new(),
|
||||
};
|
||||
// offset 仅在有 limit 或非零时才有意义；None-limit 全量场景强制忽略 offset。
|
||||
let offset_clause = if f.limit.is_some() {
|
||||
let offset_idx = all_binds.len() + 1;
|
||||
all_binds.push(Box::new(f.offset));
|
||||
format!(" OFFSET ?{}", offset_idx)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
// 排序：order_by 由 API 层编译期白名单拼出；空（如测试用 default filter）则省略子句。
|
||||
let order_clause = if f.order_by.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(" ORDER BY {}", f.order_by)
|
||||
};
|
||||
let sql = format!(
|
||||
"SELECT * FROM (
|
||||
SELECT gp.name, gp.teff, gp.logg, gp.loghe, gp.logc, gp.logn, gp.logo,
|
||||
gp.cno_sum, gp.wave, gp.status, gp.tlusty_success_method, gp.attempt_count,
|
||||
t.max_relc,
|
||||
t.seed_point_name, t.node_id,
|
||||
t.completed_at, t.error_message,
|
||||
COALESCE(t.elapsed_sec, gp.last_elapsed_sec) AS eff_elapsed,
|
||||
gp.workflow_name,
|
||||
gp.synspec_success_method,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY gp.name, gp.workflow_name
|
||||
ORDER BY t.completed_at IS NULL, t.completed_at DESC, t.created_at DESC
|
||||
) AS rn
|
||||
FROM grid_points gp
|
||||
LEFT JOIN tasks t
|
||||
ON t.point_name = gp.name AND t.workflow_name = gp.workflow_name
|
||||
) WHERE rn = 1 AND {}{}{}{}",
|
||||
where_sql, order_clause, limit_clause, offset_clause
|
||||
);
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
let rows = stmt.query_map(
|
||||
rusqlite::params_from_iter(all_binds.iter().map(|b| b.as_ref())),
|
||||
point_row_from_query,
|
||||
)?;
|
||||
let mut points = Vec::new();
|
||||
for r in rows {
|
||||
points.push(r?);
|
||||
}
|
||||
Ok((total, points))
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 单点行（带最近尝试信息），点不存在返回 None。与列表端点同一 SELECT 列序。
|
||||
pub async fn get_workflow_point_row(&self, wf: &str, point: &str) -> Result<Option<PointRow>> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = wf.to_string();
|
||||
let point = point.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<Option<PointRow>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
// 与 list_workflow_points 同构的 ROW_NUMBER() 窗口取最近任务（P3），外层 rn=1。
|
||||
"SELECT * FROM (
|
||||
SELECT gp.name, gp.teff, gp.logg, gp.loghe, gp.logc, gp.logn, gp.logo,
|
||||
gp.cno_sum, gp.wave, gp.status, gp.tlusty_success_method, gp.attempt_count,
|
||||
t.max_relc,
|
||||
t.seed_point_name, t.node_id,
|
||||
t.completed_at, t.error_message,
|
||||
COALESCE(t.elapsed_sec, gp.last_elapsed_sec) AS eff_elapsed,
|
||||
gp.workflow_name,
|
||||
gp.synspec_success_method,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY gp.name, gp.workflow_name
|
||||
ORDER BY t.completed_at IS NULL, t.completed_at DESC, t.created_at DESC
|
||||
) AS rn
|
||||
FROM grid_points gp
|
||||
LEFT JOIN tasks t
|
||||
ON t.point_name = gp.name AND t.workflow_name = gp.workflow_name
|
||||
) WHERE rn = 1 AND workflow_name = ?1 AND name = ?2",
|
||||
)?;
|
||||
let res = stmt.query_row(params![wf, point], point_row_from_query);
|
||||
match res {
|
||||
Ok(row) => Ok(Some(row)),
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
})
|
||||
.await?
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,577 @@
|
||||
//! Node/凭据（从原 db.rs 拆分，2026-08-06）。
|
||||
//! `impl Database` 的 Node/凭据 域方法。共享基础设施（Database struct、连接管理、类型、helper）见父模块 `super`（crate::db）。
|
||||
use super::*;
|
||||
|
||||
impl Database {
|
||||
// --- Node operations ---
|
||||
/// 注册/刷新节点。返回 (is_new, existing_status)：
|
||||
/// - 新申请：`(true, None)`
|
||||
/// - 已存在（含 online 等已审批态）：`(false, Some(<旧状态>))`，仅更新配置保持既有状态。
|
||||
///
|
||||
/// 返回旧状态供 API 层区分响应：已审批（online）的节点免凭据重新注册时，
|
||||
/// 不应回 "pending_approval"（误导运维以为还需审批），而应如实告知其已是已授权节点。
|
||||
pub async fn register_node(
|
||||
&self,
|
||||
req: &NodeRegisterRequest,
|
||||
) -> Result<(bool, Option<String>, Option<String>)> {
|
||||
let pool = self.pool.clone();
|
||||
let req_cloned = req.clone();
|
||||
|
||||
let (is_new, existing_status, registration_secret) =
|
||||
tokio::task::spawn_blocking(move || -> Result<(bool, Option<String>, Option<String>)> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare("SELECT status FROM nodes WHERE node_id = ?1")?;
|
||||
let existing_status: Option<String> =
|
||||
stmt.query_row(params![req_cloned.node_id], |r| r.get(0)).ok();
|
||||
|
||||
match &existing_status {
|
||||
Some(_st) => {
|
||||
// 已存在的节点：更新配置，保持既有状态
|
||||
conn.execute(
|
||||
"UPDATE nodes SET max_slots = ?1, last_heartbeat = datetime('now') WHERE node_id = ?2",
|
||||
params![req_cloned.max_slots, req_cloned.node_id],
|
||||
)?;
|
||||
Ok((false, existing_status, None))
|
||||
}
|
||||
None => {
|
||||
// 新申请节点：生成一次性 registration_secret（H8）并插入待审批状态。
|
||||
// registration_secret 用于 /node/check_status 取走专属 token 的二次凭据，
|
||||
// 防止知道 node_id（常源自主机名，可猜测）的攻击者抢先取走待发 token。
|
||||
let secret = format!(
|
||||
"{}{}",
|
||||
uuid::Uuid::new_v4().simple(),
|
||||
uuid::Uuid::new_v4().simple()
|
||||
);
|
||||
conn.execute(
|
||||
"INSERT INTO nodes (node_id, max_slots, status, last_heartbeat, registration_secret)
|
||||
VALUES (?1, ?2, 'pending_approval', datetime('now'), ?3)",
|
||||
params![req_cloned.node_id, req_cloned.max_slots, secret],
|
||||
)?;
|
||||
Ok((true, None, Some(secret)))
|
||||
}
|
||||
}
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok((is_new, existing_status, registration_secret))
|
||||
}
|
||||
|
||||
/// 管理员审批同意节点接入：将节点状态切为 online 并生成专属 node_token（返回明文 token）。
|
||||
pub async fn approve_node(&self, node_id: &str) -> Result<String> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"UPDATE nodes SET status = 'online', last_heartbeat = datetime('now') WHERE node_id = ?1",
|
||||
params![node_id_owned],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
// 颁发专属 node_token
|
||||
let new_token = self.issue_node_token(node_id).await?;
|
||||
Ok(new_token)
|
||||
}
|
||||
|
||||
/// 管理员拒绝节点接入：彻底清理该节点的注册申请记录。
|
||||
pub async fn reject_node(&self, node_id: &str) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"DELETE FROM nodes WHERE node_id = ?1",
|
||||
params![node_id_owned],
|
||||
)?;
|
||||
conn.execute(
|
||||
"DELETE FROM node_credentials WHERE node_id = ?1",
|
||||
params![node_id_owned],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 记录心跳并回读管理员配额。
|
||||
///
|
||||
/// 返回 `Option<i32>`：当前节点的 `admin_max_slots`（None = 无限制）。供 API 层
|
||||
/// 在心跳响应体透传给 Worker（见 docs/dynamic_cpu_slots_design.md §3.1）。在 UPDATE
|
||||
/// 之后立即 SELECT，保证管理员刚设置的配额在本次心跳即下发（心跳间隔内的延迟可接受）。
|
||||
pub async fn heartbeat_node(&self, req: &NodeHeartbeatRequest) -> Result<Option<i32>> {
|
||||
let pool = self.pool.clone();
|
||||
let req_cloned = req.clone();
|
||||
|
||||
let admin_max_slots = tokio::task::spawn_blocking(move || -> Result<Option<i32>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 心跳刷新指标与时间戳，status 由 CASE 决定：
|
||||
// - online/offline → online（offline 节点复活）
|
||||
// - disabled → disabled（管理员手动停用，永不被心跳复活；
|
||||
// 仅刷新 cpu/mem/心跳，让 Dashboard 仍能看到该节点存活待命）
|
||||
// - pending_approval 不在 IN 列表，心跳对其无效（维持审批语义）。
|
||||
conn.execute(
|
||||
"UPDATE nodes SET active_slots = ?1, cpu_usage = ?2, memory_usage = ?3,
|
||||
last_heartbeat = datetime('now'),
|
||||
status = CASE WHEN status = 'disabled' THEN 'disabled' ELSE 'online' END
|
||||
WHERE node_id = ?4 AND status IN ('online', 'offline', 'disabled')",
|
||||
params![
|
||||
req_cloned.active_slots,
|
||||
req_cloned.cpu_usage,
|
||||
req_cloned.memory_usage,
|
||||
req_cloned.node_id
|
||||
],
|
||||
)?;
|
||||
// 回读配额。注：即便 UPDATE 命中 0 行（如 pending_approval 节点心跳），
|
||||
// 节点行仍存在，配额回读仍应给出当前值；这里以 node_id 直查即可。
|
||||
// 仅把「无行」视为 None；真实 DB 错误（I/O、列缺失等）须向上传播，
|
||||
// 避免把故障静默降级为「无配额」而让 Worker 以物理上限满负荷运行。
|
||||
// （审查修复 N2：旧实现 .ok().flatten() 会把 SQL 错误一并吞成 None。）
|
||||
let quota: Option<i32> = {
|
||||
let res = conn.query_row(
|
||||
"SELECT admin_max_slots FROM nodes WHERE node_id = ?1",
|
||||
params![req_cloned.node_id],
|
||||
|r| r.get::<_, Option<i32>>(0),
|
||||
);
|
||||
match res {
|
||||
Ok(v) => v,
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => None,
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
};
|
||||
Ok(quota)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(admin_max_slots)
|
||||
}
|
||||
|
||||
/// 设置管理员强制并发槽位上限（动态调整 CPU 核数）。
|
||||
///
|
||||
/// - `Some(n)`（n>=0）：限制节点最多并发 n 个任务（0 = 暂停接新任务，正在跑的不会中断）。
|
||||
/// - `None`：清除限制，恢复节点物理 `max_slots` 上限。
|
||||
///
|
||||
/// 返回是否命中行（节点未注册时返回 false，调用方据此返回 404）。负数由 API 层
|
||||
/// 校验拒绝（不在此处理），避免把非法值写入 DB。
|
||||
pub async fn set_node_admin_max_slots(
|
||||
&self,
|
||||
node_id: &str,
|
||||
admin_max_slots: Option<i32>,
|
||||
) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
|
||||
let count = tokio::task::spawn_blocking(move || -> Result<u64> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let rows = conn.execute(
|
||||
"UPDATE nodes SET admin_max_slots = ?1 WHERE node_id = ?2",
|
||||
params![admin_max_slots, node_id_owned],
|
||||
)?;
|
||||
Ok(rows as u64)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(count > 0)
|
||||
}
|
||||
|
||||
/// 手动停用/启用节点（管理员运维操作）。
|
||||
///
|
||||
/// - `enabled=false`：把 `online`/`offline` 节点切为 `disabled`，claim 将不再向其分发任务。
|
||||
/// `pending_approval` 不在范围（审批流程独立），`disabled` 再点为幂等空操作。
|
||||
/// - `enabled=true`：把 `disabled` 节点切为 `offline`（而非 online），靠节点下一次心跳
|
||||
/// 自然翻成 online——既能自愈，又不会对真实离线的节点虚报在线。
|
||||
///
|
||||
/// 返回 `true` 表示状态确有迁移；`false` 表示当前状态不匹配（调用方据此返回 409）。
|
||||
pub async fn set_node_enabled(&self, node_id: &str, enabled: bool) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
|
||||
let count = tokio::task::spawn_blocking(move || -> Result<u64> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let rows = if enabled {
|
||||
conn.execute(
|
||||
"UPDATE nodes SET status = 'offline' WHERE node_id = ?1 AND status = 'disabled'",
|
||||
params![node_id_owned],
|
||||
)?
|
||||
} else {
|
||||
conn.execute(
|
||||
"UPDATE nodes SET status = 'disabled' WHERE node_id = ?1 AND status IN ('online', 'offline')",
|
||||
params![node_id_owned],
|
||||
)?
|
||||
};
|
||||
Ok(rows as u64)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(count > 0)
|
||||
}
|
||||
|
||||
/// 节点是否处于手动停用态（claim 热路径用，命中主键索引，开销可忽略）。
|
||||
pub async fn is_node_disabled(&self, node_id: &str) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
|
||||
let disabled = tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt =
|
||||
conn.prepare("SELECT 1 FROM nodes WHERE node_id = ?1 AND status = 'disabled'")?;
|
||||
let exists = stmt.exists(params![node_id_owned])?;
|
||||
Ok(exists)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(disabled)
|
||||
}
|
||||
|
||||
/// 为指定 node 颁发专属 token：生成随机明文 token，DB 存其 SHA-256 hash。
|
||||
/// 返回明文 token（仅此一次，由调用方转交 node 持久化）。
|
||||
/// 若该 node 已有凭据则覆盖（重新颁发）。
|
||||
pub async fn issue_node_token(&self, node_id: &str) -> Result<String> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
// 两个 v4 UUID（各 16 字节随机）拼接 → 各 32 hex 字符 = 64 字符 token
|
||||
let token =
|
||||
uuid::Uuid::new_v4().simple().to_string() + &uuid::Uuid::new_v4().simple().to_string();
|
||||
let token_hash = hash_token(&token);
|
||||
let token_for_ret = token.clone();
|
||||
let token_to_db = token_for_ret.clone();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"INSERT INTO node_credentials (node_id, token_hash, issued_at, raw_token_pending)
|
||||
VALUES (?1, ?2, datetime('now'), ?3)
|
||||
ON CONFLICT(node_id) DO UPDATE SET
|
||||
token_hash = excluded.token_hash,
|
||||
issued_at = datetime('now'),
|
||||
raw_token_pending = excluded.raw_token_pending",
|
||||
params![node_id_owned, token_hash, token_to_db],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
// token 轮换：旧 token_hash 已失效，新 token_hash 即将生效。整体清空缓存最稳妥
|
||||
// （issue 是低频运维动作，全清代价可忽略）。
|
||||
self.invalidate_token_cache().await;
|
||||
|
||||
Ok(token_for_ret)
|
||||
}
|
||||
|
||||
/// 一次性拉取并清除暂存的明文 node_token（取走即焚安全策略）。
|
||||
///
|
||||
/// 在单个 IMMEDIATE 事务内：先 SELECT 读出明文，再 UPDATE 置 NULL。IMMEDIATE 事务在
|
||||
/// BEGIN 时即获取写锁，保证 SELECT 与 UPDATE 之间不会被其它调用方插入，从而只有一个
|
||||
/// 调用方能取到 token（原子语义）。
|
||||
///
|
||||
/// 注：SQLite 的 `UPDATE ... RETURNING` 返回的是列的**新值**（SET 之后），故清空后
|
||||
/// RETURNING 该列只会得到 NULL，无法用于读旧值；因此这里用显式 SELECT + UPDATE。
|
||||
pub async fn take_pending_node_token(
|
||||
&self,
|
||||
node_id: &str,
|
||||
registration_secret: Option<&str>,
|
||||
) -> Result<Option<String>> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
let secret_owned = registration_secret.map(|s| s.to_string());
|
||||
|
||||
let token = tokio::task::spawn_blocking(move || -> Result<Option<String>> {
|
||||
let mut conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
|
||||
|
||||
// H8：取走待发 token 前校验 registration_secret（节点注册时下发的一次性凭据）。
|
||||
// 仅当 nodes 表记录的 registration_secret 与请求提供的一致（恒定时间比对），
|
||||
// 才允许取走 token，防止仅知道 node_id（可猜测）的攻击者抢先取走。
|
||||
let stored_secret: Option<String> = {
|
||||
let mut secret_stmt =
|
||||
tx.prepare("SELECT registration_secret FROM nodes WHERE node_id = ?1")?;
|
||||
secret_stmt
|
||||
.query_row(params![node_id_owned], |r| r.get::<_, Option<String>>(0))
|
||||
.ok()
|
||||
.flatten()
|
||||
};
|
||||
let secret_ok = match (&stored_secret, &secret_owned) {
|
||||
(Some(a), Some(b)) => ct_eq_option(a, b),
|
||||
// 旧库节点（无 registration_secret）不强制要求，保持向后兼容；
|
||||
// 新节点（有 secret）必须提供正确 secret。
|
||||
(None, _) => true,
|
||||
(Some(_), None) => false,
|
||||
};
|
||||
if !secret_ok {
|
||||
tx.commit()?;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let raw_token: Option<String> = {
|
||||
let mut select_stmt = tx.prepare(
|
||||
"SELECT raw_token_pending FROM node_credentials
|
||||
WHERE node_id = ?1 AND raw_token_pending IS NOT NULL",
|
||||
)?;
|
||||
select_stmt
|
||||
.query_row(params![node_id_owned], |r| r.get::<_, String>(0))
|
||||
.ok()
|
||||
};
|
||||
if raw_token.is_some() {
|
||||
tx.execute(
|
||||
"UPDATE node_credentials SET raw_token_pending = NULL
|
||||
WHERE node_id = ?1 AND raw_token_pending IS NOT NULL",
|
||||
params![&node_id_owned],
|
||||
)?;
|
||||
// M2 修复：取走后把 registration_secret 轮换为全新随机值（本地一次性凭据）。
|
||||
// 设计文档（node_credentials 注释）明确 registration_secret 是「审批前一次性凭据，
|
||||
// 取走专属 token 前消费」。若取走后旧 secret 仍有效，被攻陷节点/截获注册响应的
|
||||
// 攻击者可凭「可猜测的 node_id + 旧 secret」反复取走每次 reissue 产生的新 token。
|
||||
// 轮换成无人知晓的新值后旧 secret 立即失效，强制 reissue 的 token 走管理员线下
|
||||
// 下发路径（worker.rs 401 提示引导），杜绝凭 secret 复活取 token。
|
||||
// 注意刻意**不清空为 NULL**：下方鉴权把 `(None, _) => true` 当作旧库兼容放行，
|
||||
// 清空会让取走后的节点退化为「免 secret 可取」的旧库语义，反而开新洞。
|
||||
let new_secret = format!(
|
||||
"{}{}",
|
||||
uuid::Uuid::new_v4().simple(),
|
||||
uuid::Uuid::new_v4().simple()
|
||||
);
|
||||
tx.execute(
|
||||
"UPDATE nodes SET registration_secret = ?2 WHERE node_id = ?1",
|
||||
params![&node_id_owned, new_secret],
|
||||
)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(raw_token)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
/// 按 token（明文）反查所属 node_id；仅当 token 有效（未被重发覆盖）时返回 Some。
|
||||
/// 用于中间件：请求带来 node token，由此确定调用方身份。
|
||||
///
|
||||
/// 高频路径（每个 Node 请求一次）：先查内存 token_cache，命中且未过期直接返回；
|
||||
/// miss 才落 DB，并回填缓存。issue 会主动清空整个缓存。
|
||||
///
|
||||
/// 失效语义：重发（issue_node_token）会用 ON CONFLICT 覆盖该 node 的 token_hash，
|
||||
/// 旧 token 明文 hash 不再存在于表 → 查询返回 None → 401。无需独立的 revoked 标记。
|
||||
///
|
||||
/// 撤销竞态修复：历史上存在 TOCTOU 窗口——线程 A 用旧 token miss 落 DB 查到 node_id
|
||||
/// 后准备回填，期间线程 B（管理员 reissue）覆盖 DB 的 token_hash 并 clear() 缓存，
|
||||
/// 随后线程 A 拿到写锁把旧 token_hash 回填进缓存，导致已撤销的旧 token 在 TTL(60s)
|
||||
/// 内仍能鉴权。修复：回填时在同一把写锁内重新校验该 token_hash 是否仍是 DB 当前值
|
||||
/// （未被 reissue 覆盖），是才回填，杜绝旧 token 复活窗口。
|
||||
pub async fn find_node_by_token(&self, token: &str) -> Option<String> {
|
||||
let token_hash = hash_token(token);
|
||||
|
||||
// 1) 先查内存缓存
|
||||
{
|
||||
let cache = self.token_cache.read().await;
|
||||
if let Some((node_id, inserted)) = cache.entries.get(&token_hash) {
|
||||
if inserted.elapsed() < TOKEN_CACHE_TTL {
|
||||
return Some(node_id.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2) 记录 DB 查询前的缓存 generation，miss 落 DB。
|
||||
// generation 用于回填时的 TOCTOU 终极防护：若 DB 查询与回填之间发生过
|
||||
// invalidate（reissue），generation 会变化，本次回填将被丢弃。
|
||||
let gen_before = { self.token_cache.read().await.generation };
|
||||
let pool = self.pool.clone();
|
||||
let hash_for_db = token_hash.clone();
|
||||
let db_hit: Option<String> =
|
||||
tokio::task::spawn_blocking(move || -> Result<Option<String>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT node_id FROM node_credentials WHERE token_hash = ?1 LIMIT 1",
|
||||
)?;
|
||||
let res = stmt.query_row(params![hash_for_db], |r| r.get::<_, String>(0));
|
||||
match res {
|
||||
Ok(id) => Ok(Some(id)),
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|r| match r {
|
||||
Ok(id) => id,
|
||||
// 审查修复 #M4：原 `.and_then(|r| r.ok())` 把 DB 错误静默吞成 None（鉴权 401），
|
||||
// 故障期所有节点请求被判 401 且无任何日志。现至少记录 warn 便于排障。
|
||||
Err(e) => {
|
||||
tracing::warn!("find_node_by_token DB 查询失败，降级为鉴权未命中: {}", e);
|
||||
None
|
||||
}
|
||||
});
|
||||
|
||||
// 3) 命中则回填缓存；回填前校验 generation 未变化（期间无 invalidate），
|
||||
// 彻底消除"旧 token_hash 复活"窗口。generation 变化则视为已撤销，不缓存、不返回。
|
||||
if let Some(id) = db_hit {
|
||||
let mut cache = self.token_cache.write().await;
|
||||
if cache.generation == gen_before {
|
||||
cache
|
||||
.entries
|
||||
.insert(token_hash, (id.clone(), std::time::Instant::now()));
|
||||
Some(id)
|
||||
} else {
|
||||
// 期间发生过 reissue 导致的 invalidate：旧 token_hash 已不应复活。
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// 清空全部 token 反查缓存并自增 generation。在 issue（token 轮换使旧 token 失效）时调用。
|
||||
/// 自增 generation 使所有在途的 find_node_by_token 回填（gen_before 已过期）被丢弃，
|
||||
/// 彻底消除"DB 读取旧 hash → reissue clear → 回填旧 hash"的 TOCTOU 复活窗口。
|
||||
async fn invalidate_token_cache(&self) {
|
||||
let mut cache = self.token_cache.write().await;
|
||||
cache.entries.clear();
|
||||
cache.generation = cache.generation.wrapping_add(1);
|
||||
}
|
||||
|
||||
/// 判断指定 node_id 是否已存在于 nodes 表（重发 token 前置校验，防幽灵 node_id）。
|
||||
pub async fn get_node_exists(&self, node_id: &str) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let node_id_owned = node_id.to_string();
|
||||
let exists = tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare("SELECT 1 FROM nodes WHERE node_id = ?1 LIMIT 1")?;
|
||||
Ok(stmt.exists(params![node_id_owned])?)
|
||||
})
|
||||
.await??;
|
||||
Ok(exists)
|
||||
}
|
||||
|
||||
/// 统计已颁发 node 凭据数量（用于启动期半配置告警判断）。
|
||||
pub async fn node_credentials_count(&self) -> Result<i64> {
|
||||
let pool = self.pool.clone();
|
||||
let count = tokio::task::spawn_blocking(move || -> Result<i64> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let n: i64 =
|
||||
conn.query_row("SELECT COUNT(*) FROM node_credentials", [], |r| r.get(0))?;
|
||||
Ok(n)
|
||||
})
|
||||
.await??;
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
/// 列出全部节点及其凭据状态（LEFT JOIN node_credentials）。
|
||||
/// 用于管理 API：admin 可查看每个节点的在线状态、是否已颁发 token、颁发时间。
|
||||
/// 尚未注册凭据的节点（如旧数据迁移）token_status 为 "none"、token_issued_at 为 None。
|
||||
pub async fn list_nodes_with_credentials(&self) -> Result<Vec<NodeCredentialView>> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<NodeCredentialView>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT n.node_id, n.max_slots, n.active_slots, n.status,
|
||||
n.cpu_usage, n.memory_usage,
|
||||
strftime('%Y-%m-%dT%H:%M:%SZ', n.last_heartbeat),
|
||||
c.token_hash, strftime('%Y-%m-%dT%H:%M:%SZ', c.issued_at),
|
||||
n.admin_max_slots
|
||||
FROM nodes n
|
||||
LEFT JOIN node_credentials c ON c.node_id = n.node_id
|
||||
ORDER BY n.status ASC, n.node_id ASC",
|
||||
)?;
|
||||
let rows = stmt.query_map([], |r| {
|
||||
let hb_str: String = r.get::<_, String>(6)?;
|
||||
Ok(NodeCredentialView {
|
||||
node_id: r.get(0)?,
|
||||
max_slots: r.get(1)?,
|
||||
active_slots: r.get(2)?,
|
||||
status: r.get(3)?,
|
||||
cpu_usage: r.get(4)?,
|
||||
memory_usage: r.get(5)?,
|
||||
last_heartbeat: chrono::DateTime::parse_from_rfc3339(&hb_str)
|
||||
.map(|d| d.with_timezone(&chrono::Utc))
|
||||
.unwrap_or_else(|_| chrono::DateTime::UNIX_EPOCH),
|
||||
// c.token_hash 为 NULL 表示该节点无凭据记录；非空即为有效 token
|
||||
// （token 失效靠重发覆盖 hash 实现，不存在「已吊销」中间态）
|
||||
token_status: match r.get::<_, Option<String>>(7)? {
|
||||
None => "none".to_string(),
|
||||
Some(_) => "active".to_string(),
|
||||
},
|
||||
token_issued_at: r.get::<_, Option<String>>(8)?,
|
||||
admin_max_slots: r.get::<_, Option<i32>>(9)?,
|
||||
})
|
||||
})?;
|
||||
let mut list = Vec::new();
|
||||
for row in rows {
|
||||
list.push(row?);
|
||||
}
|
||||
Ok(list)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
pub async fn get_active_nodes(&self) -> Result<Vec<NodeInfo>> {
|
||||
let pool = self.pool.clone();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<NodeInfo>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT node_id, max_slots, active_slots, status, cpu_usage, memory_usage, strftime('%Y-%m-%dT%H:%M:%SZ', last_heartbeat), admin_max_slots FROM nodes WHERE status = 'online'"
|
||||
)?;
|
||||
|
||||
let node_iter = stmt.query_map([], |r| {
|
||||
let hb_str: String = r.get(6)?;
|
||||
Ok(NodeInfo {
|
||||
node_id: r.get(0)?,
|
||||
max_slots: r.get(1)?,
|
||||
active_slots: r.get(2)?,
|
||||
status: r.get(3)?,
|
||||
cpu_usage: r.get(4)?,
|
||||
memory_usage: r.get(5)?,
|
||||
last_heartbeat: chrono::DateTime::parse_from_rfc3339(&hb_str)
|
||||
.map(|d| d.with_timezone(&chrono::Utc))
|
||||
.unwrap_or_else(|_| chrono::Utc::now()),
|
||||
admin_max_slots: r.get::<_, Option<i32>>(7)?,
|
||||
})
|
||||
})?;
|
||||
|
||||
let mut nodes = Vec::new();
|
||||
for n in node_iter {
|
||||
nodes.push(n?);
|
||||
}
|
||||
Ok(nodes)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
pub async fn mark_stale_nodes_offline(&self, stale_sec: u64) -> Result<u64> {
|
||||
let pool = self.pool.clone();
|
||||
let count = tokio::task::spawn_blocking(move || -> Result<u64> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let rows = conn.execute(
|
||||
"UPDATE nodes SET status = 'offline' WHERE status = 'online' AND strftime('%s', 'now') - strftime('%s', last_heartbeat) > ?1",
|
||||
params![stale_sec as i64],
|
||||
)?;
|
||||
Ok(rows as u64)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
//! 种子（从原 db.rs 拆分，2026-08-06）。
|
||||
//! `impl Database` 的 种子 域方法。共享基础设施（Database struct、连接管理、类型、helper）见父模块 `super`（crate::db）。
|
||||
use super::*;
|
||||
|
||||
impl Database {
|
||||
pub async fn reload_seed_cache(&self) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let items = tokio::task::spawn_blocking(move || -> Result<Vec<SeedCacheItem>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT point_name, teff, logg, loghe, logc, logn, logo, file_path FROM seeds WHERE is_clean = 1"
|
||||
)?;
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok(SeedCacheItem {
|
||||
point_name: row.get(0)?,
|
||||
params: GridPointParams {
|
||||
teff: GridAxisValue::from_value(row.get::<_, f64>(1)?),
|
||||
logg: GridAxisValue::from_value(row.get::<_, f64>(2)?),
|
||||
loghe: GridAxisValue::from_value(row.get::<_, f64>(3)?),
|
||||
logc: GridAxisValue::from_value(row.get::<_, f64>(4)?),
|
||||
logn: GridAxisValue::from_value(row.get::<_, f64>(5)?),
|
||||
logo: GridAxisValue::from_value(row.get::<_, f64>(6)?),
|
||||
},
|
||||
file_path: row.get(7)?,
|
||||
})
|
||||
})?;
|
||||
let mut list = Vec::new();
|
||||
for r in rows {
|
||||
list.push(r?);
|
||||
}
|
||||
Ok(list)
|
||||
})
|
||||
.await??;
|
||||
|
||||
// 同步重建 exact_family 索引（每个种子写入其 floor/floor+1 两个桶）。
|
||||
let mut index: std::collections::HashMap<SeedBucketKey, Vec<SeedCacheItem>> =
|
||||
std::collections::HashMap::new();
|
||||
for item in &items {
|
||||
for key in SeedBucketKey::from_params(&item.params) {
|
||||
index.entry(key).or_default().push(item.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let mut lock = self.seed_cache.write().await;
|
||||
*lock = items;
|
||||
drop(lock);
|
||||
let mut idx_lock = self.seed_index.write().await;
|
||||
*idx_lock = index;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn insert_seed(&self, params_in: &GridPointParams, file_path: &str) -> Result<()> {
|
||||
let name = params_in.model_name();
|
||||
self.insert_seed_named(&name, params_in, file_path).await
|
||||
}
|
||||
|
||||
/// 与 `insert_seed` 相同，但用调用方提供的权威 `name`（而非从 params 重推）。
|
||||
///
|
||||
/// 历史种子导入专用：保证 `seeds.point_name` 与旧版 conv.json 的源精度真名一致。
|
||||
pub async fn insert_seed_named(
|
||||
&self,
|
||||
name: &str,
|
||||
params_in: &GridPointParams,
|
||||
file_path: &str,
|
||||
) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let p = params_in.clone();
|
||||
let name = name.to_string();
|
||||
let path_owned = file_path.to_string();
|
||||
|
||||
let name_db = name.clone();
|
||||
let path_db = path_owned.clone();
|
||||
let p_db = p.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"INSERT INTO seeds (point_name, teff, logg, loghe, logc, logn, logo, file_path)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
|
||||
ON CONFLICT(point_name) DO UPDATE SET file_path = excluded.file_path",
|
||||
params![
|
||||
name_db,
|
||||
p_db.teff.value(),
|
||||
p_db.logg.value(),
|
||||
p_db.loghe.value(),
|
||||
p_db.logc.value(),
|
||||
p_db.logn.value(),
|
||||
p_db.logo.value(),
|
||||
path_db
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
let item = SeedCacheItem {
|
||||
point_name: name.clone(),
|
||||
params: p.clone(),
|
||||
file_path: path_owned,
|
||||
};
|
||||
let mut lock = self.seed_cache.write().await;
|
||||
let is_new;
|
||||
if let Some(pos) = lock.iter().position(|x| x.point_name == item.point_name) {
|
||||
// 已存在：seeds 表 ON CONFLICT 只更新 file_path，point_name/物理参数不变，
|
||||
// 故 exact_family 桶键不变，索引无需重写，仅同步 Vec 里的 file_path。
|
||||
lock[pos].file_path = item.file_path.clone();
|
||||
is_new = false;
|
||||
} else {
|
||||
lock.push(item.clone());
|
||||
is_new = true;
|
||||
}
|
||||
drop(lock);
|
||||
|
||||
// 新种子才需写入索引（已存在的种子 params 不变，桶键未变）。
|
||||
if is_new {
|
||||
let mut idx_lock = self.seed_index.write().await;
|
||||
for key in SeedBucketKey::from_params(&p) {
|
||||
idx_lock.entry(key).or_default().push(item.clone());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn find_best_seed_from_db(
|
||||
&self,
|
||||
target: &GridPointParams,
|
||||
) -> Result<Option<common::seed_finder::SeedMatch>> {
|
||||
// 优先走 exact_family 索引（O(1)~O(小)）：取出 target 的两个候选桶的全部种子快照后
|
||||
// 立即释放读锁，避免阻塞 insert_seed 写。exact_family 是绝大多数命中的路径。
|
||||
let exact_candidates: Vec<SeedCacheItem> = {
|
||||
let idx_lock = self.seed_index.read().await;
|
||||
let keys = SeedBucketKey::from_params(target);
|
||||
let mut out = Vec::new();
|
||||
for key in keys {
|
||||
if let Some(bucket) = idx_lock.get(&key) {
|
||||
out.extend(bucket.iter().cloned());
|
||||
}
|
||||
}
|
||||
out
|
||||
};
|
||||
|
||||
let mut exact_family: Option<(String, std::path::PathBuf, f64)> = None;
|
||||
for item in &exact_candidates {
|
||||
let (is_exact, d) = common::seed_finder::calculate_seed_distance(&item.params, target);
|
||||
if is_exact {
|
||||
let path = std::path::PathBuf::from(&item.file_path);
|
||||
if exact_family.is_none() || d < exact_family.as_ref().unwrap().2 {
|
||||
exact_family = Some((item.point_name.clone(), path, d));
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some((name, path, d)) = exact_family {
|
||||
return Ok(Some(common::seed_finder::SeedMatch {
|
||||
name,
|
||||
path,
|
||||
distance: d,
|
||||
}));
|
||||
}
|
||||
|
||||
// exact_family 未命中：退化到全量 global 扫描。克隆参数缩小读锁持有范围。
|
||||
let snapshot: Vec<_> = {
|
||||
let lock = self.seed_cache.read().await;
|
||||
lock.iter()
|
||||
.map(|item| {
|
||||
(
|
||||
item.point_name.clone(),
|
||||
item.params.clone(),
|
||||
item.file_path.clone(),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
|
||||
let mut global_closest: Option<(String, std::path::PathBuf, f64)> = None;
|
||||
for (point_name, params, file_path) in snapshot {
|
||||
let (is_exact, d) = common::seed_finder::calculate_seed_distance(¶ms, target);
|
||||
// exact_family 路径已在上面处理过（索引已覆盖），这里只关心 global 候选。
|
||||
if !is_exact
|
||||
&& d <= common::seed_finder::MAX_GLOBAL_SEED_DISTANCE
|
||||
&& (global_closest.is_none() || d < global_closest.as_ref().unwrap().2)
|
||||
{
|
||||
let path = std::path::PathBuf::from(&file_path);
|
||||
global_closest = Some((point_name, path, d));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some((name, path, d)) = global_closest {
|
||||
Ok(Some(common::seed_finder::SeedMatch {
|
||||
name,
|
||||
path,
|
||||
distance: d,
|
||||
}))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
//! 进度快照（从原 db.rs 拆分，2026-08-06）。
|
||||
//! `impl Database` 的 进度快照 域方法。共享基础设施（Database struct、连接管理、类型、helper）见父模块 `super`（crate::db）。
|
||||
use super::*;
|
||||
|
||||
impl Database {
|
||||
/// 记录一条进度快照：与上一条快照计数完全一致时不写（去重防膨胀）。
|
||||
/// 返回是否实际写入。由后台循环对每个运行中工作流调用。
|
||||
pub async fn record_progress_snapshot(&self, wf: &str) -> Result<bool> {
|
||||
let base = self.get_grid_summary_stats(Some(wf)).await?;
|
||||
let g = |k: &str| base.get(k).and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
let cur = (
|
||||
g("total"),
|
||||
g("pending"),
|
||||
g("queued"),
|
||||
g("running"),
|
||||
g("completed"),
|
||||
g("failed"),
|
||||
);
|
||||
let pool = self.pool.clone();
|
||||
let wf = wf.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let mut conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 审查修复 #M1：原 SELECT last + INSERT 非原子，两个并发后台循环会各自读到
|
||||
// last != cur 各自 INSERT 相同快照，去重失效导致快照表膨胀。现用单条
|
||||
// INSERT ... WHERE NOT EXISTS 原子去重（单语句天然原子，无需 IMMEDIATE——
|
||||
// 写语句在提交时持有写锁，WHERE NOT EXISTS 的读与写同锁内一致）。
|
||||
let tx = conn.transaction()?;
|
||||
let inserted = tx.execute(
|
||||
"INSERT INTO workflow_progress_snapshots
|
||||
(workflow_name, total, pending, queued, running, completed, failed)
|
||||
SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM workflow_progress_snapshots
|
||||
WHERE workflow_name = ?1
|
||||
AND total = ?2 AND pending = ?3 AND queued = ?4
|
||||
AND running = ?5 AND completed = ?6 AND failed = ?7
|
||||
)",
|
||||
params![wf, cur.0, cur.1, cur.2, cur.3, cur.4, cur.5],
|
||||
)?;
|
||||
tx.commit()?;
|
||||
Ok(inserted > 0)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 清理超过保留期（天）的进度快照。返回无意义计数以外的错误。
|
||||
pub async fn purge_progress_snapshots(&self, keep_days: u64) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"DELETE FROM workflow_progress_snapshots WHERE ts < datetime('now', ?1)",
|
||||
params![format!("-{} days", keep_days)],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 最近 `hours` 小时的进度时间序列（ts 升序）。
|
||||
pub async fn get_progress_series(&self, wf: &str, hours: i64) -> Result<Vec<ProgressPoint>> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = wf.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<ProgressPoint>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT ts, total, pending, queued, running, completed, failed
|
||||
FROM workflow_progress_snapshots
|
||||
WHERE workflow_name = ?1 AND ts >= datetime('now', ?2)
|
||||
ORDER BY ts ASC, id ASC",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![wf, format!("-{} hours", hours)], |r| {
|
||||
Ok(ProgressPoint {
|
||||
ts: r.get(0)?,
|
||||
total: r.get(1)?,
|
||||
pending: r.get(2)?,
|
||||
queued: r.get(3)?,
|
||||
running: r.get(4)?,
|
||||
completed: r.get(5)?,
|
||||
failed: r.get(6)?,
|
||||
})
|
||||
})?;
|
||||
let mut series = Vec::new();
|
||||
for r in rows {
|
||||
series.push(r?);
|
||||
}
|
||||
Ok(series)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,603 @@
|
||||
//! 任务/结算（从原 db.rs 拆分，2026-08-06）。
|
||||
//! `impl Database` 的 任务/结算 域方法。共享基础设施（Database struct、连接管理、类型、helper）见父模块 `super`（crate::db）。
|
||||
use super::*;
|
||||
|
||||
impl Database {
|
||||
/// 判断某个网格点在特定工作流内是否已经派发过种子步进 (seed_step) 任务。
|
||||
///
|
||||
/// 注意：**生产回退逻辑已改用策略链弹栈**（`pop_stage_strategy_for_fallback` +
|
||||
/// `trigger_strategy_fallback`，"种子回退仅一次"守卫由「链耗尽 → 保持 failed」取代）。
|
||||
/// 本方法仅保留作**测试断言辅助**（api_tests / db.rs 单测核验"全程未产生 seed_step 行"）。
|
||||
///
|
||||
/// 判定口径（Phase 6 起派生）：`json_extract(tlusty_strategies, '$[0]') = 'seed_step'`——
|
||||
/// task_type 列已删除，策略链首项即"当前执行策略"的权威快照。
|
||||
///
|
||||
/// 为何计数 pending（2026-08-02 涡旋事故定稿，回退 2026-08-01 的 pending 排除）：
|
||||
/// pending 的 seed_step 行要么是正在排队/在途的真任务（计数它正是对在途回退的
|
||||
/// 去重——否则救援途中收到的迟到失败报告会触发第二份 seed_step 重复派发）；
|
||||
/// 要么是"无队列行"的僵尸行（insert_task 后 push 前崩溃等）。2026-08-01 事故中
|
||||
/// 僵尸行曾永久堵死 21 个点的正当回退，当时的 pending 排除是应急解法；本修复后
|
||||
/// 僵尸行被结构性清除——stop/重启卫生（clear_queue_by_workflow 返回 ids 同步删
|
||||
/// tasks 行）、派发去重（schedule 前 MQ 活性校验删死行）、回退内僵尸卫生
|
||||
/// （trigger_strategy_fallback 先删死行再弹链）——pending 行不再可能是僵尸，
|
||||
/// 计数它既安全又必要。
|
||||
pub async fn has_seed_step_attempt(&self, name: &str, workflow_name: &str) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf_owned = workflow_name.to_string();
|
||||
let exists = tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT 1 FROM tasks WHERE point_name = ?1 AND json_extract(tlusty_strategies, '$[0]') = 'seed_step' AND workflow_name = ?2 LIMIT 1",
|
||||
)?;
|
||||
let res = stmt.exists(params![name_owned, wf_owned])?;
|
||||
Ok(res)
|
||||
})
|
||||
.await??;
|
||||
Ok(exists)
|
||||
}
|
||||
|
||||
/// 幂等上报吸收（见 api/task.rs report_task）：查 tasks 表中某任务是否**已由本节点结算**
|
||||
/// （node_id 归属匹配 + status 为终态）。首轮上报成功后 `remove_task` 清掉了 MQ 领用行，
|
||||
/// 若响应在链路上丢失，节点重试上报会经 `verify_task_claim` 落空——此时凭本方法判定为
|
||||
/// 已结算的幂等重放，返回 200 并补写种子，而非 403 误诊「token 失效」。
|
||||
///
|
||||
/// 返回 `(point_name, workflow_name)` 供调用方复用定向校验；未命中返回 None。
|
||||
pub async fn find_settled_task_claim(
|
||||
&self,
|
||||
task_id: &str,
|
||||
node_id: &str,
|
||||
) -> Result<Option<(String, Option<String>)>> {
|
||||
let pool = self.pool.clone();
|
||||
let tid = task_id.to_string();
|
||||
let nid = node_id.to_string();
|
||||
let row = tokio::task::spawn_blocking(move || -> Result<Option<(String, Option<String>)>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT point_name, workflow_name FROM tasks
|
||||
WHERE task_id = ?1 AND node_id = ?2 AND status IN ('completed', 'failed', 'timeout')
|
||||
LIMIT 1",
|
||||
)?;
|
||||
Ok(stmt
|
||||
.query_row(params![tid, nid], |r| {
|
||||
Ok((r.get::<_, String>(0)?, r.get::<_, Option<String>>(1)?))
|
||||
})
|
||||
.ok())
|
||||
})
|
||||
.await??;
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
/// 策略链自动回退（见 docs/task_engine_decoupling_design.md §4.2）：
|
||||
/// 读取指定网格点最近一条**已上报**（非 pending）tasks 行的 `tlusty_strategies`，
|
||||
/// 弹出首项，返回 `(剩余链, 被弹出的策略)`。
|
||||
///
|
||||
/// 兼容别名：`trigger_strategy_fallback` 的 TLUSTY 分支使用（failed_stage 为
|
||||
/// "tlusty" 或缺省时）。SYNSPEC 链的弹栈走 `pop_stage_strategy_for_fallback(_, _, "synspec")`。
|
||||
pub async fn pop_tlusty_strategy_for_fallback(
|
||||
&self,
|
||||
name: &str,
|
||||
workflow_name: &str,
|
||||
) -> Result<Option<FallbackSnapshot>> {
|
||||
self.pop_stage_strategy_for_fallback(name, workflow_name, "tlusty")
|
||||
.await
|
||||
}
|
||||
|
||||
/// 策略链自动回退（阶段参数化版，见 docs/task_engine_decoupling_design.md §4.2 注）：
|
||||
/// 与 `pop_tlusty_strategy_for_fallback` 语义一致，但按 `stage` 选择弹哪条链——
|
||||
/// `"synspec"` → `synspec_strategies` 列，其余 → `tlusty_strategies` 列。
|
||||
///
|
||||
/// 设计 §4.2 注明确要求 SYNSPEC 策略链的自动弹栈与 TLUSTY 完全一致（若配置了多策略链），
|
||||
/// 由失败阶段归因（TaskReport.failed_stage）决定弹哪条链。
|
||||
///
|
||||
/// **只读不改写**（见审查修复）：旧实现把剩余链写回旧行，随后调度器 insert_task
|
||||
/// 新建一行携带相同链 → 两行共享同一链、旧行的 strategies 变陈旧。现改为纯读取：
|
||||
/// 调度器拿到剩余链后构造新 TaskSpec（insert_task 写入新行携带剩余链），旧行保持
|
||||
/// 原状（记录该任务实际执行的策略，审计正确）。
|
||||
///
|
||||
/// **并发安全**（审查修复 #C2）：本方法本身非原子（内存 remove(0) 不落库），并发调用
|
||||
/// 会各自读到完整链、各自派发回退任务。现由调用方 `trigger_strategy_fallback` 持有
|
||||
/// `schedule_lock` 串行化，且 fallback 的「status == failed」状态守卫保证第二次调用
|
||||
/// 进入时点已被首次 fallback 改为 queued 而被拦截。故「只读 + 锁 + 状态守卫」组合
|
||||
/// 在保证审计正确性的同时消除了并发重复派发，无需改写旧行。
|
||||
///
|
||||
/// **返回派发时快照（[`FallbackSnapshot`]）**：策略链 + 被弹策略 + 该行落库的 policy。
|
||||
/// policy 与策略链同源（同一行、同一时刻派发），供回退决策（SkipFailed 门控）与重试
|
||||
/// 任务构造使用——回退行为由派发时配置决定，不随运行期 YAML 编辑漂移（对齐 §4.2
|
||||
/// 「不修改原有 policy，保持用户初始配置」）。
|
||||
///
|
||||
/// 过滤 `status IN ('failed','timeout')`：只有**失败**任务才触发回退、其策略链才应被弹。
|
||||
/// L4 修复：原 `status != 'pending'` 也会命中已完成的 `completed` 行——在孤儿回收重派
|
||||
/// 场景下，「最新已上报行」可能是一条 stale 的 completed 行，其链与刚失败触发回退的任务
|
||||
/// 不符，弹出会移除错误链的顺位。收紧到失败/超时行后，弹栈始终对准触发回退的失败任务。
|
||||
/// pending 行（未上报僵尸/在途）与 completed 行（已成功、链已消费完）均不参与弹栈。
|
||||
///
|
||||
/// 仅匹配精确 workflow_name（移除旧 `IS NULL / = ''` 兜底子句——多工作流分区迁移
|
||||
/// 已把历史行回填为 `__legacy__`，NULL 兜底反而让同名点跨工作流命中 legacy 行）。
|
||||
pub async fn pop_stage_strategy_for_fallback(
|
||||
&self,
|
||||
name: &str,
|
||||
workflow_name: &str,
|
||||
stage: &str,
|
||||
) -> Result<Option<FallbackSnapshot>> {
|
||||
// 阶段 → 策略链列 / policy 列的白名单映射（防注入：非法 stage 兜底为 tlusty 列）。
|
||||
let (col, policy_col) = match stage {
|
||||
"synspec" => ("synspec_strategies", "synspec_policy"),
|
||||
_ => ("tlusty_strategies", "tlusty_policy"),
|
||||
};
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
let sql = format!(
|
||||
"SELECT {col}, {policy_col} FROM tasks
|
||||
WHERE point_name = ?1 AND workflow_name = ?2 AND status IN ('failed', 'timeout')
|
||||
ORDER BY created_at DESC, rowid DESC LIMIT 1"
|
||||
);
|
||||
tokio::task::spawn_blocking(move || -> Result<Option<FallbackSnapshot>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 只读最新已上报行（非 pending），单条查询无需事务。
|
||||
let row: Option<(String, String)> = {
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
stmt.query_row(params![name_owned, wf], |r| {
|
||||
Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))
|
||||
})
|
||||
.ok()
|
||||
};
|
||||
let Some((json_str, policy_str)) = row else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut strategies: Vec<String> = serde_json::from_str(&json_str).unwrap_or_default();
|
||||
if strategies.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let popped = strategies.remove(0);
|
||||
Ok(Some(FallbackSnapshot {
|
||||
rest_strategies: strategies,
|
||||
popped,
|
||||
policy: ResumePolicy::from_str_lossy(&policy_str),
|
||||
}))
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 读取指定网格点最近一条 tasks 行的 TLUSTY 策略链（不修改）。供回退守卫判断
|
||||
/// 「策略链是否已含 seed_step 且为当前执行策略」等。无行返回空 Vec。
|
||||
pub async fn get_latest_tlusty_strategies(
|
||||
&self,
|
||||
name: &str,
|
||||
workflow_name: &str,
|
||||
) -> Result<Vec<String>> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
let strategies = tokio::task::spawn_blocking(move || -> Result<Vec<String>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let json_str: Option<String> = {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT tlusty_strategies FROM tasks
|
||||
WHERE point_name = ?1 AND workflow_name = ?2
|
||||
ORDER BY created_at DESC, rowid DESC LIMIT 1",
|
||||
)?;
|
||||
stmt.query_row(params![name_owned, wf], |r| r.get::<_, String>(0))
|
||||
.ok()
|
||||
};
|
||||
let json_str = match json_str {
|
||||
Some(s) => s,
|
||||
None => return Ok(Vec::new()),
|
||||
};
|
||||
Ok(serde_json::from_str(&json_str).unwrap_or_default())
|
||||
})
|
||||
.await??;
|
||||
Ok(strategies)
|
||||
}
|
||||
|
||||
pub async fn insert_task(&self, spec: &common::models::TaskSpec) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let spec = spec.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// 阶段配置打平入库（见 docs/task_engine_decoupling_design.md §4.1）。
|
||||
// strategies 序列化为 JSON 数组文本列；policy 取 as_str。
|
||||
// Phase 6 起无 task_type 列，当前执行策略由 tlusty_strategies[0] 派生。
|
||||
let tlusty_enabled = spec.tlusty_config.enabled;
|
||||
let tlusty_policy = spec.tlusty_config.policy.as_str();
|
||||
let tlusty_strategies = serde_json::to_string(&spec.tlusty_config.strategies)?;
|
||||
let synspec_enabled = spec.synspec_config.enabled;
|
||||
let synspec_policy = spec.synspec_config.policy.as_str();
|
||||
let synspec_strategies = serde_json::to_string(&spec.synspec_config.strategies)?;
|
||||
conn.execute(
|
||||
"INSERT INTO tasks (task_id, point_name, seed_point_name, status, created_at, workflow_name,
|
||||
tlusty_enabled, tlusty_policy, tlusty_strategies,
|
||||
synspec_enabled, synspec_policy, synspec_strategies, atmosphere_ref)
|
||||
VALUES (?1, ?2, ?3, 'pending', datetime('now'), ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)
|
||||
ON CONFLICT(task_id) DO NOTHING",
|
||||
params![
|
||||
spec.task_id.to_string(),
|
||||
spec.point_name,
|
||||
spec.seed_point_name,
|
||||
spec.workflow_name,
|
||||
tlusty_enabled,
|
||||
tlusty_policy,
|
||||
tlusty_strategies,
|
||||
synspec_enabled,
|
||||
synspec_policy,
|
||||
synspec_strategies,
|
||||
spec.atmosphere_ref,
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 删除 tasks 历史表中指定 task_id 的行。
|
||||
///
|
||||
/// 用于调度回滚：当 push_task 失败时，grid_points 已回滚、queue 已清理，
|
||||
/// 但先于 push 插入的 tasks 历史行（status='pending'）会遗留，污染每点尝试计数统计。
|
||||
/// 此方法在回滚路径中调用以保持三者一致。
|
||||
pub async fn delete_task(&self, task_id: &uuid::Uuid) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let id = task_id.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute("DELETE FROM tasks WHERE task_id = ?1", params![id])?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 列出指定网格点在当前工作流内仍处于 `pending` 的 tasks 行 task_id。
|
||||
///
|
||||
/// tasks 行的 pending 意为"已创建但从未上报"，可能是真在途（队列行仍 pending/
|
||||
/// claimed）或僵尸（队列行已不存在）。调用方须配合 SqliteTaskQueue::task_row_exists
|
||||
/// 做 MQ 活性交叉校验区分二者（2026-08-02 涡旋事故修复引入）：
|
||||
/// - 派发去重：任一活 → 跳过派发；全死 → 清僵尸后正常派发。
|
||||
/// - 孤儿回收 / 回退种子卫生：同上分流。
|
||||
///
|
||||
/// `strategy` 传 Some("seed_step")/Some("cold_run") 时按当前策略过滤（Phase 6 起派生：
|
||||
/// 比较 `json_extract(tlusty_strategies, '$[0]')`，不再依赖已删除的 task_type 列），
|
||||
/// None 不过滤。生产调用仅传 None，Some 分支供测试断言用。
|
||||
pub async fn has_pending_tasks_for_point(
|
||||
&self,
|
||||
point_name: &str,
|
||||
workflow_name: &str,
|
||||
strategy: Option<&str>,
|
||||
) -> Result<Vec<String>> {
|
||||
let pool = self.pool.clone();
|
||||
let point = point_name.to_string();
|
||||
let wf = workflow_name.to_string();
|
||||
let tt = strategy.map(|s| s.to_string());
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<String>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut ids = Vec::new();
|
||||
match tt {
|
||||
Some(tt) => {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT task_id FROM tasks WHERE point_name = ?1 AND workflow_name = ?2 AND status = 'pending' AND json_extract(tlusty_strategies, '$[0]') = ?3",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![point, wf, tt], |r| r.get::<_, String>(0))?;
|
||||
for r in rows {
|
||||
ids.push(r?);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT task_id FROM tasks WHERE point_name = ?1 AND workflow_name = ?2 AND status = 'pending'",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![point, wf], |r| r.get::<_, String>(0))?;
|
||||
for r in rows {
|
||||
ids.push(r?);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(ids)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 批量删除 tasks 表中指定 task_id 且仍为 `pending` 的行，返回删除行数。
|
||||
///
|
||||
/// 用于僵尸行清理（派发去重 / 孤儿回收 / stop·重启卫生）。`AND status = 'pending'`
|
||||
/// 是 TOCTOU 防护：校验活性与删除之间若有上报事务恰好提交（行变 completed/failed），
|
||||
/// 本删除不会误删已完成的审计历史（attempt_count/阶段归因列均不受影响）。
|
||||
/// 空切片短路返回 0；按 500 个一批分块（SQLite 默认宿主参数上限 999）。
|
||||
pub async fn delete_tasks_by_ids(&self, task_ids: &[String]) -> Result<usize> {
|
||||
if task_ids.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let pool = self.pool.clone();
|
||||
let ids = task_ids.to_vec();
|
||||
tokio::task::spawn_blocking(move || -> Result<usize> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut total = 0usize;
|
||||
for chunk in ids.chunks(500) {
|
||||
let placeholders = vec!["?"; chunk.len()].join(", ");
|
||||
let sql = format!(
|
||||
"DELETE FROM tasks WHERE task_id IN ({}) AND status = 'pending'",
|
||||
placeholders
|
||||
);
|
||||
let params: Vec<&dyn rusqlite::ToSql> =
|
||||
chunk.iter().map(|s| s as &dyn rusqlite::ToSql).collect();
|
||||
total += conn.execute(&sql, params.as_slice())?;
|
||||
}
|
||||
Ok(total)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 记录节点上报的任务结果。返回网格点状态是否发生**实际迁移**（bool）：
|
||||
/// 上层 report_task 据此决定失败报告是否触发种子回退——被终态守卫吸收的
|
||||
/// 重复报告返回 false，不再触发回退（2026-08-02 涡旋事故修复）。
|
||||
pub async fn record_task_report(
|
||||
&self,
|
||||
report: &TaskReport,
|
||||
workflow_name: &str,
|
||||
) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let report_cloned = report.clone();
|
||||
let point_name = report.point_name.clone();
|
||||
let wf = workflow_name.to_string();
|
||||
let converged = report.result_valid;
|
||||
let atmo_has_nan = report.atmosphere_has_nan;
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let mut conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let tx = conn.transaction()?;
|
||||
|
||||
let status_str = match report_cloned.status {
|
||||
TaskStatus::Completed => "completed",
|
||||
TaskStatus::Failed => "failed",
|
||||
TaskStatus::Timeout => "timeout",
|
||||
_ => "pending",
|
||||
};
|
||||
|
||||
// P1（Phase 1）：结算补落阶段信息。failed_stage 仅对失败/超时任务落库（缺省兜底
|
||||
// 'tlusty'，旧节点不携带该字段 → 按 TLUSTY 链回退，与调度端 infer_failed_stage
|
||||
// 默认一致）；成功任务显式置 NULL，避免污染按 failed_stage 过滤的审计/统计
|
||||
// （审查修复：旧实现 COALESCE 无条件填充，成功任务也被写入 'tlusty'）。
|
||||
// summary_json 透传 ModelSummary 序列化（错误路径为 {"error": ...}，服务端只透传不解析）。
|
||||
let is_failure = matches!(
|
||||
report_cloned.status,
|
||||
TaskStatus::Failed | TaskStatus::Timeout
|
||||
);
|
||||
let failed_stage_value: Option<&str> = if is_failure {
|
||||
report_cloned.failed_stage.as_deref().or(Some("tlusty"))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// H2 修复：任务结算去重守卫。并发重复上报同一 task 时，仅首个报告能把 tasks 行
|
||||
// 从非终态置为终态（WHERE status NOT IN 终态）。若 UPDATE 命中 0 行且任务行仍存在，
|
||||
// 说明该 task 已被先前报告结算为终态 → 本报告为重复结算 → 直接返回 changed=false、
|
||||
// 跳过网格点翻转。这消除了「report2 把 report1 回退后已置为 queued 的网格点再翻回
|
||||
// failed、重复触发 seed_step 回退」的 TOCTOU（2026-08-02 涡旋事故的并发残余）：
|
||||
// 两个报告即便都通过 API 层的 verify_task_claim（纯 SELECT 不消费），也只有一个
|
||||
// 会在 DB 层把任务置为终态并结算网格点。
|
||||
let task_updated = tx.execute(
|
||||
"UPDATE tasks SET status = ?1, node_id = ?2, max_relc = ?3, atmosphere_has_nan = ?4, completed_at = datetime('now'), error_message = ?5, elapsed_sec = ?6, failed_stage = ?8, summary_json = ?9 WHERE task_id = ?7 AND status NOT IN ('completed', 'failed', 'timeout')",
|
||||
params![
|
||||
status_str,
|
||||
report_cloned.node_id,
|
||||
report_cloned.max_relc,
|
||||
report_cloned.atmosphere_has_nan,
|
||||
report_cloned.error_message,
|
||||
report_cloned.elapsed_sec,
|
||||
report_cloned.task_id.to_string(),
|
||||
failed_stage_value,
|
||||
report_cloned.summary_json,
|
||||
],
|
||||
)?;
|
||||
|
||||
if task_updated == 0 {
|
||||
// 0 行更新：要么任务行已终态（重复结算），要么任务行不存在（被清理）。
|
||||
// 区分二者：任务行仍存在则为重复结算，跳过网格点翻转（首次结算的审计信息
|
||||
// 已保留在任务行）；任务行不存在则继续走原逻辑（策略读取回落到安全默认）。
|
||||
let task_exists = match tx.query_row(
|
||||
"SELECT 1 FROM tasks WHERE task_id = ?1",
|
||||
params![report_cloned.task_id.to_string()],
|
||||
|_| Ok(true),
|
||||
) {
|
||||
Ok(_) => true,
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => false,
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
if task_exists {
|
||||
tx.rollback()?;
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
|
||||
// 失败次数计数自增（attempt_count 仅作观测/统计用途，保留原子 UPDATE 避免并发竞态）。
|
||||
// 注意：状态迁移不再依赖该计数值（失败统一置 failed，是否回退复活由
|
||||
// trigger_strategy_fallback 按策略链弹栈 + policy 快照决定，见 db.rs FallbackSnapshot）。
|
||||
// 三条 UPDATE grid_points 均带 workflow_name 过滤，避免跨工作流误改同名点。
|
||||
//
|
||||
// 审查修复 #S2：原 `unwrap_or(0)` 把任何 rusqlite::Error（锁超时、磁盘 I/O）吞成 0。
|
||||
// 现区分 QueryReturnedNoRows（grid_points 无该点行，返回 0 合理）与其余错误（上抛）。
|
||||
match tx.query_row(
|
||||
"UPDATE grid_points SET attempt_count = attempt_count + 1 WHERE name = ?1 AND workflow_name = ?2 RETURNING attempt_count",
|
||||
params![point_name, wf],
|
||||
|r| r.get::<_, i32>(0),
|
||||
) {
|
||||
Ok(_) => {}
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
|
||||
// 成功归因（修复审查 #4 + Phase 6 派生口径 + Phase 5a + P9 命名拆分）：
|
||||
// - tlusty_success_method：TLUSTY 阶段以何策略收敛。tlusty_enabled=1 时取
|
||||
// `tlusty_strategies[0]`（cold_run/seed_step）；TLUSTY 禁用（synspec-only）为 NULL。
|
||||
// - synspec_success_method（5a）：光谱阶段以何策略收敛。synspec_enabled=1 时取
|
||||
// `synspec_strategies[0]`（如 "standard"）；TLUSTY-only 成功保持 NULL。
|
||||
// 成功分支（status=Completed）已由 derive_report_status 保证「synspec 启用则无错」，
|
||||
// 故"synspec 成功" ⟺ synspec_enabled=1，无需再查 synspec_rc。
|
||||
// 整体归因（原 success_method 值域混用）改为消费方派生：tlusty ?? synspec。
|
||||
// 注：依赖 SQLite 内建 JSON1 的 json_extract（rusqlite bundled SQLite 默认启用）。
|
||||
//
|
||||
// 审查修复 #S2（严重）：原 `unwrap_or((None, None, true, true))` 把任何 DB 错误
|
||||
// （锁超时、I/O）当成「行不存在」，用 tlusty_enabled=true/synspec_enabled=true 默认值
|
||||
// 继续写阶段状态，故障期会静默写入错误归因数据。现区分 QueryReturnedNoRows（行不存在，
|
||||
// 用安全默认）与其余错误（上抛）。
|
||||
let (tlusty_success_method, synspec_success_method, tlusty_enabled, synspec_enabled): (
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
bool,
|
||||
bool,
|
||||
) = match tx.query_row(
|
||||
"SELECT
|
||||
CASE WHEN tlusty_enabled = 1
|
||||
THEN json_extract(tlusty_strategies, '$[0]')
|
||||
END,
|
||||
CASE WHEN synspec_enabled = 1
|
||||
THEN json_extract(synspec_strategies, '$[0]')
|
||||
END,
|
||||
tlusty_enabled,
|
||||
synspec_enabled
|
||||
FROM tasks WHERE task_id = ?1",
|
||||
params![report_cloned.task_id.to_string()],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
||||
) {
|
||||
Ok(v) => v,
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => (None, None, true, true),
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
|
||||
// Phase 5b：阶段状态（§7.2 语义表）。NULL = 阶段不适用（enabled=0）。
|
||||
// 整体 grid_points.status 仍是权威状态，阶段列是补充可查信息。
|
||||
// **阶段列值固定用 'converged'**（阶段收敛语义，与 StageSummary.converged 一致）——
|
||||
// claim/running 守卫按 'converged' 保留终态，二者必须匹配（审查 CRITICAL#1：7c 曾误
|
||||
// 把这里改成 'completed' 导致半失败守卫失效，已回退）。
|
||||
let pipe_ok = report_cloned.status == TaskStatus::Completed && converged && !atmo_has_nan;
|
||||
let (tlusty_status, synspec_status): (Option<&str>, Option<&str>) = if pipe_ok {
|
||||
// 整管线成功：启用阶段均 converged。
|
||||
(
|
||||
tlusty_enabled.then_some("converged"),
|
||||
synspec_enabled.then_some("converged"),
|
||||
)
|
||||
} else if converged {
|
||||
// 半失败：大气收敛 + 光谱失败。tlusty 阶段确已收敛（tlusty_enabled 防御性守卫——
|
||||
// 该分支在 synspec-only 下不可达，但避免未来代码路径把禁用阶段误标为 converged）。
|
||||
(tlusty_enabled.then_some("converged"), Some("failed"))
|
||||
} else if tlusty_enabled {
|
||||
// 大气未收敛（TLUSTY 启用）→ TLUSTY 失败、SYNSPEC 未运行。
|
||||
(Some("failed"), Some("pending"))
|
||||
} else {
|
||||
// synspec-only 失败（converged 由 synspec_rc 决定）。
|
||||
(None, Some("failed"))
|
||||
};
|
||||
|
||||
// 终态守卫（2026-08-02 涡旋事故修复）：
|
||||
// - 成功分支：converged 为吸收态，迟到的重复成功报告不覆盖阶段归因列；
|
||||
// 允许 failed/queued/pending/running → converged（迟到的真收敛结果获胜，
|
||||
// 含种子救援路径 failed → converged）。
|
||||
// - 失败分支：NOT IN ('completed','failed')——converged 不可被迟到失败报告翻黑
|
||||
// （事故直接症状）；failed 吸收重复失败报告（changed=0 → 不触发重复回退）。
|
||||
// queued/pending → failed 是正确语义：能通过 verify_task_claim 的报告必为
|
||||
// 真实领用并计算过的任务（如回收器重置后在途任务才上报、领用后 mark 失败等）。
|
||||
// - 阶段列（5b）：tlusty_status 用 CASE 守卫保留既有终态——半失败重试的 synspec-only
|
||||
// 任务（tlusty 关闭）不覆盖已收敛的 tlusty_status（设计 §7.3 打开项 #2：仅 synspec
|
||||
// 侧流转）；synspec_status 自由流转。
|
||||
let grid_changed = if pipe_ok {
|
||||
tx.execute(
|
||||
"UPDATE grid_points SET status = 'completed', last_elapsed_sec = ?1, tlusty_success_method = ?2, synspec_success_method = ?3,
|
||||
tlusty_status = CASE WHEN ?6 IS NOT NULL THEN ?6 ELSE tlusty_status END,
|
||||
synspec_status = ?7
|
||||
WHERE name = ?4 AND workflow_name = ?5 AND status != 'completed'",
|
||||
params![
|
||||
report_cloned.elapsed_sec,
|
||||
tlusty_success_method,
|
||||
synspec_success_method,
|
||||
point_name,
|
||||
wf,
|
||||
tlusty_status,
|
||||
synspec_status
|
||||
],
|
||||
)?
|
||||
} else {
|
||||
tx.execute(
|
||||
"UPDATE grid_points SET status = 'failed', last_elapsed_sec = ?1,
|
||||
tlusty_status = CASE WHEN ?4 IS NOT NULL THEN ?4 ELSE tlusty_status END,
|
||||
synspec_status = ?5
|
||||
WHERE name = ?2 AND workflow_name = ?3 AND status NOT IN ('completed', 'failed')",
|
||||
params![report_cloned.elapsed_sec, point_name, wf, tlusty_status, synspec_status],
|
||||
)?
|
||||
};
|
||||
|
||||
// 审查修复 #S1：workflow 完成 flip 并入同一事务（原实现在 tx.commit() 后单独 UPDATE，
|
||||
// 崩溃窗口期 task 已终态但 workflow 卡 running；且 `let _ =` 丢弃 I/O 错误）。
|
||||
// 现在事务内更新，与 task/grid_points 结算原子提交，错误正常传播。
|
||||
tx.execute(
|
||||
"UPDATE workflows
|
||||
SET status = 'completed', updated_at = datetime('now')
|
||||
WHERE name = ?1
|
||||
AND status = 'running'
|
||||
AND EXISTS (SELECT 1 FROM grid_points WHERE workflow_name = workflows.name)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM grid_points
|
||||
WHERE workflow_name = workflows.name
|
||||
AND status IN ('pending', 'queued', 'running')
|
||||
)",
|
||||
params![wf],
|
||||
)?;
|
||||
|
||||
tx.commit()?;
|
||||
|
||||
Ok(grid_changed > 0)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 单点的全部任务尝试（created_at 升序）。
|
||||
///
|
||||
/// 严格匹配 `workflow_name = ?`：分区迁移前的历史任务（NULL/''）不会出现，
|
||||
/// 这是有意的多工作流隔离语义。
|
||||
pub async fn list_point_attempts(&self, wf: &str, point: &str) -> Result<Vec<AttemptRow>> {
|
||||
let pool = self.pool.clone();
|
||||
let wf = wf.to_string();
|
||||
let point = point.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<AttemptRow>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT task_id, seed_point_name, status, max_relc,
|
||||
atmosphere_has_nan, node_id, error_message, created_at, completed_at,
|
||||
elapsed_sec, failed_stage, summary_json
|
||||
FROM tasks
|
||||
WHERE point_name = ?1 AND workflow_name = ?2
|
||||
ORDER BY created_at ASC, completed_at ASC",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![point, wf], |r| {
|
||||
Ok(AttemptRow {
|
||||
task_id: r.get(0)?,
|
||||
seed_point_name: r.get(1)?,
|
||||
status: r.get(2)?,
|
||||
max_relc: r.get(3)?,
|
||||
atmosphere_has_nan: r.get(4)?,
|
||||
node_id: r.get(5)?,
|
||||
error_message: r.get(6)?,
|
||||
created_at: r.get(7)?,
|
||||
completed_at: r.get(8)?,
|
||||
elapsed_sec: r.get(9)?,
|
||||
failed_stage: r.get(10)?,
|
||||
summary_json: r.get(11)?,
|
||||
})
|
||||
})?;
|
||||
let mut attempts = Vec::new();
|
||||
for r in rows {
|
||||
attempts.push(r?);
|
||||
}
|
||||
Ok(attempts)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
//! 工作流（从原 db.rs 拆分，2026-08-06）。
|
||||
//! `impl Database` 的 工作流 域方法。共享基础设施（Database struct、连接管理、类型、helper）见父模块 `super`（crate::db）。
|
||||
use super::*;
|
||||
|
||||
impl Database {
|
||||
pub async fn upsert_workflow(
|
||||
&self,
|
||||
name: &str,
|
||||
description: Option<&str>,
|
||||
config_yaml: &str,
|
||||
status: &str,
|
||||
) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let desc_owned = description.map(|s| s.to_string());
|
||||
let yaml_owned = config_yaml.to_string();
|
||||
let status_owned = status.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"INSERT INTO workflows (name, description, config_yaml, status, created_at, updated_at)
|
||||
VALUES (?1, ?2, ?3, ?4, datetime('now'), datetime('now'))
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
description = excluded.description,
|
||||
config_yaml = excluded.config_yaml,
|
||||
status = CASE WHEN workflows.status = 'running' THEN workflows.status ELSE excluded.status END,
|
||||
updated_at = datetime('now')",
|
||||
params![name_owned, desc_owned, yaml_owned, status_owned],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 自动巡检所有处于 running 状态的工作流：
|
||||
/// 若某个工作流下的所有网格点均已到达终态（无 pending/queued/running 点），
|
||||
/// 则自动将该工作流的数据库 status 翻转为 'completed'。
|
||||
pub async fn sync_all_running_workflows_completion(&self) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
// L6 修复：原 `let _ =` 丢弃 r2d2/rusqlite 错误——若该后台对账 UPDATE 失败，
|
||||
// 工作流可持续卡在 running 无任何提示。现记录错误（仍返回 Ok 不中断主流程，
|
||||
// 因为每份上报内的 workflow-completion flip 才是主路径，见 record_task_report）。
|
||||
if let Err(e) = conn.execute(
|
||||
"UPDATE workflows
|
||||
SET status = 'completed', updated_at = datetime('now')
|
||||
WHERE status = 'running'
|
||||
AND EXISTS (SELECT 1 FROM grid_points WHERE workflow_name = workflows.name)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM grid_points
|
||||
WHERE workflow_name = workflows.name
|
||||
AND status IN ('pending', 'queued', 'running')
|
||||
)",
|
||||
[],
|
||||
) {
|
||||
tracing::error!("后台对账：同步 running 工作流完成态失败: {}", e);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
pub async fn list_workflows(&self) -> Result<Vec<WorkflowSummary>> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<WorkflowSummary>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT name, description, status, created_at, updated_at FROM workflows ORDER BY updated_at DESC"
|
||||
)?;
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok(WorkflowSummary {
|
||||
name: row.get(0)?,
|
||||
description: row.get(1)?,
|
||||
status: row.get(2)?,
|
||||
created_at: row.get(3)?,
|
||||
updated_at: row.get(4)?,
|
||||
stats: None,
|
||||
})
|
||||
})?;
|
||||
|
||||
let mut list = Vec::new();
|
||||
for r in rows {
|
||||
list.push(r?);
|
||||
}
|
||||
|
||||
// 一次 GROUP BY 聚合全部工作流的网格计数并回填（不做逐工作流查询，无 N+1）。
|
||||
let mut agg_stmt = conn.prepare(
|
||||
"SELECT workflow_name,
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) AS completed,
|
||||
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed,
|
||||
SUM(CASE WHEN status = 'running' THEN 1 ELSE 0 END) AS running,
|
||||
SUM(CASE WHEN status = 'completed' AND tlusty_success_method = 'cold_run' THEN 1 ELSE 0 END) AS cold,
|
||||
SUM(CASE WHEN status = 'completed' AND tlusty_success_method = 'seed_step' THEN 1 ELSE 0 END) AS seed,
|
||||
SUM(CASE WHEN status = 'completed' AND synspec_success_method IS NOT NULL THEN 1 ELSE 0 END) AS synspec
|
||||
FROM grid_points GROUP BY workflow_name",
|
||||
)?;
|
||||
let agg_rows = agg_stmt.query_map([], |r| {
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
WorkflowListStats {
|
||||
total: r.get(1)?,
|
||||
completed: r.get::<_, Option<i64>>(2)?.unwrap_or(0),
|
||||
failed: r.get::<_, Option<i64>>(3)?.unwrap_or(0),
|
||||
running: r.get::<_, Option<i64>>(4)?.unwrap_or(0),
|
||||
cold_run_converged: r.get::<_, Option<i64>>(5)?.unwrap_or(0),
|
||||
seed_step_converged: r.get::<_, Option<i64>>(6)?.unwrap_or(0),
|
||||
synspec_converged: r.get::<_, Option<i64>>(7)?.unwrap_or(0),
|
||||
},
|
||||
))
|
||||
})?;
|
||||
let mut stats_map = std::collections::HashMap::new();
|
||||
for r in agg_rows {
|
||||
let (wf, s) = r?;
|
||||
stats_map.insert(wf, s);
|
||||
}
|
||||
for wf in &mut list {
|
||||
wf.stats = stats_map.remove(&wf.name);
|
||||
}
|
||||
|
||||
Ok(list)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
pub async fn get_workflow(&self, name: &str) -> Result<Option<WorkflowItem>> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
tokio::task::spawn_blocking(move || -> Result<Option<WorkflowItem>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT name, description, config_yaml, status, created_at, updated_at FROM workflows WHERE name = ?1"
|
||||
)?;
|
||||
let row = stmt.query_row(params![name_owned], |row| {
|
||||
Ok(WorkflowItem {
|
||||
name: row.get(0)?,
|
||||
description: row.get(1)?,
|
||||
config_yaml: row.get(2)?,
|
||||
status: row.get(3)?,
|
||||
created_at: row.get(4)?,
|
||||
updated_at: row.get(5)?,
|
||||
})
|
||||
});
|
||||
|
||||
match row {
|
||||
Ok(item) => Ok(Some(item)),
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
pub async fn update_workflow_status(&self, name: &str, status: &str) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let status_owned = status.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
conn.execute(
|
||||
"UPDATE workflows SET status = ?1, updated_at = datetime('now') WHERE name = ?2",
|
||||
params![status_owned, name_owned],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete_workflow(&self, name: &str) -> Result<()> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
let mut conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let tx = conn.transaction()?;
|
||||
tx.execute("DELETE FROM workflows WHERE name = ?1", params![name_owned])?;
|
||||
tx.execute(
|
||||
"DELETE FROM grid_points WHERE workflow_name = ?1",
|
||||
params![name_owned],
|
||||
)?;
|
||||
tx.execute(
|
||||
"DELETE FROM tasks WHERE workflow_name = ?1",
|
||||
params![name_owned],
|
||||
)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn has_running_workflow(&self) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<bool> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let count: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM workflows WHERE status = 'running'",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
Ok(count > 0)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 返回当前处于 running / initializing 状态的**全部**工作流名称。
|
||||
///
|
||||
/// 多工作流并发分区：后台调度需对每个 running 工作流分别派发任务，
|
||||
/// 替代原来「全局只有一个 running workflow」的 LIMIT 1 假设。
|
||||
pub async fn get_running_workflow_names(&self) -> Result<Vec<String>> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<String>> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT name FROM workflows WHERE status IN ('running', 'initializing') ORDER BY updated_at ASC",
|
||||
)?;
|
||||
let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;
|
||||
let mut list = Vec::new();
|
||||
for r in rows {
|
||||
list.push(r?);
|
||||
}
|
||||
Ok(list)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 原子切转工作流至 initializing 预占启动状态，杜绝高并发 POST /start 触发双重全量排队与重置网格竞态
|
||||
pub async fn transition_workflow_to_initializing(&self, name: &str) -> Result<bool> {
|
||||
let pool = self.pool.clone();
|
||||
let name_owned = name.to_string();
|
||||
let affected = tokio::task::spawn_blocking(move || -> Result<usize> {
|
||||
let conn = pool.get().map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let count = conn.execute(
|
||||
"UPDATE workflows SET status = 'initializing', updated_at = datetime('now') WHERE name = ?1 AND status NOT IN ('running', 'initializing')",
|
||||
params![name_owned],
|
||||
)?;
|
||||
Ok(count)
|
||||
})
|
||||
.await??;
|
||||
Ok(affected > 0)
|
||||
}
|
||||
|
||||
/// 获取运行或启动态中的所有工作流 YAML 配置（替代原来低效 N 次循环与嵌套查询）
|
||||
pub async fn get_running_workflow_config_yamls(&self) -> Result<Vec<String>> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<String>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT config_yaml FROM workflows WHERE status IN ('running', 'initializing')",
|
||||
)?;
|
||||
let rows = stmt.query_map([], |row| row.get(0))?;
|
||||
let mut list = Vec::new();
|
||||
for r in rows {
|
||||
list.push(r?);
|
||||
}
|
||||
Ok(list)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// 获取所有处于 `initializing` 态的工作流 (name, config_yaml)。
|
||||
///
|
||||
/// 用于服务端启动恢复：`start_workflow` 把状态切到 `initializing` 后在后台 spawn
|
||||
/// `initialize_grid`。若进程在初始化中途崩溃/重启，工作流会永久卡在 `initializing`
|
||||
/// （`get_running_workflow_names` 仍把它算作可调度，但无人完成网格展开）。
|
||||
/// 启动时检测到这些半初始化工作流后重新跑 `initialize_grid`（幂等，ON CONFLICT DO NOTHING）
|
||||
/// 把状态推进到 `running`，避免半初始化网格被调度。
|
||||
pub async fn get_initializing_workflows(&self) -> Result<Vec<(String, String)>> {
|
||||
let pool = self.pool.clone();
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<(String, String)>> {
|
||||
let conn = pool
|
||||
.get()
|
||||
.map_err(|e| anyhow::anyhow!("DB Pool Error: {}", e))?;
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT name, config_yaml FROM workflows WHERE status = 'initializing'")?;
|
||||
let rows = stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?;
|
||||
let mut list = Vec::new();
|
||||
for r in rows {
|
||||
list.push(r?);
|
||||
}
|
||||
Ok(list)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod api;
|
||||
pub mod cors;
|
||||
pub mod db;
|
||||
pub mod migrations;
|
||||
pub mod scheduler;
|
||||
|
||||
@@ -228,7 +228,17 @@ async fn main() -> Result<()> {
|
||||
// 校验确认真孤儿后重置为 pending 让调度器重新派发，并清除作为判据的
|
||||
// 僵尸 tasks 行（旧实现仅凭 tasks 表 stale pending 行判定，僵尸行使
|
||||
// 判据恒真 → 重复派发涡旋，已废弃）。
|
||||
match bg_scheduler_clone.reclaim_orphaned_points(stale_sec).await {
|
||||
//
|
||||
// 审查修复 #M4：reclaim 用独立且更大的阈值（2 * stale_sec）。reclaim 的语义是
|
||||
// 「孤儿回收」（队列凭证完全丢失），时间尺度应比 requeue 的「claim 超时重投」
|
||||
// 更宽松：刚被 claim 的任务 tasks 行仍 pending，过小阈值会把它误判孤儿候选、
|
||||
// 在 requeue 把队列行打回 pending 到节点重新 claim 的窗口内增加抖动。2x 给
|
||||
// 正常长任务足够缓冲。
|
||||
let reclaim_threshold = stale_sec.saturating_mul(2);
|
||||
match bg_scheduler_clone
|
||||
.reclaim_orphaned_points(reclaim_threshold)
|
||||
.await
|
||||
{
|
||||
Ok(reset) => {
|
||||
if reset > 0 {
|
||||
info!(
|
||||
@@ -494,14 +504,15 @@ async fn security_headers_middleware(
|
||||
|
||||
let headers = resp.headers_mut();
|
||||
// CSP：default-src 'self'；放行 Google Fonts（index.html 引用）；允许 data: 图片。
|
||||
// 已移除 'unsafe-eval'：dashboard 构建产物不使用 eval/new Function（已核实），保留它会
|
||||
// 显著削弱 CSP 的脚本注入防护。'unsafe-inline' 暂留（静态 SPA 内联脚本/handler 需要），
|
||||
// 彻底方案需前端改造为外链 + per-request nonce 注入，见 docs TODO。
|
||||
// 已移除 'unsafe-eval'（dashboard 不用 eval/new Function）与 script-src 'unsafe-inline'
|
||||
// （主题初始化脚本已移为外链 public/theme-init.js，见 dashboard/index.html）。无内联
|
||||
// 脚本/内联事件处理器，外链脚本走 'self' 即可，脚本注入防御最大化。
|
||||
// style-src 保留 'unsafe-inline'：index.html 含内联 style 属性（SVG flex 布局等），移除会破坏渲染。
|
||||
headers
|
||||
.entry(axum::http::header::CONTENT_SECURITY_POLICY)
|
||||
.or_insert_with(|| {
|
||||
HeaderValue::from_static(
|
||||
"default-src 'self'; script-src 'self' 'unsafe-inline'; \
|
||||
"default-src 'self'; script-src 'self'; \
|
||||
style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; \
|
||||
font-src 'self' data: https://fonts.gstatic.com; \
|
||||
connect-src 'self'; img-src 'self' data: blob:; \
|
||||
|
||||
@@ -0,0 +1,565 @@
|
||||
//! 主库版本化迁移基础设施（Phase 0，见 docs/database_refactor_design.md §2）。
|
||||
//!
|
||||
//! 背景：init_tables 内持续堆积手写幂等 ALTER 块，无版本追踪，风险随 schema 演进累积。
|
||||
//! 本模块引入 `PRAGMA user_version` 驱动的版本化迁移，为后续各 Phase 的结构变更
|
||||
//! （P1/P2/P4/P5a/P6）提供统一、可检测、事务化、可中断恢复的迁移通道。
|
||||
//!
|
||||
//! 约定：
|
||||
//! - **V0 = 0**（`PRAGMA user_version` 对全新库的默认值）；后续编号迁移从 1 开始。
|
||||
//! - **新库**：bootstrap 的 CREATE TABLE 始终是最新形态（含各 Phase 新增列）→ 置 V0 →
|
||||
//! 顺序应用 V0+1..N。每个迁移自带 detect 守卫，已存在的列/索引直接跳过 → 新库上所有迁移为 no-op。
|
||||
//! - **旧库**（user_version=0 但表已存在）：bootstrap 幂等补全既有列 → 置 V0 → 应用后续迁移，
|
||||
//! detect 守卫保证只补缺的列/索引，数据零搬运。
|
||||
//! - **幂等性关键**：SQLite 无 `ADD COLUMN IF NOT EXISTS`，迁移必须靠 detect 守卫而非裸 SQL
|
||||
//! 数组实现幂等（审查 CRITICAL#3——否则全新库上 bootstrap 已建新列，迁移再 ADD 会报
|
||||
//! duplicate column 崩启动）。
|
||||
//!
|
||||
//! 队列库（dcts_queue.db）无版本迁移，维持现状；本模块**仅主库**引入版本号。
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use rusqlite::{params, Connection, TransactionBehavior};
|
||||
|
||||
/// 一个版本化迁移。
|
||||
///
|
||||
/// - `version`：> V0(=0) 的顺序号（1..N），`PRAGMA user_version = version` 即代表已应用。
|
||||
/// - `name`：便于日志与审计。
|
||||
/// - `detect`：该迁移是否已应用（列/索引存在性检测）。为 true 时跳过 `up`，仅推进版本号。
|
||||
/// - `up`：未应用时才执行，同一事务内顺序执行。
|
||||
///
|
||||
/// `Clone + Copy`：fn 指针与 `&'static str` 均 Copy，迁移定义可原地复用（测试重跑场景）。
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct Migration {
|
||||
pub version: u32,
|
||||
pub name: &'static str,
|
||||
pub detect: fn(&Connection) -> Result<bool>,
|
||||
pub up: &'static [&'static str],
|
||||
}
|
||||
|
||||
/// 全部迁移。随各 Phase 追加（Phase 0 交付基础设施，Phase 1 起逐个加入）。
|
||||
pub const MIGRATIONS: &[Migration] = &[
|
||||
// M1（Phase 1，P1）：tasks 阶段信息补全。failed_stage = 失败阶段归因，
|
||||
// summary_json = ModelSummary 全保真 JSON。两列均在线 ADD COLUMN，旧节点上报不破坏结算。
|
||||
Migration {
|
||||
version: 1,
|
||||
name: "tasks-stage-info",
|
||||
detect: |c| {
|
||||
Ok(has_column(c, "tasks", "failed_stage")?
|
||||
&& has_column(c, "tasks", "summary_json")?)
|
||||
},
|
||||
up: &[
|
||||
"ALTER TABLE tasks ADD COLUMN failed_stage TEXT",
|
||||
"ALTER TABLE tasks ADD COLUMN summary_json TEXT",
|
||||
],
|
||||
},
|
||||
// M2（Phase 2，P2）：tasks 单列 workflow_name 查询的覆盖索引。
|
||||
// 覆盖 `COUNT(*) WHERE workflow_name=?` 及详情页按工作流统计；用户决策不做 tasks 清理。
|
||||
// CREATE INDEX IF NOT EXISTS 天然幂等，新库/旧库统一由此迁移建立（无需进 init_tables）。
|
||||
Migration {
|
||||
version: 2,
|
||||
name: "tasks-wf-status-created-index",
|
||||
detect: |c| has_index(c, "idx_tasks_wf_status_created"),
|
||||
up: &["CREATE INDEX IF NOT EXISTS idx_tasks_wf_status_created ON tasks(workflow_name, status, created_at)"],
|
||||
},
|
||||
// M4（Phase 4，P5）：清除 node_credentials 死列 revoked。
|
||||
// 新代码不读写它；registration_secret 保留在 nodes（审查 CRITICAL#1/#2：pending 节点无
|
||||
// node_credentials 行，迁移会静默丢凭据；token_hash NOT NULL + 唯一索引塞不下空占位）。
|
||||
// DROP COLUMN 涉及表重建（bundled SQLite 3.45+），部署走低峰窗口 + 手动备份（§11）。
|
||||
Migration {
|
||||
version: 4,
|
||||
name: "drop-revoked-dead-column",
|
||||
detect: |c| Ok(!has_column(c, "node_credentials", "revoked")?),
|
||||
up: &["ALTER TABLE node_credentials DROP COLUMN revoked"],
|
||||
},
|
||||
// M6（Phase 6，P8）：删除 tasks 冗余列 task_type。
|
||||
// 该列与 tlusty_strategies[0] 恒等、synspec-only 场景为"假值"，执行链已改由 strategies[0]
|
||||
// 推导（executor.rs），归因/过滤全部改派生口径。前提：集群无历史节点（用户决策）。
|
||||
// DROP COLUMN 涉及表重建，部署走低峰窗口 + 手动备份（§11）；升级前确认队列为空（§8.8）。
|
||||
Migration {
|
||||
version: 6,
|
||||
name: "drop-task-type-column",
|
||||
detect: |c| Ok(!has_column(c, "tasks", "task_type")?),
|
||||
up: &["ALTER TABLE tasks DROP COLUMN task_type"],
|
||||
},
|
||||
// M7（Phase 5a，P6）：grid_points 补 synspec 收敛归因列。
|
||||
// 与 success_method 镜像的 synspec 分支（光谱以什么策略收敛），解锁「光谱以 standard 等
|
||||
// 策略收敛了多少点」的 SQL 统计；TLUSTY-only 成功保持 NULL。在线 ADD COLUMN，无停写窗口。
|
||||
//
|
||||
// 版本号 = 7（而非 5）：**迁移版本必须与部署顺序单调一致**——§11 部署顺序是 6 → 5a，
|
||||
// 若 5a 编号为 5，则已升到 v6 的库会因 `version <= current` 跳过它，synspec 列永不创建。
|
||||
//（设计 §2.2 的 "M5" 标签是早期命名，此处按部署序改号 M7。）
|
||||
Migration {
|
||||
version: 7,
|
||||
name: "synspec-success-method",
|
||||
detect: |c| has_column(c, "grid_points", "synspec_success_method"),
|
||||
up: &["ALTER TABLE grid_points ADD COLUMN synspec_success_method TEXT"],
|
||||
},
|
||||
// M8（Phase 5b，P6）：grid_points 阶段状态列。
|
||||
// 解除点级单值 status 掩盖两阶段管线：半失败点（大气收敛+光谱失败）可查
|
||||
// tlusty_status='converged' + synspec_status='failed'。NULL = 阶段不适用（tlusty_enabled=0
|
||||
// 或 synspec_enabled=0）。整体 grid_points.status 仍是权威状态，阶段列是补充可查信息。
|
||||
// 在线 ADD COLUMN；同步触点见 docs/database_refactor_design.md §7.3 5b。
|
||||
Migration {
|
||||
version: 8,
|
||||
name: "grid-point-stage-status",
|
||||
detect: |c| {
|
||||
Ok(has_column(c, "grid_points", "tlusty_status")?
|
||||
&& has_column(c, "grid_points", "synspec_status")?)
|
||||
},
|
||||
up: &[
|
||||
"ALTER TABLE grid_points ADD COLUMN tlusty_status TEXT",
|
||||
"ALTER TABLE grid_points ADD COLUMN synspec_status TEXT",
|
||||
],
|
||||
},
|
||||
// M9（Phase 7c）：grid_points.status 值 'converged' → 'completed'。
|
||||
// TLUSTY-first 残留：'converged' 暗示"大气收敛"，实为"管线完成"（大气+光谱）。
|
||||
// 数据迁移 + 全链 SQL 字面量同步（见 db.rs/scheduler.rs，值全部改 'completed'）。
|
||||
Migration {
|
||||
version: 9,
|
||||
name: "grid-status-converged-to-completed",
|
||||
detect: |c| {
|
||||
// 已迁移 = 不再存在旧值 'converged'（detect 检查数据而非列）。
|
||||
// 安全性：apply_migrations_with 以 user_version 闸控——M9 只在 version<9 时评估，
|
||||
// 一旦版本推进到 9 即永不再走此 detect，故即便后续代码意外再写入 'converged'
|
||||
// 也不会触发本迁移重放（user_version 不会回退）。
|
||||
let mut stmt = c.prepare("SELECT 1 FROM grid_points WHERE status = 'converged' LIMIT 1")?;
|
||||
Ok(!stmt.exists([])?)
|
||||
},
|
||||
up: &["UPDATE grid_points SET status = 'completed' WHERE status = 'converged'"],
|
||||
},
|
||||
// M10（Phase 7c）：workflow_progress_snapshots 列名 converged → completed。
|
||||
// 该列存"管线完成点数"，列名随状态值改名保持一致（§9.5 耦合项）。RENAME COLUMN 在线。
|
||||
Migration {
|
||||
version: 10,
|
||||
name: "snapshots-converged-column-rename",
|
||||
detect: |c| has_column(c, "workflow_progress_snapshots", "completed"),
|
||||
up: &["ALTER TABLE workflow_progress_snapshots RENAME COLUMN converged TO completed"],
|
||||
},
|
||||
// M11（H1 活锁修复）：grid_points 补 pending_strategies 列。
|
||||
// 运行时回退（trigger_strategy_fallback）把点打回 pending 等种子时，记录「剩余策略链」
|
||||
// （JSON 数组），调度路径据此用剩余链重派、避免重跑已失败策略导致的无界失败重试活锁。
|
||||
// 在线 ADD COLUMN，无停写窗口。detect 幂等（全新库 bootstrap 已含该列 → 跳过）。
|
||||
Migration {
|
||||
version: 11,
|
||||
name: "grid-point-pending-strategies",
|
||||
detect: |c| has_column(c, "grid_points", "pending_strategies"),
|
||||
up: &["ALTER TABLE grid_points ADD COLUMN pending_strategies TEXT"],
|
||||
},
|
||||
// M12（P9 命名拆分）：rid_points.success_method 值域混用列拆为阶段列 tlusty_success_method。
|
||||
// success_method 原是 TLUSTY-first 整体归因：TLUSTY 任务存 tlusty_strategies[0]
|
||||
// （cold_run/seed_step），synspec-only 任务却存 synspec_strategies[0]（standard）——
|
||||
// 同一列两个值域，前端需猜策略名区分。拆后：
|
||||
// - tlusty_success_method：TLUSTY 阶段策略（tlusty 禁用为 NULL）
|
||||
// - synspec_success_method（既有）：光谱阶段策略（synspec 禁用为 NULL）
|
||||
// 整体归因改由消费方派生（前端 tlusty ?? synspec）。
|
||||
//
|
||||
// 回填判别：旧数据里 synspec-only 点 success_method 与 synspec_success_method 同值
|
||||
// （正是被清理的冗余）；故命中该等式的点不写 tlusty（保持 NULL），其余（正常双阶段
|
||||
// cold_run/seed_step）→ tlusty = 原 success_method。局限：极端情形下 TLUSTY 点
|
||||
// tlusty_strategies[0]==synspec_strategies[0]（如都叫 standard）会被误判为 NULL；
|
||||
// 实际 sdB_cno 中大气策略与光谱策略不冲突，可接受。在线 ADD + UPDATE，无停写窗口。
|
||||
Migration {
|
||||
version: 12,
|
||||
name: "tlusty-success-method",
|
||||
detect: |c| has_column(c, "grid_points", "tlusty_success_method"),
|
||||
up: &[
|
||||
"ALTER TABLE grid_points ADD COLUMN tlusty_success_method TEXT",
|
||||
"UPDATE grid_points SET tlusty_success_method = success_method \
|
||||
WHERE success_method IS NOT NULL \
|
||||
AND NOT (success_method = synspec_success_method AND synspec_success_method IS NOT NULL)",
|
||||
],
|
||||
},
|
||||
// M13（P9 命名拆分）：删除值域混用列 success_method（M12 已回填 tlusty_success_method）。
|
||||
// DROP COLUMN 涉及表重建（bundled SQLite 3.45+），部署走低峰窗口 + 手动备份（§11）。
|
||||
Migration {
|
||||
version: 13,
|
||||
name: "drop-success-method-column",
|
||||
detect: |c| Ok(!has_column(c, "grid_points", "success_method")?),
|
||||
up: &["ALTER TABLE grid_points DROP COLUMN success_method"],
|
||||
},
|
||||
];
|
||||
|
||||
/// 当前 schema 版本（`PRAGMA user_version`）。
|
||||
pub fn current_version(conn: &Connection) -> Result<u32> {
|
||||
Ok(conn.pragma_query_value(None, "user_version", |r| r.get(0))?)
|
||||
}
|
||||
|
||||
/// 从当前版本顺序应用常量表 `MIGRATIONS` 中尚未执行的迁移。
|
||||
pub fn apply_migrations(conn: &mut Connection) -> Result<()> {
|
||||
apply_migrations_with(conn, MIGRATIONS)
|
||||
}
|
||||
|
||||
/// 应用给定迁移列表中尚未执行的部分，每个迁移独立事务（供测试传入自定义列表）。
|
||||
///
|
||||
/// 对每个 `version > current` 的迁移：
|
||||
/// - `detect = true`（已应用，如新库 bootstrap 已建列）→ 仅推进 user_version，不执行 `up`；
|
||||
/// - `detect = false` → `BEGIN IMMEDIATE` → 执行 `up` SQL → `PRAGMA user_version = V` → `COMMIT`。
|
||||
///
|
||||
/// 中途失败不推进版本（进程启动时重试）：当前迁移所在事务回滚，之前迁移的版本号已持久化。
|
||||
fn apply_migrations_with(conn: &mut Connection, migrations: &[Migration]) -> Result<()> {
|
||||
let mut current = current_version(conn)?;
|
||||
for m in migrations {
|
||||
if m.version <= current {
|
||||
continue;
|
||||
}
|
||||
if (m.detect)(conn)? {
|
||||
// 已应用（detect 命中，如全新库 bootstrap 已建列）→ 仅推进版本号，不执行 up。
|
||||
conn.pragma_update(None, "user_version", m.version)?;
|
||||
tracing::info!(
|
||||
version = m.version,
|
||||
name = m.name,
|
||||
"迁移已应用（detect 跳过）"
|
||||
);
|
||||
current = m.version;
|
||||
continue;
|
||||
}
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
for sql in m.up {
|
||||
tx.execute_batch(sql)
|
||||
.with_context(|| format!("迁移 M{} ({}) 失败：{}", m.version, m.name, sql))?;
|
||||
}
|
||||
tx.pragma_update(None, "user_version", m.version)?;
|
||||
tx.commit()?;
|
||||
tracing::info!(version = m.version, name = m.name, "迁移已应用");
|
||||
current = m.version;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 检测表是否已含指定列（PRAGMA table_info）。`table` 必须是 MIGRATIONS 内写死的常量表名，
|
||||
/// 绝不来自外部输入（表名不参与任何用户数据路径）。
|
||||
fn has_column(conn: &Connection, table: &str, column: &str) -> Result<bool> {
|
||||
let mut stmt = conn.prepare(&format!("PRAGMA table_info({table})"))?;
|
||||
let rows = stmt.query_map([], |r| r.get::<_, String>(1))?;
|
||||
for r in rows {
|
||||
if r.map(|name| name == column).unwrap_or(false) {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// 检测索引是否已存在（sqlite_master）。
|
||||
fn has_index(conn: &Connection, index: &str) -> Result<bool> {
|
||||
let mut stmt = conn.prepare("SELECT 1 FROM sqlite_master WHERE type='index' AND name=?1")?;
|
||||
Ok(stmt.exists(params![index])?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn mem_conn() -> Connection {
|
||||
Connection::open_in_memory().unwrap()
|
||||
}
|
||||
|
||||
/// 新库全流程（对齐 Database::new）：bootstrap 已建出最新形态 schema（含 M1 新增列）
|
||||
/// → 版本 0 → apply_migrations 应**无报错**地把版本推进到最新（已存在的列/索引经 detect
|
||||
/// 跳过，不因 duplicate column 崩溃——审查 CRITICAL#3 回归；索引类迁移在最新列上正常建立）。
|
||||
#[test]
|
||||
fn fresh_db_bootstrap_then_migrations_advance_version() {
|
||||
let mut conn = mem_conn();
|
||||
// 模拟 init_tables bootstrap：tasks 是含全部迁移引用列的最新形态（M1 列已存在、
|
||||
// M2 索引目标列已存在但索引本身未建）。
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE tasks (
|
||||
task_id TEXT PRIMARY KEY,
|
||||
point_name TEXT NOT NULL,
|
||||
node_id TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
created_at DATETIME NOT NULL,
|
||||
completed_at DATETIME,
|
||||
workflow_name TEXT,
|
||||
failed_stage TEXT,
|
||||
summary_json TEXT
|
||||
);
|
||||
CREATE TABLE grid_points (
|
||||
name TEXT NOT NULL,
|
||||
workflow_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
tlusty_success_method TEXT,
|
||||
synspec_success_method TEXT,
|
||||
tlusty_status TEXT,
|
||||
synspec_status TEXT,
|
||||
pending_strategies TEXT
|
||||
);
|
||||
CREATE TABLE workflow_progress_snapshots (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
workflow_name TEXT NOT NULL,
|
||||
ts DATETIME NOT NULL,
|
||||
total INTEGER NOT NULL,
|
||||
pending INTEGER NOT NULL,
|
||||
queued INTEGER NOT NULL,
|
||||
running INTEGER NOT NULL,
|
||||
completed INTEGER NOT NULL,
|
||||
failed INTEGER NOT NULL
|
||||
)",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(current_version(&conn).unwrap(), 0);
|
||||
apply_migrations(&mut conn).unwrap();
|
||||
let latest = MIGRATIONS.last().map(|m| m.version).unwrap_or(0);
|
||||
assert_eq!(current_version(&conn).unwrap(), latest);
|
||||
}
|
||||
|
||||
/// 正常迁移：up 顺序执行，版本推进。
|
||||
#[test]
|
||||
fn migration_applies_and_advances_version() {
|
||||
let mut conn = mem_conn();
|
||||
let m = Migration {
|
||||
version: 1,
|
||||
name: "test-add-col",
|
||||
detect: |c| has_column(c, "t", "c"),
|
||||
up: &[
|
||||
"CREATE TABLE t(id INTEGER PRIMARY KEY)",
|
||||
"ALTER TABLE t ADD COLUMN c TEXT",
|
||||
],
|
||||
};
|
||||
apply_migrations_with(&mut conn, &[m]).unwrap();
|
||||
assert_eq!(current_version(&conn).unwrap(), 1);
|
||||
assert!(has_column(&conn, "t", "c").unwrap());
|
||||
}
|
||||
|
||||
/// detect=true（已应用）→ 跳过 up（若执行会 duplicate column 崩），仅推进版本。
|
||||
#[test]
|
||||
fn detect_skip_advances_version_without_running_up() {
|
||||
let mut conn = mem_conn();
|
||||
conn.execute_batch("CREATE TABLE t(id INTEGER PRIMARY KEY, c TEXT)")
|
||||
.unwrap();
|
||||
let m = Migration {
|
||||
version: 1,
|
||||
name: "test-add-col",
|
||||
detect: |c| has_column(c, "t", "c"),
|
||||
up: &["ALTER TABLE t ADD COLUMN c TEXT"],
|
||||
};
|
||||
apply_migrations_with(&mut conn, &[m]).unwrap();
|
||||
assert_eq!(current_version(&conn).unwrap(), 1);
|
||||
}
|
||||
|
||||
/// M4 专项：旧库 node_credentials 含 revoked 死列 → apply_migrations 后列消失、版本推进。
|
||||
/// 同时验证 M1（列已存在跳过）+ M2（索引缺失建立）+ M4（DROP COLUMN）在同库顺序生效。
|
||||
#[test]
|
||||
fn m4_drops_revoked_dead_column_on_old_db() {
|
||||
let mut conn = mem_conn();
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE tasks (
|
||||
task_id TEXT PRIMARY KEY,
|
||||
point_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
created_at DATETIME NOT NULL,
|
||||
workflow_name TEXT,
|
||||
failed_stage TEXT,
|
||||
summary_json TEXT
|
||||
);
|
||||
CREATE TABLE grid_points (
|
||||
name TEXT NOT NULL,
|
||||
workflow_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
success_method TEXT,
|
||||
synspec_success_method TEXT
|
||||
);
|
||||
CREATE TABLE workflow_progress_snapshots (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
workflow_name TEXT NOT NULL,
|
||||
ts DATETIME NOT NULL,
|
||||
total INTEGER NOT NULL,
|
||||
pending INTEGER NOT NULL,
|
||||
queued INTEGER NOT NULL,
|
||||
running INTEGER NOT NULL,
|
||||
converged INTEGER NOT NULL,
|
||||
failed INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE node_credentials (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
token_hash TEXT NOT NULL,
|
||||
issued_at DATETIME NOT NULL,
|
||||
revoked INTEGER NOT NULL DEFAULT 0,
|
||||
raw_token_pending TEXT
|
||||
);",
|
||||
)
|
||||
.unwrap();
|
||||
apply_migrations(&mut conn).unwrap();
|
||||
assert!(
|
||||
!has_column(&conn, "node_credentials", "revoked").unwrap(),
|
||||
"revoked 死列应被 M4 清除"
|
||||
);
|
||||
assert!(
|
||||
has_index(&conn, "idx_tasks_wf_status_created").unwrap(),
|
||||
"M2 索引应建立"
|
||||
);
|
||||
// M10：旧 snapshots 的 converged 列应被重命名为 completed。
|
||||
assert!(has_column(&conn, "workflow_progress_snapshots", "completed").unwrap());
|
||||
assert!(!has_column(&conn, "workflow_progress_snapshots", "converged").unwrap());
|
||||
// M11：grid_points 的 pending_strategies 列应被补齐（H1 活锁修复标记）。
|
||||
assert!(has_column(&conn, "grid_points", "pending_strategies").unwrap());
|
||||
let latest = MIGRATIONS.last().map(|m| m.version).unwrap_or(0);
|
||||
assert_eq!(current_version(&conn).unwrap(), latest);
|
||||
}
|
||||
|
||||
/// M9（Phase 7c）：grid_points.status 值 'converged' → 'completed' 数据迁移。
|
||||
/// 旧库残留 'converged' 值 → apply_migrations 后全部转为 'completed'。
|
||||
#[test]
|
||||
fn m9_converged_status_value_migrated() {
|
||||
let mut conn = mem_conn();
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE tasks (
|
||||
task_id TEXT PRIMARY KEY,
|
||||
point_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
created_at DATETIME NOT NULL,
|
||||
workflow_name TEXT
|
||||
);
|
||||
CREATE TABLE grid_points (
|
||||
name TEXT NOT NULL,
|
||||
workflow_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
success_method TEXT,
|
||||
synspec_success_method TEXT
|
||||
);
|
||||
INSERT INTO grid_points (name, workflow_name, status) VALUES ('p1','wf_a','converged');
|
||||
INSERT INTO grid_points (name, workflow_name, status) VALUES ('p2','wf_a','failed');
|
||||
CREATE TABLE workflow_progress_snapshots (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
workflow_name TEXT NOT NULL,
|
||||
ts DATETIME NOT NULL,
|
||||
total INTEGER NOT NULL,
|
||||
pending INTEGER NOT NULL,
|
||||
queued INTEGER NOT NULL,
|
||||
running INTEGER NOT NULL,
|
||||
converged INTEGER NOT NULL,
|
||||
failed INTEGER NOT NULL
|
||||
);",
|
||||
)
|
||||
.unwrap();
|
||||
apply_migrations(&mut conn).unwrap();
|
||||
// 旧值迁为 'completed'；'failed' 不受影响。
|
||||
let statuses: Vec<String> = {
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT status FROM grid_points ORDER BY name")
|
||||
.unwrap();
|
||||
stmt.query_map([], |r| r.get::<_, String>(0))
|
||||
.unwrap()
|
||||
.filter_map(Result::ok)
|
||||
.collect()
|
||||
};
|
||||
assert_eq!(
|
||||
statuses,
|
||||
vec!["completed".to_string(), "failed".to_string()]
|
||||
);
|
||||
// M10：snapshots 列改名。
|
||||
assert!(has_column(&conn, "workflow_progress_snapshots", "completed").unwrap());
|
||||
}
|
||||
|
||||
/// 中断恢复：M1 成功（版本 1），M2 中途失败（事务回滚，版本停在 1）；
|
||||
/// 修正 M2 后重跑，M1 跳过、M2 成功。
|
||||
#[test]
|
||||
fn interrupted_migration_rolls_back_and_retries() {
|
||||
let mut conn = mem_conn();
|
||||
let m1 = Migration {
|
||||
version: 1,
|
||||
name: "m1",
|
||||
detect: |c| has_column(c, "t", "c1"),
|
||||
up: &[
|
||||
"CREATE TABLE t(id INTEGER PRIMARY KEY)",
|
||||
"ALTER TABLE t ADD COLUMN c1 TEXT",
|
||||
],
|
||||
};
|
||||
let m2_bad = Migration {
|
||||
version: 2,
|
||||
name: "m2-bad",
|
||||
detect: |c| has_column(c, "t", "c2"),
|
||||
up: &["ALTER TABLE t ADD COLUMN c2 TEXT", "THIS IS NOT VALID SQL"],
|
||||
};
|
||||
let err = apply_migrations_with(&mut conn, &[m1, m2_bad]).unwrap_err();
|
||||
assert!(err.to_string().contains("M2 (m2-bad) 失败"));
|
||||
// M1 已提交、版本停在 1；M2 回滚（c2 未建）。
|
||||
assert_eq!(current_version(&conn).unwrap(), 1);
|
||||
assert!(!has_column(&conn, "t", "c2").unwrap());
|
||||
// 重跑：M1（version 1）跳过，M2 修正后成功。
|
||||
let m2_good = Migration {
|
||||
version: 2,
|
||||
name: "m2-good",
|
||||
detect: |c| has_column(c, "t", "c2"),
|
||||
up: &["ALTER TABLE t ADD COLUMN c2 TEXT"],
|
||||
};
|
||||
apply_migrations_with(&mut conn, &[m1, m2_good]).unwrap();
|
||||
assert_eq!(current_version(&conn).unwrap(), 2);
|
||||
assert!(has_column(&conn, "t", "c2").unwrap());
|
||||
}
|
||||
|
||||
/// M12/M13 专项（P9 命名拆分）：旧库 success_method 值域混用列 → 拆为
|
||||
/// tlusty_success_method（TLUSTY 阶段策略）+ 既有 synspec_success_method，再删 success_method。
|
||||
/// 回填判别：synspec-only 点 success_method 与 synspec_success_method 同值（值域混用冗余），
|
||||
/// 命中该等式 → tlusty 保持 NULL；其余（正常双阶段 cold_run/seed_step）→ tlusty = 原值。
|
||||
#[test]
|
||||
fn m12_m13_split_success_method_backfills_and_drops() {
|
||||
let mut conn = mem_conn();
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE grid_points (
|
||||
name TEXT NOT NULL,
|
||||
workflow_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
success_method TEXT,
|
||||
synspec_success_method TEXT
|
||||
);
|
||||
-- 正常双阶段点：success_method = 大气策略 cold_run（≠ 光谱归因 standard）
|
||||
INSERT INTO grid_points (name, workflow_name, status, success_method, synspec_success_method)
|
||||
VALUES ('p_tlusty', 'wf', 'completed', 'cold_run', 'standard');
|
||||
-- 种子步进点
|
||||
INSERT INTO grid_points (name, workflow_name, status, success_method, synspec_success_method)
|
||||
VALUES ('p_seed', 'wf', 'completed', 'seed_step', 'standard');
|
||||
-- synspec-only 点：success_method == synspec_success_method（值域混用冗余）
|
||||
INSERT INTO grid_points (name, workflow_name, status, success_method, synspec_success_method)
|
||||
VALUES ('p_syn', 'wf', 'completed', 'standard', 'standard');
|
||||
-- 失败点：归因为 NULL
|
||||
INSERT INTO grid_points (name, workflow_name, status)
|
||||
VALUES ('p_failed', 'wf', 'failed')",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// M12 单跑：ADD tlusty_success_method + 回填。
|
||||
let m12 = MIGRATIONS.iter().find(|m| m.version == 12).unwrap();
|
||||
apply_migrations_with(&mut conn, &[*m12]).unwrap();
|
||||
let read = |conn: &Connection, name: &str, col: &str| -> Option<String> {
|
||||
// `col` 为测试内写死的列名常量（非用户输入）。
|
||||
conn.query_row(
|
||||
&format!("SELECT {col} FROM grid_points WHERE name = ?1"),
|
||||
rusqlite::params![name],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.ok()
|
||||
};
|
||||
assert_eq!(
|
||||
read(&conn, "p_tlusty", "tlusty_success_method").as_deref(),
|
||||
Some("cold_run"),
|
||||
"双阶段点回填大气策略"
|
||||
);
|
||||
assert_eq!(
|
||||
read(&conn, "p_seed", "tlusty_success_method").as_deref(),
|
||||
Some("seed_step"),
|
||||
"种子步进点回填 seed_step"
|
||||
);
|
||||
assert_eq!(
|
||||
read(&conn, "p_syn", "tlusty_success_method"),
|
||||
None,
|
||||
"synspec-only 点（两列同值）tlusty 保持 NULL"
|
||||
);
|
||||
assert_eq!(read(&conn, "p_failed", "tlusty_success_method"), None);
|
||||
assert_eq!(current_version(&conn).unwrap(), 12);
|
||||
|
||||
// M13 单跑：DROP success_method 列。
|
||||
let m13 = MIGRATIONS.iter().find(|m| m.version == 13).unwrap();
|
||||
apply_migrations_with(&mut conn, &[*m13]).unwrap();
|
||||
assert!(
|
||||
!has_column(&conn, "grid_points", "success_method").unwrap(),
|
||||
"success_method 值域混用列应被删除"
|
||||
);
|
||||
assert!(has_column(&conn, "grid_points", "tlusty_success_method").unwrap());
|
||||
assert!(has_column(&conn, "grid_points", "synspec_success_method").unwrap());
|
||||
// 回填数据在 DROP 后仍保留（tlusty_success_method 是独立列）。
|
||||
assert_eq!(
|
||||
read(&conn, "p_tlusty", "tlusty_success_method").as_deref(),
|
||||
Some("cold_run")
|
||||
);
|
||||
assert_eq!(current_version(&conn).unwrap(), 13);
|
||||
}
|
||||
}
|
||||
+361
-157
File diff suppressed because it is too large
Load Diff
@@ -266,6 +266,17 @@ async fn test_l2_node_token_issue_reissue_flow() {
|
||||
);
|
||||
// 旧 token 仍失效（已被覆盖）
|
||||
assert!(db.find_node_by_token(&token).await.is_none());
|
||||
|
||||
// M2：registration_secret 被首次 take 轮换后，旧 secret 无法取走 reissue 产生的新 token。
|
||||
// 首次 take（上文）已把 secret 轮换为无人知晓的新值，旧 secret（注册时下发的）立即失效。
|
||||
let stolen = db
|
||||
.take_pending_node_token("node-l2-test", secret.as_deref())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
stolen.is_none(),
|
||||
"被轮换的旧 registration_secret 不得取走 reissue 后的新 token（M2 一次性凭据）"
|
||||
);
|
||||
}
|
||||
|
||||
/// 验证中间件对 node token 的端到端鉴权：
|
||||
@@ -1323,9 +1334,9 @@ fn make_import_multipart(
|
||||
body.extend_from_slice(b"Content-Type: application/octet-stream\r\n\r\n");
|
||||
body.extend_from_slice(seed_bytes);
|
||||
body.extend_from_slice(b"\r\n");
|
||||
// 收敛途径字段（cold_run/seed_step）：模拟 import_results 工具判定后透传的途径。
|
||||
// 大气收敛途径字段（cold_run/seed_step）：模拟 import_results 工具判定后透传的途径。
|
||||
body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes());
|
||||
body.extend_from_slice(b"Content-Disposition: form-data; name=\"success_method\"\r\n");
|
||||
body.extend_from_slice(b"Content-Disposition: form-data; name=\"tlusty_success_method\"\r\n");
|
||||
body.extend_from_slice(b"Content-Type: text/plain\r\n\r\n");
|
||||
body.extend_from_slice(success_method.as_bytes());
|
||||
body.extend_from_slice(b"\r\n");
|
||||
@@ -1433,7 +1444,7 @@ async fn test_import_seed_admin_endpoint() {
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("grid_points 应存在");
|
||||
assert_eq!(gp.0, "converged", "导入的收敛点应为 converged 状态");
|
||||
assert_eq!(gp.0, "completed", "导入的收敛点应为 converged 状态");
|
||||
|
||||
// 3. 幂等：重复导入同名点不应报错，状态仍 converged。
|
||||
let body_bytes = make_import_multipart(
|
||||
@@ -1462,7 +1473,7 @@ async fn test_import_seed_admin_endpoint() {
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(gp.0, "converged");
|
||||
assert_eq!(gp.0, "completed");
|
||||
|
||||
// 4. 未收敛点 → 200，但不写 .7、grid_points 维持 pending（未建 converged）。
|
||||
let conv_fail = make_legacy_conv_json("t20000_g5.0_he-2_c-4_n-4_o-4_fail", false);
|
||||
@@ -1570,8 +1581,8 @@ async fn test_import_seed_python_legacy_conv_json() {
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("grid_points 应存在");
|
||||
assert_eq!(row.status, "converged");
|
||||
assert_eq!(row.success_method.as_deref(), Some("cold_run"));
|
||||
assert_eq!(row.status, "completed");
|
||||
assert_eq!(row.tlusty_success_method.as_deref(), Some("cold_run"));
|
||||
assert_eq!(
|
||||
row.last_elapsed_sec,
|
||||
Some(715.0),
|
||||
@@ -1585,7 +1596,7 @@ async fn test_import_seed_python_legacy_conv_json() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_node_disable_enable_flow() {
|
||||
use common::models::{GridAxisValue, GridPointParams, TaskSpec, TaskType};
|
||||
use common::models::{GridAxisValue, GridPointParams, TaskSpec};
|
||||
use uuid::Uuid;
|
||||
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
@@ -1668,7 +1679,6 @@ async fn test_node_disable_enable_flow() {
|
||||
logn: GridAxisValue::from_value(-2.0),
|
||||
logo: GridAxisValue::from_value(-2.0),
|
||||
},
|
||||
task_type: TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 60,
|
||||
workflow_name: Some("wf_test".to_string()),
|
||||
@@ -2120,13 +2130,13 @@ async fn test_admin_set_node_quota_and_heartbeat_sync() {
|
||||
|
||||
/// 测试专用：走真实写路径派发并回报一个网格点任务。
|
||||
///
|
||||
/// `insert_task` → `record_task_report` 会回填 `grid_points.success_method`、
|
||||
/// `insert_task` → `record_task_report` 会回填 `grid_points.tlusty_success_method`、
|
||||
/// `attempt_count` 与 `tasks.completed_at`，与生产链路一致（不绕过任何状态机逻辑）。
|
||||
async fn dispatch_and_report(
|
||||
db: &Database,
|
||||
wf: &str,
|
||||
p: &common::models::GridPointParams,
|
||||
task_type: common::models::TaskType,
|
||||
strategy: &str,
|
||||
seed: Option<String>,
|
||||
converged: bool,
|
||||
) {
|
||||
@@ -2135,11 +2145,15 @@ async fn dispatch_and_report(
|
||||
task_id,
|
||||
point_name: p.model_name(),
|
||||
params: p.clone(),
|
||||
task_type,
|
||||
seed_point_name: seed,
|
||||
timeout_sec: 7200,
|
||||
workflow_name: Some(wf.to_string()),
|
||||
wave: 0,
|
||||
// Phase 6 起策略链首项即"当前策略"（归因/过滤全派生自它）。
|
||||
tlusty_config: common::models::PhaseConfig {
|
||||
strategies: vec![strategy.to_string()],
|
||||
..common::models::PhaseConfig::default_tlusty()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
db.insert_task(&spec).await.unwrap();
|
||||
@@ -2153,7 +2167,7 @@ async fn dispatch_and_report(
|
||||
} else {
|
||||
common::models::TaskStatus::Failed
|
||||
},
|
||||
converged,
|
||||
result_valid: converged,
|
||||
max_relc: if converged { Some(0.0005) } else { Some(9.5e5) },
|
||||
atmosphere_has_nan: false,
|
||||
elapsed_sec: 120.0,
|
||||
@@ -2267,33 +2281,17 @@ async fn test_wf_stats_endpoint() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
dispatch_and_report(
|
||||
&db,
|
||||
"wf_stats",
|
||||
&p_cold,
|
||||
common::models::TaskType::ColdRun,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(&db, "wf_stats", &p_cold, "cold_run", None, true).await;
|
||||
dispatch_and_report(
|
||||
&db,
|
||||
"wf_stats",
|
||||
&p_seed,
|
||||
common::models::TaskType::SeedStep,
|
||||
"seed_step",
|
||||
Some(p_cold.model_name()),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(
|
||||
&db,
|
||||
"wf_stats",
|
||||
&p_failed,
|
||||
common::models::TaskType::ColdRun,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(&db, "wf_stats", &p_failed, "cold_run", None, false).await;
|
||||
db.mark_grid_point_imported(&p_imported.model_name(), "wf_stats", None, "cold_run")
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -2347,7 +2345,7 @@ async fn test_wf_stats_endpoint() {
|
||||
assert_eq!(data["pending"], 1, "pending 与 queued 必须分开计数");
|
||||
assert_eq!(data["queued"], 1);
|
||||
assert_eq!(data["running"], 1);
|
||||
assert_eq!(data["converged"], 3);
|
||||
assert_eq!(data["completed"], 3);
|
||||
assert_eq!(data["failed"], 1);
|
||||
assert_eq!(data["cold_run_converged"], 2);
|
||||
assert_eq!(data["seed_step_converged"], 1);
|
||||
@@ -2356,8 +2354,8 @@ async fn test_wf_stats_endpoint() {
|
||||
assert_eq!(waves.len(), 3);
|
||||
assert_eq!(waves[0]["wave"], 0);
|
||||
assert_eq!(waves[0]["total"], 3);
|
||||
assert_eq!(waves[0]["converged"], 0);
|
||||
assert_eq!(waves[1]["converged"], 2);
|
||||
assert_eq!(waves[0]["completed"], 0);
|
||||
assert_eq!(waves[1]["completed"], 2);
|
||||
assert_eq!(waves[2]["failed"], 1);
|
||||
// 有已完成任务且有剩余点 → ETA 可估算。
|
||||
// P3 后 avg 取精确 elapsed_sec（夹具每次回报 120s），无在线节点按串行兜底：
|
||||
@@ -2436,48 +2434,24 @@ async fn seed_obs_fixture(db: &Database, db_path: &std::path::Path, wf: &str) ->
|
||||
}
|
||||
db.upsert_grid_point(&p_imported, 1, wf).await.unwrap();
|
||||
|
||||
dispatch_and_report(
|
||||
db,
|
||||
wf,
|
||||
&p_cold,
|
||||
common::models::TaskType::ColdRun,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(db, wf, &p_cold, "cold_run", None, true).await;
|
||||
dispatch_and_report(
|
||||
db,
|
||||
wf,
|
||||
&p_seed,
|
||||
common::models::TaskType::SeedStep,
|
||||
"seed_step",
|
||||
Some(p_cold.model_name()),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(
|
||||
db,
|
||||
wf,
|
||||
&p_failed,
|
||||
common::models::TaskType::ColdRun,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(db, wf, &p_failed, "cold_run", None, false).await;
|
||||
// rescued：先冷启动失败，再种子步进救回（2 次尝试，最终 converged/seed_step）
|
||||
dispatch_and_report(db, wf, &p_rescued, "cold_run", None, false).await;
|
||||
dispatch_and_report(
|
||||
db,
|
||||
wf,
|
||||
&p_rescued,
|
||||
common::models::TaskType::ColdRun,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(
|
||||
db,
|
||||
wf,
|
||||
&p_rescued,
|
||||
common::models::TaskType::SeedStep,
|
||||
"seed_step",
|
||||
Some(p_cold.model_name()),
|
||||
true,
|
||||
)
|
||||
@@ -2490,7 +2464,7 @@ async fn seed_obs_fixture(db: &Database, db_path: &std::path::Path, wf: &str) ->
|
||||
conn.execute(
|
||||
"UPDATE tasks SET created_at = datetime('now','-60 seconds'),
|
||||
completed_at = datetime('now','-60 seconds')
|
||||
WHERE workflow_name = ?1 AND NOT (point_name = ?2 AND task_type = 'seed_step')",
|
||||
WHERE workflow_name = ?1 AND NOT (point_name = ?2 AND json_extract(tlusty_strategies, '$[0]') = 'seed_step')",
|
||||
rusqlite::params![wf, p_rescued.model_name()],
|
||||
)
|
||||
.unwrap();
|
||||
@@ -2670,19 +2644,14 @@ async fn test_wf_points_endpoint() {
|
||||
// ---- 10. 最近尝试 JOIN：rescued 双尝试取最新（seed_step 救回） ----
|
||||
let (_, data) = get_points(&app, "/api/workflows/wf_pts/points").await;
|
||||
let rescued = find_point(&data, &n.rescued);
|
||||
assert_eq!(rescued["status"], "converged");
|
||||
assert_eq!(rescued["success_method"], "seed_step");
|
||||
assert_eq!(rescued["status"], "completed");
|
||||
assert_eq!(rescued["tlusty_success_method"], "seed_step");
|
||||
assert_eq!(rescued["attempt_count"], 2, "两次尝试都应计数");
|
||||
assert_eq!(
|
||||
rescued["last_task_type"], "seed_step",
|
||||
"最近尝试应为种子步进"
|
||||
);
|
||||
assert_eq!(rescued["seed_point_name"], n.cold, "种子来源应为 cold 点");
|
||||
assert_eq!(rescued["last_max_relc"], 0.0005);
|
||||
assert_eq!(rescued["last_elapsed_sec"], 120.0, "真实墙钟耗时应落库");
|
||||
// pending 点无任何尝试 → last_* 全 null
|
||||
let pending = find_point(&data, &n.pending);
|
||||
assert!(pending["last_task_type"].is_null());
|
||||
assert!(pending["last_completed_at"].is_null());
|
||||
assert!(pending["last_elapsed_sec"].is_null());
|
||||
assert_eq!(pending["attempt_count"], 0);
|
||||
@@ -2782,13 +2751,13 @@ async fn test_point_detail_endpoint() {
|
||||
let (st, data) = get_detail(&app, &uri).await;
|
||||
assert_eq!(st, StatusCode::OK);
|
||||
assert_eq!(data["point"]["name"], n.cold);
|
||||
assert_eq!(data["point"]["status"], "converged");
|
||||
assert_eq!(data["point"]["success_method"], "cold_run");
|
||||
assert_eq!(data["point"]["status"], "completed");
|
||||
assert_eq!(data["point"]["tlusty_success_method"], "cold_run");
|
||||
let attempts = data["attempts"].as_array().unwrap();
|
||||
assert_eq!(attempts.len(), 1);
|
||||
assert_eq!(attempts[0]["task_type"], "cold_run");
|
||||
assert!(attempts[0]["seed_point_name"].is_null(), "冷启动无种子来源");
|
||||
assert_eq!(attempts[0]["status"], "completed");
|
||||
assert_eq!(data["conv"]["converged"], true, "conv.json 应被解析");
|
||||
assert_eq!(data["conv"]["result_valid"], true, "conv.json 应被解析");
|
||||
assert_eq!(data["conv"]["final_max_relc"], 0.000321);
|
||||
|
||||
// ---- 3. rescued 点：2 次尝试按时间升序（冷启失败 → 种子步进救回），conv 为 null ----
|
||||
@@ -2798,10 +2767,9 @@ async fn test_point_detail_endpoint() {
|
||||
assert_eq!(data["point"]["attempt_count"], 2);
|
||||
let attempts = data["attempts"].as_array().unwrap();
|
||||
assert_eq!(attempts.len(), 2);
|
||||
assert_eq!(attempts[0]["task_type"], "cold_run", "首次应为冷启动");
|
||||
assert!(attempts[0]["seed_point_name"].is_null(), "首次冷启动无种子");
|
||||
assert_eq!(attempts[0]["status"], "failed");
|
||||
assert_eq!(attempts[0]["elapsed_sec"], 120.0, "每次尝试耗时应落库");
|
||||
assert_eq!(attempts[1]["task_type"], "seed_step", "第二次应为种子步进");
|
||||
assert_eq!(attempts[1]["status"], "completed");
|
||||
assert_eq!(
|
||||
attempts[1]["seed_point_name"], n.cold,
|
||||
@@ -2878,15 +2846,7 @@ async fn test_list_workflows_inline_stats() {
|
||||
let p2 = wf_stats_test_params(25000.0, -4.0);
|
||||
db.upsert_grid_point(&p1, 0, "wf_list_a").await.unwrap();
|
||||
db.upsert_grid_point(&p2, 0, "wf_list_a").await.unwrap();
|
||||
dispatch_and_report(
|
||||
&db,
|
||||
"wf_list_a",
|
||||
&p1,
|
||||
common::models::TaskType::ColdRun,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(&db, "wf_list_a", &p1, "cold_run", None, true).await;
|
||||
|
||||
// 无 token → 401
|
||||
let res = app
|
||||
@@ -2923,7 +2883,7 @@ async fn test_list_workflows_inline_stats() {
|
||||
let wf_b = list.iter().find(|w| w["name"] == "wf_list_b").unwrap();
|
||||
|
||||
assert_eq!(wf_a["stats"]["total"], 2);
|
||||
assert_eq!(wf_a["stats"]["converged"], 1);
|
||||
assert_eq!(wf_a["stats"]["completed"], 1);
|
||||
assert_eq!(wf_a["stats"]["cold_run_converged"], 1);
|
||||
assert_eq!(wf_a["stats"]["failed"], 0);
|
||||
assert_eq!(wf_a["stats"]["running"], 0);
|
||||
@@ -2985,15 +2945,7 @@ async fn test_wf_progress_endpoint() {
|
||||
db.record_progress_snapshot("wf_prog").await.unwrap(),
|
||||
"首次记录应写入"
|
||||
);
|
||||
dispatch_and_report(
|
||||
&db,
|
||||
"wf_prog",
|
||||
&p1,
|
||||
common::models::TaskType::ColdRun,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
dispatch_and_report(&db, "wf_prog", &p1, "cold_run", None, true).await;
|
||||
assert!(
|
||||
db.record_progress_snapshot("wf_prog").await.unwrap(),
|
||||
"计数变化应写入"
|
||||
@@ -3008,7 +2960,7 @@ async fn test_wf_progress_endpoint() {
|
||||
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||
conn.execute(
|
||||
"UPDATE workflow_progress_snapshots SET ts = datetime('now', '-2 hours') \
|
||||
WHERE workflow_name = 'wf_prog' AND converged = 0",
|
||||
WHERE workflow_name = 'wf_prog' AND completed = 0",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
@@ -3057,7 +3009,7 @@ async fn test_wf_progress_endpoint() {
|
||||
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||
conn.execute(
|
||||
"UPDATE workflow_progress_snapshots SET ts = datetime('now', '-10 days') \
|
||||
WHERE workflow_name = 'wf_prog' AND converged = 0",
|
||||
WHERE workflow_name = 'wf_prog' AND completed = 0",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
@@ -3171,7 +3123,6 @@ async fn test_duplicate_failure_report_cannot_flip_converged() {
|
||||
task_id: uuid::Uuid::new_v4(),
|
||||
point_name: name.clone(),
|
||||
params: params.clone(),
|
||||
task_type: common::models::TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 7200,
|
||||
workflow_name: Some("wf_flip".to_string()),
|
||||
@@ -3201,7 +3152,7 @@ async fn test_duplicate_failure_report_cannot_flip_converged() {
|
||||
params: Some(params.clone()),
|
||||
node_id: "node-flip".to_string(),
|
||||
status: common::models::TaskStatus::Completed,
|
||||
converged: true,
|
||||
result_valid: true,
|
||||
max_relc: Some(0.0005),
|
||||
atmosphere_has_nan: false,
|
||||
elapsed_sec: 120.0,
|
||||
@@ -3231,7 +3182,7 @@ async fn test_duplicate_failure_report_cannot_flip_converged() {
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.0,
|
||||
"converged"
|
||||
"completed"
|
||||
);
|
||||
|
||||
// ---- 任务 B：迟到的重复失败报告（涡旋残留任务的典型行为）----
|
||||
@@ -3239,7 +3190,6 @@ async fn test_duplicate_failure_report_cannot_flip_converged() {
|
||||
task_id: uuid::Uuid::new_v4(),
|
||||
point_name: name.clone(),
|
||||
params: params.clone(),
|
||||
task_type: common::models::TaskType::ColdRun,
|
||||
seed_point_name: None,
|
||||
timeout_sec: 7200,
|
||||
workflow_name: Some("wf_flip".to_string()),
|
||||
@@ -3269,7 +3219,7 @@ async fn test_duplicate_failure_report_cannot_flip_converged() {
|
||||
params: Some(params.clone()),
|
||||
node_id: "node-flip".to_string(),
|
||||
status: common::models::TaskStatus::Failed,
|
||||
converged: false,
|
||||
result_valid: false,
|
||||
max_relc: Some(9.5e5),
|
||||
atmosphere_has_nan: false,
|
||||
elapsed_sec: 130.0,
|
||||
@@ -3305,7 +3255,7 @@ async fn test_duplicate_failure_report_cannot_flip_converged() {
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.0,
|
||||
"converged",
|
||||
"completed",
|
||||
"迟到失败报告不得翻黑 converged 点"
|
||||
);
|
||||
assert!(
|
||||
@@ -3384,7 +3334,7 @@ async fn test_save_workflow_validates_stage_configs() {
|
||||
|
||||
// 1. 双阶段全关 → 400
|
||||
let both_off = format!(
|
||||
"{}\ntlusty:\n enabled: false\n policy: skip_converged\n strategies: [cold_run, seed_step]\n\
|
||||
"{}\ntlusty_stage:\n enabled: false\n policy: skip_converged\n strategies: [cold_run, seed_step]\n\
|
||||
synspec_stage:\n enabled: false\n policy: skip_converged\n strategies: [standard]\n",
|
||||
base_yaml
|
||||
);
|
||||
@@ -3393,7 +3343,7 @@ async fn test_save_workflow_validates_stage_configs() {
|
||||
|
||||
// 2. 启用阶段空策略链 → 400
|
||||
let empty_chain = format!(
|
||||
"{}\ntlusty:\n enabled: true\n policy: skip_converged\n strategies: []\n",
|
||||
"{}\ntlusty_stage:\n enabled: true\n policy: skip_converged\n strategies: []\n",
|
||||
base_yaml
|
||||
);
|
||||
let res = post_save("wf_val_b", &empty_chain).await;
|
||||
@@ -3405,7 +3355,7 @@ async fn test_save_workflow_validates_stage_configs() {
|
||||
|
||||
// 3. 合法：TLUSTY 关 + SYNSPEC 启（设计 §2.2 场景 B：仅更新光谱）→ 200
|
||||
let syn_only = format!(
|
||||
"{}\ntlusty:\n enabled: false\n policy: skip_converged\n strategies: [cold_run, seed_step]\n\
|
||||
"{}\ntlusty_stage:\n enabled: false\n policy: skip_converged\n strategies: [cold_run, seed_step]\n\
|
||||
synspec_stage:\n enabled: true\n policy: force_recompute\n strategies: [standard]\n",
|
||||
base_yaml
|
||||
);
|
||||
|
||||
@@ -66,7 +66,7 @@ async fn test_same_workflow_name_preserves_converged() {
|
||||
let st = db.get_grid_point_status(name, "sdB_cno").await.unwrap();
|
||||
assert_eq!(
|
||||
st.unwrap().0,
|
||||
"converged",
|
||||
"completed",
|
||||
"同名工作流：导入的 converged 应被保留，避免重算"
|
||||
);
|
||||
println!("✓ 场景1（同名）：status=converged，旧结果被保留，不会重算");
|
||||
@@ -96,7 +96,7 @@ async fn test_different_workflow_name_causes_recompute() {
|
||||
|
||||
// imported 分区：converged（种子库有，但不会被 sdB_cno 调度看到）
|
||||
let st_imp = db.get_grid_point_status(name, "imported").await.unwrap();
|
||||
assert_eq!(st_imp.unwrap().0, "converged");
|
||||
assert_eq!(st_imp.unwrap().0, "completed");
|
||||
|
||||
// sdB_cno 分区：pending（重新算！看不到 imported 的 converged）
|
||||
let st_real = db.get_grid_point_status(name, "sdB_cno").await.unwrap();
|
||||
@@ -138,7 +138,7 @@ async fn test_mixed_grid_import_then_init_avoids_recompute() {
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
st_old.unwrap().0,
|
||||
"converged",
|
||||
"completed",
|
||||
"旧点应保持 converged 不重算"
|
||||
);
|
||||
|
||||
@@ -192,7 +192,7 @@ async fn test_precision_diff_import_then_init_preserves_converged() {
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
st.unwrap().0,
|
||||
"converged",
|
||||
"completed",
|
||||
"精度一致(g5.0=g5.0)时导入的 converged 必须保留，不重算"
|
||||
);
|
||||
println!("✓ 场景4（精度差异命门）：g5.0 入库 + initialize_grid → converged 保留，避免重算");
|
||||
|
||||
Reference in New Issue
Block a user