fmq/DCTS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server,dashboard): 引入多工作流数据隔离、安全中间件与前端 ESM 模块化重构

Browse Source 
- server: 实现按 workflow_name 的多工作流数据隔离与旧数据库平滑迁移机制
- server: 新增 API Key 认证(auth)、限流中间件(rate_limit)与运维备份接口(admin)
- server: 统一 AppError 错误处理体系，重构调度器 scheduler 支持工作流级重置与抢占
- node: 节点 ID 缺失时自动生成随机 UUID，原生支持 `docker compose --scale node=N` 动态扩容
- dashboard: 前端模块化重构(state/api/components)，升级 CSS 变量设计系统与 Toast 通知
- docker/docs: 更新 /healthz 健康检查、部署脚本 IP 配置及数据库设计文档
This commit is contained in:
fmq
2026-07-28 21:54:02 +08:00
parent 4b4238d702
commit b91f1e4fa5
59 changed files with 7697 additions and 1490 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/server/src/main.rs
+280 -43
View File
@@ -3,8 +3,9 @@ use server::api::{self, AppState};
use server::db::Database;
use server::scheduler::GridScheduler;
use axum::{
extract::DefaultBodyLimit,
http::HeaderValue,
routing::{get, post},
Router,
};
@@ -16,12 +17,15 @@ use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tokio::time::{sleep, Duration};
use tower_http::cors::CorsLayer;
use tower_http::services::{ServeDir, ServeFile};
use tracing::info;
#[derive(Parser, Debug)]
#[command(name = "server", version = "0.1.0", about = "Distributed Computing TLUSTY/SYNSPEC (DCTS) Server")]
#[command(
name = "server",
version = "0.1.0",
about = "Distributed Computing TLUSTY/SYNSPEC (DCTS) Server"
)]
struct CliArgs {
/// Optional path to workflow configuration YAML file to auto-register on startup
#[arg(short = 'w', long = "workflow")]
@@ -61,12 +65,15 @@ async fn main() -> Result<()> {
let default_wf_path = Path::new(&server_cfg.grid_config);
if default_wf_path.is_file() {
if let Ok(yaml_content) = std::fs::read_to_string(default_wf_path) {
if let Err(e) = db.upsert_workflow(
"sdB_cno",
Some("sdB CNO 6D Stellar Atmosphere Grid"),
&yaml_content,
"idle",
).await {
if let Err(e) = db
.upsert_workflow(
"sdB_cno",
Some("sdB CNO 6D Stellar Atmosphere Grid"),
&yaml_content,
"idle",
)
.await
{
tracing::warn!("预注册默认工作流失败: {}", e);
} else {
info!("已在数据库中成功预注册默认工作流 'sdB_cno'");
@@ -74,15 +81,38 @@ async fn main() -> Result<()> {
}
}
// 弱口令凭据安全警告检测
let is_weak_token = |t: Option<&str>| -> bool {
match t {
Some(s) => {
s.len() < 12 || s == "fmqi123" || s == "admin" || s == "123456" || s == "secret"
}
None => false,
}
};
if is_weak_token(server_cfg.auth_token.as_deref())
|| is_weak_token(server_cfg.admin_token.as_deref())
{
tracing::warn!("⚠️ 检测到系统当前正在使用弱口令凭据或默认 Token！建议生产环境在 .env 中配置使用 openssl rand -hex 32 生成的高强度 Token！");
}
let rate_limiter = api::rate_limit::RateLimiter::new(5, std::time::Duration::from_secs(300));
let state = AppState {
db,
queue: queue.clone(),
scheduler: scheduler.clone(),
results_dir: server_cfg.results_dir,
results_dir: server_cfg.results_dir.clone(),
rate_limiter,
auth_token: server_cfg.auth_token.clone(),
admin_token: server_cfg.admin_token.clone(),
auth_disabled: server_cfg.auth_disabled,
admin_sessions: std::sync::Arc::new(tokio::sync::RwLock::new(
std::collections::HashMap::new(),
)),
};
// Background loop for stale task requeueing, offline node detection, and scheduler checking
// Background maintenance & scheduling with Exponential Backoff
let bg_db = state.db.clone();
let bg_queue = queue.clone();
let bg_scheduler = scheduler.clone();
@@ -90,58 +120,226 @@ async fn main() -> Result<()> {
let node_stale_sec = server_cfg.node_stale_sec;
tokio::spawn(async move {
let mut fail_count: u32 = 0;
let mut first_run = true;
loop {
sleep(Duration::from_secs(30)).await;
if let Ok(requeued_points) = bg_queue.requeue_stale_tasks(stale_sec).await {
if !requeued_points.is_empty() {
info!("重新将 {} 个超时/掉线任务放回待计算队列", requeued_points.len());
let _ = bg_db.reset_specific_grid_points_to_pending(&requeued_points).await;
}
if first_run {
first_run = false;
} else {
let base_delay = 30u64;
let current_delay = if fail_count == 0 {
base_delay
} else {
(base_delay * (1u64 << fail_count.min(4))).min(300)
};
sleep(Duration::from_secs(current_delay)).await;
}
if let Ok(offline) = bg_db.mark_stale_nodes_offline(node_stale_sec).await {
if offline > 0 {
info!("已标记 {} 个心跳超时的计算节点为离线状态", offline);
let bg_db_clone = bg_db.clone();
let bg_queue_clone = bg_queue.clone();
let bg_scheduler_clone = bg_scheduler.clone();
let join_handle = tokio::spawn(async move {
let mut has_error = false;
match bg_queue_clone.requeue_stale_tasks(stale_sec).await {
Ok(requeued) => {
if !requeued.is_empty() {
info!("重新将 {} 个超时/掉线任务放回待计算队列", requeued.len());
let mut by_wf: std::collections::HashMap<String, Vec<String>> =
std::collections::HashMap::new();
for (point, wf) in &requeued {
by_wf
.entry(wf.clone().unwrap_or_default())
.or_default()
.push(point.clone());
}
for (wf, points) in by_wf {
let _ = bg_db_clone
.reset_specific_grid_points_to_pending(&points, &wf)
.await;
}
}
}
Err(e) => {
tracing::warn!("重投超时任务失败: {}", e);
has_error = true;
}
}
match bg_db_clone.mark_stale_nodes_offline(node_stale_sec).await {
Ok(offline) => {
if offline > 0 {
info!("已标记 {} 个心跳超时的计算节点为离线状态", offline);
}
}
Err(e) => {
tracing::warn!("标记超时节点离线失败: {}", e);
has_error = true;
}
}
if let Err(e) = bg_scheduler_clone.schedule_pending_tasks().await {
tracing::warn!("后台定时性任务调度检测失败: {}", e);
has_error = true;
}
if let Err(e) = bg_db_clone.sync_all_running_workflows_completion().await {
tracing::warn!("后台同步已完成工作流状态失败: {}", e);
has_error = true;
}
has_error
});
match join_handle.await {
Ok(has_error) => {
if has_error {
fail_count = fail_count.saturating_add(1);
} else {
fail_count = 0;
}
}
Err(e) => {
tracing::error!("后台维护任务内部发生 Panic: {:?}", e);
fail_count = fail_count.saturating_add(1);
}
}
if let Err(e) = bg_scheduler.schedule_pending_tasks().await {
tracing::warn!("后台定时性任务调度检测失败: {}", e);
}
}
});
// 每天自动触发一次数据库备份。
// 备份目录跟随 server_cfg.backup_dir（DCTS_BACKUP_DIR，默认 data/backups），
// 与 DB_PATH 解耦，避免 DB 卷与备份卷不一致时备份落到未持久化层。
// 首次延迟 1 小时，避免频繁重启（如调试阶段）短时间堆积备份文件；backup_database
// 自身还带有 7 天保留期清理兜底。
let backup_db = state.db.clone();
let backup_dir = server_cfg.backup_dir.clone();
tokio::spawn(async move {
sleep(Duration::from_secs(3600)).await;
loop {
if let Err(e) = backup_db.backup_database(&backup_dir).await {
tracing::warn!("自动备份数据库失败: {}", e);
}
sleep(Duration::from_secs(24 * 3600)).await;
}
});
// 大体积上传端点单独拎出，套用更宽松的 body limit（256MB，覆盖收敛种子 .7 文件量级）
// 并限制并发数：每个 report 请求最多 256MB 驻留内存，无并发上限时 N 个请求可耗尽内存。
// 限流后超出并发数的请求排队等待（而非直接拒绝），保证正常业务不被误伤。
// 其余 API 用 10MB 默认上限，防止大文件内存耗尽 DoS。
const REPORT_BODY_LIMIT: usize = 256 * 1024 * 1024;
const DEFAULT_BODY_LIMIT: usize = 10 * 1024 * 1024;
const REPORT_MAX_CONCURRENCY: usize = 4;
let report_router = Router::new()
.route("/task/report", post(api::task::report_task))
.layer(DefaultBodyLimit::max(REPORT_BODY_LIMIT))
.layer(tower::ServiceBuilder::new().concurrency_limit(REPORT_MAX_CONCURRENCY));
// 节点注册接口独立 IP 限流保护（每分钟最多 10 次申请，无论成败都计数，防恶意频繁注册）
// 使用 new_count_all：此 limiter 专挂 /node/register，对注册路径的所有响应计入窗口。
// 通用 API 限流器（见下方 auth_enabled 分支）用 new 构造（count_all=false），不会因
// 成功注册把 IP 锁出整个 /api/*，避免跨端点连锁限流。
let register_limiter =
api::rate_limit::RateLimiter::new_count_all(10, std::time::Duration::from_secs(60));
let register_rate_limit_layer = axum::middleware::from_fn_with_state(
register_limiter,
api::rate_limit::rate_limit_middleware,
);
let api_router = Router::new()
// Auth API
.route("/login", post(api::auth::login))
.route("/auth/check", get(api::auth::check_auth))
// Core Node & Task API
.route("/node/register", post(api::node::register_node))
.route(
"/node/register",
post(api::node::register_node).layer(register_rate_limit_layer),
)
.route("/node/check_status", post(api::node::check_node_status))
.route("/node/heartbeat", post(api::node::heartbeat_node))
.route("/task/claim", post(api::task::claim_task))
.route("/task/report", post(api::task::report_task))
.route("/seed/:name", get(api::seed::download_seed))
.route("/status", get(api::status::get_status))
// Static Data API
.route("/data/file/*filename", get(api::data::download_single_data_file))
.route(
"/data/file/*filename",
get(api::data::download_single_data_file),
)
.route("/data/linelist", get(api::data::download_linelist))
// Workflow Management CRUD API
.route("/workflows", get(api::workflow::list_workflows).post(api::workflow::save_workflow))
.route("/workflows/:name", get(api::workflow::get_workflow).put(api::workflow::save_workflow).delete(api::workflow::delete_workflow))
.route("/workflows/:name/start", post(api::workflow::start_workflow))
.route("/workflows/:name/stop", post(api::workflow::stop_workflow));
.route(
"/workflows",
get(api::workflow::list_workflows).post(api::workflow::save_workflow),
)
.route(
"/workflows/:name",
get(api::workflow::get_workflow)
.put(api::workflow::save_workflow)
.delete(api::workflow::delete_workflow),
)
.route(
"/workflows/:name/start",
post(api::workflow::start_workflow),
)
.route("/workflows/:name/stop", post(api::workflow::stop_workflow))
// Admin Management API（节点凭据查看/审批/吊销/重发，均要求 Admin 角色）
.route("/admin/nodes", get(api::admin::list_nodes))
.route(
"/admin/nodes/:node_id/approve",
post(api::admin::approve_node),
)
.route(
"/admin/nodes/:node_id/reject",
post(api::admin::reject_node),
)
.route(
"/admin/nodes/:node_id/revoke",
post(api::admin::revoke_node),
)
.route(
"/admin/nodes/:node_id/reissue",
post(api::admin::reissue_node),
)
// 合并大体积上报路由（继承各自的 body limit）
.merge(report_router)
.layer(DefaultBodyLimit::max(DEFAULT_BODY_LIMIT));
let api_router = if state.auth_token.is_some() {
info!("已为 DCTS 服务端 API 路由启用 Bearer Token / X-API-Key 访问控制鉴权");
// 鉴权启用条件：未应急关闭，且配置了 admin 凭据。
let auth_enabled = !state.auth_disabled && state.admin_token.is_some();
let api_router = if auth_enabled {
info!("已启用 API 身份鉴权保护（Admin 端点需 admin token 验证；Node 节点免 Token 提交申请，经 Dashboard 管理员审批授权下发）");
// 鉴权失败限流（防 token 在线暴力）：外层先判 IP 限流，内层再做鉴权。
// 限流状态为 20 次/分钟（按 IP），超阈值返回 429。
let limiter = api::rate_limit::RateLimiter::new(20, std::time::Duration::from_secs(60));
let rate_limit_layer =
axum::middleware::from_fn_with_state(limiter, api::rate_limit::rate_limit_middleware);
let auth_layer = axum::middleware::from_fn_with_state(state.clone(), api::auth_middleware);
api_router.layer(auth_layer)
api_router.layer(auth_layer).layer(rate_limit_layer)
} else {
tracing::warn!("⚠️ 警告：未检测到 DCTS_AUTH_TOKEN 环境变量，服务端目前运行在【内网无鉴权模式】！所有 REST API 接口均为公开可访问状态。");
tracing::warn!(
"⚠️ 警告：未配置 DCTS_ADMIN_TOKEN / DCTS_ENROLLMENT_TOKEN（且未启用 DCTS_AUTH_DISABLE），\
服务端运行在【无鉴权模式】！公网部署务必配置凭据。"
);
api_router
};
// Host Dashboard SPA static files from dashboard/dist if directory exists or fallback to index.html
let serve_dir = ServeDir::new("dashboard/dist")
.fallback(ServeFile::new("dashboard/dist/index.html"));
let serve_dir =
ServeDir::new("dashboard/dist").fallback(ServeFile::new("dashboard/dist/index.html"));
// 安全响应头（CSP / nosniff / DENY / Referrer-Policy）。
let security_headers = axum::middleware::from_fn(security_headers_middleware);
let app = Router::new()
// 独立健康检查端点：不走鉴权、不走 CORS/body 限制，专供 docker healthcheck 与外部监控探测。
// 开启鉴权后 /api/status 会返回 401，导致容器被判定不健康而反复重启，故单独提供 /healthz。
.route("/healthz", get(api::status::healthz))
.nest("/api", api_router)
.layer(CorsLayer::permissive())
.layer(server::cors::build_cors_layer())
.layer(security_headers)
.fallback_service(serve_dir)
.with_state(state);
@@ -149,14 +347,53 @@ async fn main() -> Result<()> {
info!("DCTS 服务端已在 http://{} 启动监听", addr);
let listener = tokio::net::TcpListener::bind(addr).await?;
axum::serve(listener, app)
.with_graceful_shutdown(async {
let _ = tokio::signal::ctrl_c().await;
info!("收到 Ctrl+C 终止信号，DCTS 服务端准备优雅关闭...");
})
.await?;
// into_make_service_with_connect_info：让限流中间件能从连接拿到客户端 IP（反代场景则用 X-Forwarded-For）
axum::serve(
listener,
app.into_make_service_with_connect_info::<SocketAddr>(),
)
.with_graceful_shutdown(async {
let _ = tokio::signal::ctrl_c().await;
info!("收到 Ctrl+C 终止信号，DCTS 服务端准备优雅关闭...");
})
.await?;
info!("DCTS 服务端已安全关闭。");
Ok(())
}
/// 注入安全响应头的中间件函数。
async fn security_headers_middleware(
req: axum::http::Request<axum::body::Body>,
next: axum::middleware::Next,
) -> axum::response::Response {
let mut resp = next.run(req).await;
let headers = resp.headers_mut();
// CSP：default-src 'self'；放行 Google Fonts（index.html 引用）；允许 data: 图片。
// 已移除 'unsafe-eval'：dashboard 构建产物不使用 eval/new Function（已核实），保留它会
// 显著削弱 CSP 的脚本注入防护。'unsafe-inline' 暂留（静态 SPA 内联脚本/handler 需要），
// 彻底方案需前端改造为外链 + per-request nonce 注入，见 docs TODO。
headers
.entry(axum::http::header::CONTENT_SECURITY_POLICY)
.or_insert_with(|| {
HeaderValue::from_static(
"default-src 'self'; script-src 'self' 'unsafe-inline'; \
style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; \
font-src 'self' data: https://fonts.gstatic.com; \
connect-src 'self'; img-src 'self' data: blob:; \
frame-ancestors 'none'",
)
});
headers
.entry(axum::http::header::X_CONTENT_TYPE_OPTIONS)
.or_insert_with(|| HeaderValue::from_static("nosniff"));
headers
.entry(axum::http::header::X_FRAME_OPTIONS)
.or_insert_with(|| HeaderValue::from_static("DENY"));
headers
.entry(axum::http::HeaderName::from_static("referrer-policy"))
.or_insert_with(|| HeaderValue::from_static("strict-origin-when-cross-origin"));
resp
}