fmq/DCTS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(all): 源精度命名体系、工作流可观测台、节点停用管理与白名单归档

Browse Source 
核心变更：

  1. GridAxisValue 源精度命名
     - 新增 GridAxisValue 类型，携带 f64 数值 + YAML 源书写文本（Deref 透明兼容算术）
     - config.rs 绕过 serde_yaml 归一化，逐 token 捕获轴值原文（logg: 5.0 → g5.0）
     - runner/executor/scheduler 全链路改用 DB TEXT 列权威 point_name，
       修复 REAL 列回读丢精度导致的 model_name 错配

  2. 工作流执行可观测台
     - 新增 stats/progress/points 三组 API（进度时间序列、经验速率 ETA、
       停滞预警、逐点明细分页、收敛性热力图数据）
     - 新增 workflow_progress_snapshots 表 + tasks/grid_points 耗时列
     - runner 携带 last_iter/worst_depth/n_depths 进 conv.json
     - 前端新增 hash 路由、工作流详情页（概览/网格点/收敛分析三 Tab）、YAML 编辑器

  3. 节点停用/启用管理
     - 新增 disabled 状态 + disable/enable API；停用节点保持心跳但停止分发，
       worker 空闲待命而非退出；移除 revoke API，token 失效统一走重发覆盖；
       移除 host_name 字段

  4. 白名单结果归档
     - 新增 result_filter 模块，只归档有语义产物，丢弃 Tlusty 中间单元（~2MB/模型）
     - executor 原子写入归档 + 200 点 LRU 上限

  5. 历史数据导入
     - sync_seeds 重写为 import_results：经 /admin/import_seed 标记 converged +
       按新版命名迁移产物树

  6. 部署与目录重规划
     - data/results→seeds、data/archive→result + migrate_data_dirs.sh
     - deploy.sh 增强（SSH 复用、Profile、远程 env）；Dockerfile 瘦身

  7. 文档同步更新 api/database/architecture/deployment
This commit is contained in:
fmq
2026-07-31 01:34:05 +08:00
parent b91f1e4fa5
commit 1bfa240cb0
73 changed files with 12332 additions and 1608 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/deploy.sh
+546 -54
View File
@@ -12,13 +12,14 @@
# ./scripts/deploy.sh -e remote -b compose -r all --skip-assets # 跳过静态资源同步，仅更新程序代码
# ==============================================================================
set -e
set -eo pipefail
GREEN='\033[0;32m'
BLUE='\033[0;34m'
RED='\033[0;31m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
MAGENTA='\033[0;35m'
NC='\033[0m' # No Color
# 定位至项目根目录
@@ -30,15 +31,260 @@ cd "${WORK_DIR}"
# 默认配置与环境变量
# =============================================================================
REMOTE_USER="${REMOTE_USER:-fmq}"
REMOTE_IP="${REMOTE_IP:-192.168.5.3}"
REMOTE_IP="${REMOTE_IP:-100.66.1.2}"
REMOTE_PORT="${REMOTE_PORT:-22}"
REMOTE_DIR="${REMOTE_DIR:-/vol1/1000/program/dcts}"
# SSH ControlMaster 连接复用目录 (全局统一，使父子/兄弟进程共享 socket)
SSH_CONTROL_DIR="${SSH_CONTROL_DIR:-$HOME/.ssh/cm}"
ENV_MODE="${DEPLOY_ENV:-}"
BACKEND_MODE="${DEPLOY_BACKEND:-}"
ROLE_TARGET="${DEPLOY_ROLE:-}"
ACTION="install"
SYNC_ASSETS="auto" # auto / skip / force
PROFILE_FILE=""
PROFILE_LOADED=""
PASSED_CLI_ARGS=("$@")
load_profile_file() {
local pfile="$1"
if [ -f "$pfile" ]; then
echo -e "${CYAN}[i] 正在加载 Profile 配置文件: ${pfile}${NC}"
set -a
# shellcheck disable=SC1090
source "$pfile"
set +a
PROFILE_LOADED="$pfile"
# 优先级: 命令行显式参数 > Profile > 默认值。仅在命令行未指定时才采用 Profile 的值。
[ -z "${CLI_ENV_SET:-}" ] && [ -n "${DEPLOY_ENV}" ] && ENV_MODE="${DEPLOY_ENV}"
[ -z "${CLI_BACKEND_SET:-}" ] && [ -n "${DEPLOY_BACKEND}" ] && BACKEND_MODE="${DEPLOY_BACKEND}"
[ -z "${CLI_ROLE_SET:-}" ] && [ -n "${DEPLOY_ROLE}" ] && ROLE_TARGET="${DEPLOY_ROLE}"
if [ -z "${CLI_ASSETS_SET:-}" ]; then
[ -n "${SYNC_ASSETS}" ] || SYNC_ASSETS="auto"
fi
[ -n "${ACTION}" ] || ACTION="install"
return 0
else
echo -e "${RED}错误: 找不到 Profile 配置文件 '${pfile}'${NC}"
exit 1
fi
}
write_remote_runtime_env() {
local is_win="$1"
local env_content=""
if [ -n "${PROFILE_LOADED}" ] && [ -f "${PROFILE_LOADED}" ]; then
env_content=$(grep -E '^(DCTS_|LOG_|RUST_LOG)' "${PROFILE_LOADED}" 2>/dev/null || echo "")
fi
local var_list=(
DCTS_NODE_ID
DCTS_SERVER_URL
DCTS_MAX_SLOTS
DCTS_PORT
DCTS_ADMIN_TOKEN
DCTS_AUTH_DISABLE
LOG_DIR
DCTS_LOG
DCTS_HEARTBEAT_SEC
)
for var in "${var_list[@]}"; do
if ! echo "${env_content}" | grep -q "^${var}=" 2>/dev/null; then
local val="${!var}"
if [ -n "${val}" ]; then
if [ -n "${env_content}" ]; then
env_content="${env_content}"$'\n'"${var}=${val}"
else
env_content="${var}=${val}"
fi
fi
fi
done
if [ -z "${env_content}" ]; then
env_content="DCTS_SERVER_URL=${DCTS_SERVER_URL:-http://100.66.1.2:8090}"$'\n'"DCTS_MAX_SLOTS=${DCTS_MAX_SLOTS:-4}"$'\n'"LOG_DIR=${LOG_DIR:-/app/data/logs}"
fi
echo "=== 在远端服务器直接生成专属 .env 配置文件 (无需 SCP 传输文件) ==="
# 统一通过 SSH stdin 管道写入 .env 内容。
# Windows 侧原用 PowerShell here-string (@'...'@)，但其闭合标记 '@ 要求独占
# 行首，经 bash→SSH→cmd.exe→PowerShell 多层转义与换行重组后无法被识别，
# 报 "字符串缺少终止符: '@" (TerminatorExpectedAtEndOfString)。
# 改用 stdin 管道 + [Console]::In.ReadToEnd() 读取，命令字符串内不含任何
# 多行内容或 here-string，彻底规避该转义问题。
if [ "$is_win" = "true" ]; then
printf '%s\n' "${env_content}" | ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" \
"powershell -NoProfile -Command \"Set-Content -Path '${REMOTE_DIR}/.env' -Value ([Console]::In.ReadToEnd()) -Encoding utf8\""
else
printf '%s\n' "${env_content}" | ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" \
"cat > '${REMOTE_DIR}/.env' && chmod 600 '${REMOTE_DIR}/.env'"
fi
}
batch_deploy_profiles() {
local profiles=("$@")
# 批量部署面向多台远程主机，剔除本地 Profile (DEPLOY_ENV=local)，避免无意义的
# SSH 预连接与本地构建混入远程批量流程。
local filtered_profiles=()
for p in "${profiles[@]}"; do
local p_env
p_env=$(grep -E '^DEPLOY_ENV=' "$p" 2>/dev/null | cut -d'=' -f2 | tr -d '"' | tr -d "'" || echo "")
if [ "$p_env" = "local" ]; then
echo -e "${YELLOW}[i] 已从批量部署中排除本地 Profile: ${p} (本地部署请单独执行)${NC}"
continue
fi
filtered_profiles+=("$p")
done
profiles=("${filtered_profiles[@]}")
local total=${#profiles[@]}
if [ "$total" -eq 0 ]; then
echo -e "${YELLOW}[i] 批量部署列表中无远程节点，已跳过。${NC}"
return 0
fi
echo -e "\n${GREEN}==============================================================${NC}"
echo -e "${GREEN}🚀 启动高效多节点批量部署引擎: 目标节点总数 [${total}]${NC}"
echo -e "${GREEN}==============================================================${NC}"
local need_server=false
local need_node=false
local need_all=false
local need_compose=false
local need_systemd=false
local p
for p in "${profiles[@]}"; do
local r_role
r_role=$(grep -E '^DEPLOY_ROLE=' "$p" 2>/dev/null | cut -d'=' -f2 | tr -d '"' | tr -d "'" || echo "all")
local r_backend
r_backend=$(grep -E '^DEPLOY_BACKEND=' "$p" 2>/dev/null | cut -d'=' -f2 | tr -d '"' | tr -d "'" || echo "compose")
case "$r_role" in
server) need_server=true ;;
node) need_node=true ;;
all|*) need_server=true; need_node=true; need_all=true ;;
esac
if [ "$r_backend" = "systemd" ]; then need_systemd=true; else need_compose=true; fi
done
# ─── 【阶段 0】提前为所有目标节点建立 SSH 主连接 ─────────────────────────────
# 在本地构建/打包之前完成所有密码输入，避免"构建→输密码→推送→输密码→推送"的割裂等待。
# 连接经 ControlPersist 长期保持，后续递归子进程通过共享 ControlPath 直接复用，无需重复认证。
echo -e "\n${CYAN}=== 【阶段 0/2】预建立所有目标节点 SSH 连接 (请在此时集中输入各节点密码) ===${NC}"
mkdir -p "${SSH_CONTROL_DIR}"
local pre_idx=0
for p in "${profiles[@]}"; do
pre_idx=$((pre_idx + 1))
local r_user r_ip r_port
r_user=$(grep -E '^REMOTE_USER=' "$p" 2>/dev/null | cut -d'=' -f2 | tr -d '"' | tr -d "'" || echo "${REMOTE_USER}")
r_ip=$(grep -E '^REMOTE_IP=' "$p" 2>/dev/null | cut -d'=' -f2 | tr -d '"' | tr -d "'" || echo "${REMOTE_IP}")
r_port=$(grep -E '^REMOTE_PORT=' "$p" 2>/dev/null | cut -d'=' -f2 | tr -d '"' | tr -d "'" || echo "${REMOTE_PORT}")
[ -z "$r_user" ] && r_user="${REMOTE_USER}"
[ -z "$r_ip" ] && r_ip="${REMOTE_IP}"
[ -z "$r_port" ] && r_port="${REMOTE_PORT}"
local sock="${SSH_CONTROL_DIR}/${r_user}@${r_ip}:${r_port}"
if ssh -p "${r_port}" -o ControlPath="${sock}" -O check "${r_user}@${r_ip}" >/dev/null 2>&1; then
echo " -> [${pre_idx}/${total}] ${r_user}@${r_ip}:${r_port} (连接已存在，复用)"
continue
fi
echo " -> [${pre_idx}/${total}] 正在连接 ${r_user}@${r_ip}:${r_port} ..."
ssh -p "${r_port}" -o ControlMaster=yes -o ControlPath="${sock}" -o ControlPersist=1800 -fN "${r_user}@${r_ip}" \
&& echo -e " ${GREEN}[√] 连接已建立${NC}" \
|| echo -e " ${YELLOW}[!] 连接失败，将在部署该节点时重试${NC}"
done
local build_services=""
if [ "$need_server" = true ] && [ "$need_node" = true ]; then
build_services="server node"
elif [ "$need_server" = true ]; then
build_services="server"
else
build_services="node"
fi
echo -e "${CYAN}=== 【阶段 1/2】本地全局单次构建与程序/镜像打包 [服务: ${build_services}] ===${NC}"
if [ "$need_compose" = true ]; then
echo " -> [Docker Compose] 统一构建镜像 (仅构建 1 次)..."
docker compose build ${build_services}
fi
if [ "$need_systemd" = true ]; then
echo " -> [Systemd] 统一编译 release 二进制程序 (仅编译 1 次)..."
cargo build --release --workspace
fi
# 按角色分别打包镜像 (server/node 各自独立)，避免给单角色节点传输不需要的镜像。
local server_tar=""
local node_tar=""
if [ "$need_compose" = true ]; then
mkdir -p data
if [ "$need_server" = true ]; then
server_tar="data/.batch_dcts-server.tar.gz"
echo " -> [Docker Compose] 导出 server 镜像归档包: ${server_tar} (仅打包 1 次)..."
docker save dcts-server:latest | gzip > "${server_tar}"
fi
if [ "$need_node" = true ]; then
node_tar="data/.batch_dcts-node.tar.gz"
echo " -> [Docker Compose] 导出 node 镜像归档包: ${node_tar} (仅打包 1 次)..."
docker save dcts-node:latest | gzip > "${node_tar}"
fi
fi
echo -e "\n${CYAN}=== 【阶段 2/2】同步镜像并为各个节点定制生成专属 .env ===${NC}"
local idx=1
local all_ok=true
for p in "${profiles[@]}"; do
echo -e "\n${MAGENTA}--------------------------------------------------------------${NC}"
echo -e "${MAGENTA}▶▶▶ [节点 ${idx}/${total}] 正在部署 Profile: ${p}${NC}"
echo -e "${MAGENTA}--------------------------------------------------------------${NC}"
# 依据该 Profile 的角色选择对应的镜像归档包 (server/node 分开)，all 角色传两个。
local p_role
p_role=$(grep -E '^DEPLOY_ROLE=' "$p" 2>/dev/null | cut -d'=' -f2 | tr -d '"' | tr -d "'" || echo "all")
local selected_tar=""
case "$p_role" in
server) selected_tar="${server_tar}" ;;
node) selected_tar="${node_tar}" ;;
all|*) selected_tar="${server_tar} ${node_tar}" ;;
esac
if ! PREBUILT_TAR="${selected_tar}" bash "$0" -f "${p}"; then
echo -e "${RED}[✗] 节点 Profile '${p}' 部署失败！${NC}"
all_ok=false
else
echo -e "${GREEN}[√] 节点 Profile '${p}' 部署成功！${NC}"
fi
((idx++))
done
# 清理本阶段打包的临时镜像归档包
[ -n "${server_tar}" ] && rm -f "${server_tar}" 2>/dev/null || true
[ -n "${node_tar}" ] && rm -f "${node_tar}" 2>/dev/null || true
# 关闭阶段 0 预建立的所有 SSH 主连接
for p in "${profiles[@]}"; do
local cu cip cp2 csock
cu=$(grep -E '^REMOTE_USER=' "$p" 2>/dev/null | cut -d'=' -f2 | tr -d '"' | tr -d "'" || echo "${REMOTE_USER}")
cip=$(grep -E '^REMOTE_IP=' "$p" 2>/dev/null | cut -d'=' -f2 | tr -d '"' | tr -d "'" || echo "${REMOTE_IP}")
cp2=$(grep -E '^REMOTE_PORT=' "$p" 2>/dev/null | cut -d'=' -f2 | tr -d '"' | tr -d "'" || echo "${REMOTE_PORT}")
[ -z "$cu" ] && cu="${REMOTE_USER}"; [ -z "$cip" ] && cip="${REMOTE_IP}"; [ -z "$cp2" ] && cp2="${REMOTE_PORT}"
csock="${SSH_CONTROL_DIR}/${cu}@${cip}:${cp2}"
ssh -p "${cp2}" -o ControlPath="${csock}" "${cu}@${cip}" -O exit 2>/dev/null || true
done
echo -e "\n${BLUE}==============================================================${NC}"
if [ "$all_ok" = true ]; then
echo -e "${GREEN}🎉 批量部署完成！所有 ${total} 个节点均已一次性更新并启动服务！${NC}"
else
echo -e "${YELLOW}⚠️ 批量部署结束，部分节点出现错误，请检查日志。${NC}"
exit 1
fi
exit 0
}
show_help() {
echo -e "${BLUE}DCTS 部署与管理控制台${NC}"
@@ -50,6 +296,8 @@ show_help() {
echo " remove | clean 停止并卸载指定服务"
echo ""
echo "选项 (Options):"
echo " -f, --profile <file|dir|list> 指定节点 Profile (例: deploy.env.d/node1.env, 或 node1,node2, 或 deploy.env.d/)"
echo " --all-profiles 批量部署 deploy.env.d/ 下的所有生效 Profile"
echo " -e, --env <local|remote> 部署环境: local (本地) 或 remote (远程)"
echo " -b, --backend <compose|systemd> 底层平台: compose (Docker Compose) 或 systemd (Systemd 服务)"
echo " -r, --role <all|server|node> 目标模块: all (全量), server (服务端), node (计算节点)"
@@ -57,7 +305,7 @@ show_help() {
echo " --force-assets 强制重新全量同步 assets/ 静态资源"
echo " -h, --help 显示帮助信息"
echo ""
echo "示例: $0 -e remote -b compose -r all --skip-assets"
echo "示例: $0 -f deploy.env.d/node-win-01.env -e remote -b compose -r node"
exit 0
}
@@ -74,24 +322,37 @@ while [[ $# -gt 0 ]]; do
fi
shift
;;
-f|--profile|--env-file)
PROFILE_ARG="$2"
shift 2
;;
--all-profiles)
PROFILE_ARG="all"
shift
;;
-e|--env)
ENV_MODE="${2,,}"
CLI_ENV_SET=1
shift 2
;;
-b|--backend)
BACKEND_MODE="${2,,}"
CLI_BACKEND_SET=1
shift 2
;;
-r|--role)
ROLE_TARGET="${2,,}"
CLI_ROLE_SET=1
shift 2
;;
--skip-assets)
SYNC_ASSETS="skip"
CLI_ASSETS_SET=1
shift
;;
--force-assets)
SYNC_ASSETS="force"
CLI_ASSETS_SET=1
shift
;;
-h|--help)
@@ -100,10 +361,13 @@ while [[ $# -gt 0 ]]; do
*)
if [ "$arg_lower" = "server" ] || [ "$arg_lower" = "node" ] || [ "$arg_lower" = "all" ]; then
ROLE_TARGET="$arg_lower"
CLI_ROLE_SET=1
elif [ "$arg_lower" = "local" ] || [ "$arg_lower" = "remote" ]; then
ENV_MODE="$arg_lower"
CLI_ENV_SET=1
elif [ "$arg_lower" = "compose" ] || [ "$arg_lower" = "systemd" ]; then
BACKEND_MODE="$arg_lower"
CLI_BACKEND_SET=1
else
echo -e "${RED}错误: 未知参数 '$1'。使用 '$0 --help' 查看用法。${NC}"
exit 1
@@ -113,6 +377,37 @@ while [[ $# -gt 0 ]]; do
esac
done
# 命令行指定多 Profile 或目录自动展开解析
if [ -n "${PROFILE_ARG}" ]; then
CLI_PROFILES=()
if [ "${PROFILE_ARG}" = "all" ] || [ -d "${PROFILE_ARG}" ]; then
pdir="${PROFILE_ARG}"
[ "${pdir}" = "all" ] && pdir="deploy.env.d"
for p in "${pdir}"/*.env; do
if [ -f "$p" ] && [[ "$p" != *.example ]]; then
CLI_PROFILES+=("$p")
fi
done
else
IFS=',' read -ra ADDR <<< "${PROFILE_ARG}"
for p in "${ADDR[@]}"; do
if [ -f "$p" ]; then
CLI_PROFILES+=("$p")
elif [ -f "deploy.env.d/${p}" ]; then
CLI_PROFILES+=("deploy.env.d/${p}")
elif [ -f "deploy.env.d/${p}.env" ]; then
CLI_PROFILES+=("deploy.env.d/${p}.env")
fi
done
fi
if [ ${#CLI_PROFILES[@]} -gt 1 ]; then
batch_deploy_profiles "${CLI_PROFILES[@]}"
elif [ ${#CLI_PROFILES[@]} -eq 1 ]; then
load_profile_file "${CLI_PROFILES[0]}"
fi
fi
# =============================================================================
# 2. 交互式向导 (参数未指定时调用)
# =============================================================================
@@ -121,6 +416,63 @@ if [ -z "${ENV_MODE}" ] || [ -z "${BACKEND_MODE}" ] || [ -z "${ROLE_TARGET}" ];
echo -e "${CYAN} 🚀 DCTS 部署与管理控制台 ${NC}"
echo -e "${BLUE}==============================================================${NC}"
# [零步]：检测 deploy.env.d 中的 Profile 预设
if [ -z "${PROFILE_LOADED}" ] && [ -z "${PROFILE_ARG}" ] && [ -d "deploy.env.d" ]; then
valid_profiles=()
for p in deploy.env.d/*.env; do
if [ -f "$p" ] && [[ "$p" != *.example ]]; then
valid_profiles+=("$p")
fi
done
if [ ${#valid_profiles[@]} -gt 0 ]; then
echo -e "\n${YELLOW}【预设 Profile】检测到可用的服务器部署 Profile 配置文件:${NC}"
echo " 0) 手动交互式配置新服务器 (不加载预设 Profile)"
idx=1
for p in "${valid_profiles[@]}"; do
pname=$(basename "$p" .env)
p_ip=$(grep -E '^REMOTE_IP=' "$p" 2>/dev/null | cut -d'=' -f2 | tr -d '"' | tr -d "'" || echo "")
p_role=$(grep -E '^DEPLOY_ROLE=' "$p" 2>/dev/null | cut -d'=' -f2 | tr -d '"' | tr -d "'" || echo "")
p_node=$(grep -E '^DCTS_NODE_ID=' "$p" 2>/dev/null | cut -d'=' -f2 | tr -d '"' | tr -d "'" || echo "")
echo " ${idx}) ${pname} (IP: ${p_ip:-未指定}, 角色: ${p_role:-全量}, NodeID: ${p_node:-默认})"
((idx++))
done
if [ ${#valid_profiles[@]} -gt 1 ]; then
echo " a) [批量部署] 依次一键部署上述全部 ${#valid_profiles[@]} 个 Profile 节点"
fi
read -p "请选择预设 Profile [序号如 1, 多个如 1,2 或 1-2, 全部如 a, 默认 0]: " PROFILE_CHOICE
PROFILE_CHOICE=${PROFILE_CHOICE:-0}
SELECTED_PROFILES=()
if [[ "${PROFILE_CHOICE,,}" == "a" ]] || [[ "${PROFILE_CHOICE,,}" == "all" ]]; then
SELECTED_PROFILES=("${valid_profiles[@]}")
elif [[ "${PROFILE_CHOICE}" =~ [0-9] ]]; then
IFS=',' read -ra ADDR <<< "${PROFILE_CHOICE}"
for part in "${ADDR[@]}"; do
if [[ "$part" =~ ^([0-9]+)-([0-9]+)$ ]]; then
s="${BASH_REMATCH[1]}"
e="${BASH_REMATCH[2]}"
for ((i=s; i<=e; i++)); do
if [ "$i" -ge 1 ] && [ "$i" -le "${#valid_profiles[@]}" ]; then
SELECTED_PROFILES+=("${valid_profiles[$((i-1))]}")
fi
done
elif [[ "$part" =~ ^[0-9]+$ ]]; then
if [ "$part" -ge 1 ] && [ "$part" -le "${#valid_profiles[@]}" ]; then
SELECTED_PROFILES+=("${valid_profiles[$((part-1))]}")
fi
fi
done
fi
if [ ${#SELECTED_PROFILES[@]} -gt 1 ]; then
batch_deploy_profiles "${SELECTED_PROFILES[@]}"
elif [ ${#SELECTED_PROFILES[@]} -eq 1 ]; then
load_profile_file "${SELECTED_PROFILES[0]}"
fi
fi
fi
# [第一步]：选择环境
if [ -z "${ENV_MODE}" ]; then
echo -e "\n${YELLOW}【第一步】请选择部署环境:${NC}"
@@ -169,19 +521,26 @@ if [ -z "${ENV_MODE}" ] || [ -z "${BACKEND_MODE}" ] || [ -z "${ROLE_TARGET}" ];
esac
fi
# [第四步]：远程部署安装时选择静态资源同步策略
if [ "${ENV_MODE}" = "remote" ] && [ "${ACTION}" != "remove" ]; then
# [第四步]：远程部署安装时选择静态资源同步策略 (已加载 Profile 时自动跳过)
if [ -z "${PROFILE_LOADED}" ] && [ "${ENV_MODE}" = "remote" ] && [ "${ACTION}" != "remove" ]; then
default_asset_choice=1
default_asset_desc="1 (Auto)"
if [ "${ROLE_TARGET}" = "node" ]; then
default_asset_choice=2
default_asset_desc="2 (Skip, 仅节点推荐)"
fi
echo -e "\n${YELLOW}【第四步】请选择静态资源 (assets/) 同步策略:${NC}"
echo " 1) 自动模式 [Auto] (推荐: rsync 增量同步；若退化至 scp，目标端已有资源时自动跳过)"
echo " 2) 跳过同步 [Skip] (等同 --skip-assets，仅更新程序代码，不传输静态资源)"
echo " 3) 强制同步 [Force] (等同 --force-assets，无视目标端状态，强制全量覆盖传输)"
read -p "请选择 [1-3, 默认 1]: " ASSET_CHOICE
ASSET_CHOICE=${ASSET_CHOICE:-1}
read -p "请选择 [1-3, 默认 ${default_asset_desc}]: " ASSET_CHOICE
ASSET_CHOICE=${ASSET_CHOICE:-$default_asset_choice}
case "$ASSET_CHOICE" in
1) SYNC_ASSETS="auto" ;;
2) SYNC_ASSETS="skip" ;;
3) SYNC_ASSETS="force" ;;
*) echo -e "${RED}输入无效，默认选择 auto。${NC}"; SYNC_ASSETS="auto" ;;
*) SYNC_ASSETS=$([ "$default_asset_choice" = "2" ] && echo "skip" || echo "auto") ;;
esac
fi
@@ -220,22 +579,55 @@ else
TAR_NAME="dcts_images_node.tar.gz"
fi
# 纯 CLI 未指定资源同步策略时，默认 auto（增量同步）
SYNC_ASSETS="${SYNC_ASSETS:-auto}"
echo -e "${CYAN}📢 配置确认: 环境=[${YELLOW}${ENV_MODE}${CYAN}] 平台=[${YELLOW}${BACKEND_MODE}${CYAN}] 模块=[${YELLOW}${ROLE_DESC}${CYAN}] 动作=[${YELLOW}${ACTION}${CYAN}] 资源同步=[${YELLOW}${SYNC_ASSETS}${CYAN}]${NC}\n"
# =============================================================================
# 静态资源同步函数
# =============================================================================
check_remote_is_windows() {
local kernel
kernel=$(ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "uname -s 2>/dev/null || cmd /c ver 2>/dev/null" 2>/dev/null || echo "Windows")
if [[ "${kernel}" == *"Linux"* ]] || [[ "${kernel}" == *"GNU"* ]]; then
echo "false"
else
echo "true"
fi
}
sync_remote_assets() {
if [ "${SYNC_ASSETS}" = "skip" ]; then
echo -e "${YELLOW}[i] 已启用 --skip-assets，跳过静态资源 (assets/) 同步。${NC}"
return 0
fi
if [ "${ROLE_TARGET}" = "node" ] && [ "${SYNC_ASSETS}" != "force" ]; then
echo -e "${YELLOW}[i] 目标模块为仅节点 (node)，默认跳过静态资源 (assets/) 同步。${NC}"
return 0
fi
if [ ! -d "assets" ]; then
return 0
fi
echo "=== 检查并同步静态资源 (assets/) ==="
local is_win
is_win=$(check_remote_is_windows)
if [ "$is_win" = "true" ]; then
local remote_has_assets
remote_has_assets=$(ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "powershell -Command \"Test-Path '${REMOTE_DIR}/assets'\" 2>NUL || cmd /c \"if exist \\\"${REMOTE_DIR}\\assets\\\" echo True\" 2>NUL" 2>/dev/null | grep -i "True" || echo "")
if [ -n "${remote_has_assets}" ] && [ "${SYNC_ASSETS}" != "force" ]; then
echo -e "${GREEN}[√] 远端 Windows 已有 assets/ 目录。为避免 scp 全量传输数据，本次自动跳过 assets/ 同步。${NC}"
else
echo -e "${CYAN} -> 开始向远端 Windows 传输 assets/ 静态资源数据...${NC}"
scp -P "${REMOTE_PORT}" ${SSH_OPTS} -r assets "${REMOTE_USER}@${REMOTE_IP}:${REMOTE_DIR}/"
fi
return 0
fi
local local_rsync=false
local remote_rsync=false
if command -v rsync >/dev/null 2>&1; then local_rsync=true; fi
@@ -263,6 +655,36 @@ sync_remote_assets() {
fi
}
sync_remote_workflows() {
if [ -d "workflows" ]; then
echo -e "${CYAN} -> 同步工作流配置文件 workflows/ 到远端...${NC}"
scp -P "${REMOTE_PORT}" ${SSH_OPTS} -r workflows "${REMOTE_USER}@${REMOTE_IP}:${REMOTE_DIR}/"
fi
}
setup_ssh_control() {
# ControlPath 以 user@ip:port 为 key (不含 PID)，使同一目标主机的父子/兄弟进程
# 共享同一条复用 socket；批量部署递归调用子进程时无需重复输入密码。
local socket_key="${REMOTE_USER}@${REMOTE_IP}:${REMOTE_PORT}"
SSH_CONTROL_PATH="${SSH_CONTROL_DIR}/${socket_key}"
SSH_OPTS="-o ControlMaster=auto -o ControlPath=${SSH_CONTROL_PATH} -o ControlPersist=1800"
mkdir -p "${SSH_CONTROL_DIR}"
# 若已有活跃的主连接 (如批量部署父进程已建立)，直接复用，避免重复认证。
# 此时不注册 cleanup (连接由建立者管理)，避免子进程退出时关闭共享连接影响后续节点。
if ssh -p "${REMOTE_PORT}" -o ControlPath="${SSH_CONTROL_PATH}" -O check "${REMOTE_USER}@${REMOTE_IP}" >/dev/null 2>&1; then
SSH_REUSED_CONNECTION=true
return 0
fi
echo "=== 1. 建立 SSH 主连接复用 (SSH ControlMaster) ==="
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} -fN "${REMOTE_USER}@${REMOTE_IP}" || true
# 仅当本进程建立了主连接时，才在退出时清理；复用的连接交给建立者管理。
SSH_REUSED_CONNECTION=false
cleanup_cm() { [ "${SSH_REUSED_CONNECTION}" = "true" ] && return 0; ssh -p "${REMOTE_PORT}" -o ControlPath="${SSH_CONTROL_PATH}" "${REMOTE_USER}@${REMOTE_IP}" -O exit 2>/dev/null || true; }
trap cleanup_cm EXIT
}
# =============================================================================
# 3. 清理/卸载流程 (ACTION="remove")
# =============================================================================
@@ -295,16 +717,32 @@ if [ "${ACTION}" = "remove" ]; then
elif [ "${ENV_MODE}" = "remote" ] && [ "${BACKEND_MODE}" = "compose" ]; then
echo -e "${YELLOW}清理远端 Docker Compose 服务 [${REMOTE_USER}@${REMOTE_IP}]...${NC}"
setup_ssh_control
is_win=$(check_remote_is_windows)
if [ "${ROLE_TARGET}" = "all" ]; then
ssh -p "${REMOTE_PORT}" "${REMOTE_USER}@${REMOTE_IP}" "cd '${REMOTE_DIR}' 2>/dev/null && docker compose down 2>/dev/null || true"
if [ "$is_win" = "true" ]; then
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "powershell -Command \"Set-Location '${REMOTE_DIR}'; docker compose down\"" 2>/dev/null || true
else
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "cd '${REMOTE_DIR}' && docker compose down" 2>/dev/null || true
fi
else
ssh -p "${REMOTE_PORT}" "${REMOTE_USER}@${REMOTE_IP}" "cd '${REMOTE_DIR}' 2>/dev/null && docker compose stop ${SERVICES} 2>/dev/null && docker compose rm -f -s -v ${SERVICES} 2>/dev/null || true"
if [ "$is_win" = "true" ]; then
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "powershell -Command \"Set-Location '${REMOTE_DIR}'; docker compose stop ${SERVICES}; docker compose rm -f -s -v ${SERVICES}\"" 2>/dev/null || true
else
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "cd '${REMOTE_DIR}' && docker compose stop ${SERVICES} && docker compose rm -f -s -v ${SERVICES}" 2>/dev/null || true
fi
fi
echo -e "${GREEN}[✓] 远端 Docker Compose 服务已清理。${NC}"
elif [ "${ENV_MODE}" = "remote" ] && [ "${BACKEND_MODE}" = "systemd" ]; then
echo -e "${YELLOW}清理远端 Systemd 服务 [${REMOTE_USER}@${REMOTE_IP}]...${NC}"
ssh -t -p "${REMOTE_PORT}" "${REMOTE_USER}@${REMOTE_IP}" "cd '${REMOTE_DIR}' 2>/dev/null && sudo ./scripts/deploy.sh -e local -b systemd -r ${ROLE_TARGET} remove"
setup_ssh_control
is_win=$(check_remote_is_windows)
if [ "$is_win" = "true" ]; then
echo -e "${RED}错误: Systemd 不支持 Windows 远端目标，无法清理。${NC}"
exit 1
fi
ssh -t -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "cd '${REMOTE_DIR}' && sudo ./scripts/deploy.sh -e local -b systemd -r ${ROLE_TARGET} remove"
echo -e "${GREEN}[✓] 远端 Systemd 服务已清理。${NC}"
fi
exit 0
@@ -316,12 +754,19 @@ fi
# ─── 4.1 本地 + Docker Compose ────────────────────────────────────────────────
if [ "${ENV_MODE}" = "local" ] && [ "${BACKEND_MODE}" = "compose" ]; then
echo -e "${CYAN}>>> 预先创建挂载目录并配置读取与写入权限 (UID 65532)...${NC}"
mkdir -p ./data ./data/logs ./data/runtime ./data/work ./data/seeds ./data/result ./assets ./workflows
chmod -R 777 ./data 2>/dev/null || true
if command -v docker >/dev/null 2>&1; then
docker run --rm -v "$(pwd)":/work alpine chown -R 65532:65532 /work/data 2>/dev/null || true
fi
echo -e "${CYAN}>>> 启动本地 Docker Compose 服务构建与运行...${NC}"
docker compose build ${SERVICES}
if [ "${ROLE_TARGET}" = "all" ]; then
docker compose up -d
docker compose up -d --force-recreate
else
docker compose up -d ${SERVICES}
docker compose up -d --force-recreate ${SERVICES}
fi
echo -e " -> 等待 3 秒进行运行状态检测..."
sleep 3 && docker compose ps
@@ -333,42 +778,92 @@ if [ "${ENV_MODE}" = "local" ] && [ "${BACKEND_MODE}" = "compose" ]; then
# ─── 4.2 远程 + Docker Compose ────────────────────────────────────────────────
elif [ "${ENV_MODE}" = "remote" ] && [ "${BACKEND_MODE}" = "compose" ]; then
SSH_CONTROL_DIR="$HOME/.ssh/cm"
SSH_CONTROL_PATH="${SSH_CONTROL_DIR}/${REMOTE_USER}@${REMOTE_IP}:${REMOTE_PORT}-$$"
SSH_OPTS="-o ControlMaster=auto -o ControlPath=${SSH_CONTROL_PATH} -o ControlPersist=1800"
mkdir -p "${SSH_CONTROL_DIR}"
cleanup_cm() { ssh -p "${REMOTE_PORT}" -o ControlPath="${SSH_CONTROL_PATH}" "${REMOTE_USER}@${REMOTE_IP}" -O exit 2>/dev/null || true; rm -f "${SSH_CONTROL_PATH}" 2>/dev/null || true; }
trap cleanup_cm EXIT
setup_ssh_control
echo "=== 1. 建立 SSH 主连接复用 (SSH ControlMaster) ==="
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} -fN "${REMOTE_USER}@${REMOTE_IP}" || true
if [ -n "${PREBUILT_TAR}" ]; then
# 批量模式: PREBUILT_TAR 为父进程按角色分包的镜像归档 (空格分隔，可能多个)。
# 仅当所有包均存在时才复用；否则回退到本进程自行构建打包。
all_prebuilt_exist=true
PREBUILT_LIST=()
for t in ${PREBUILT_TAR}; do
if [ -f "${t}" ]; then
PREBUILT_LIST+=("${t}")
else
all_prebuilt_exist=false
fi
done
if [ "${all_prebuilt_exist}" = true ] && [ ${#PREBUILT_LIST[@]} -gt 0 ]; then
echo "=== 2 & 3. 复用全局构建阶段一生成的镜像归档包 [${PREBUILT_LIST[*]}] ==="
TAR_LIST=("${PREBUILT_LIST[@]}")
else
echo -e "${YELLOW}[!] 部分预构建镜像包缺失，回退至本进程自行构建打包...${NC}"
PREBUILT_TAR=""
fi
fi
echo "=== 2. 在本地构建 Docker 镜像 [${SERVICES}] ==="
docker compose build ${SERVICES}
if [ -z "${PREBUILT_TAR}" ]; then
echo "=== 2. 在本地构建 Docker 镜像 [${SERVICES}] ==="
docker compose build ${SERVICES}
echo "=== 3. 导出镜像至归档包: ${TAR_NAME} ==="
if [ "$ROLE_TARGET" = "all" ]; then
docker save dcts-server:latest dcts-node:latest | gzip > "${TAR_NAME}"
elif [ "$ROLE_TARGET" = "server" ]; then
docker save dcts-server:latest | gzip > "${TAR_NAME}"
else
docker save dcts-node:latest | gzip > "${TAR_NAME}"
echo "=== 3. 导出镜像至归档包: ${TAR_NAME} ==="
if [ "$ROLE_TARGET" = "all" ]; then
docker save dcts-server:latest dcts-node:latest | gzip > "${TAR_NAME}"
elif [ "$ROLE_TARGET" = "server" ]; then
docker save dcts-server:latest | gzip > "${TAR_NAME}"
else
docker save dcts-node:latest | gzip > "${TAR_NAME}"
fi
TAR_LIST=("${TAR_NAME}")
fi
echo "=== 4. 同步静态资源与配置文件 ==="
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "mkdir -p '${REMOTE_DIR}'"
is_win=$(check_remote_is_windows)
if [ "$is_win" = "true" ]; then
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "powershell -Command \"New-Item -ItemType Directory -Force -Path '${REMOTE_DIR}'\" 2>NUL || cmd /c \"if not exist \\\"${REMOTE_DIR}\\\" mkdir \\\"${REMOTE_DIR}\\\"\" 2>NUL" >/dev/null 2>&1 || true
else
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "mkdir -p '${REMOTE_DIR}'"
fi
sync_remote_assets
sync_remote_workflows
ENV_FILE=".env"
[ ! -f ".env" ] && ENV_FILE=".env.example"
scp -P "${REMOTE_PORT}" ${SSH_OPTS} "${TAR_NAME}" docker-compose.yml "${ENV_FILE}" "${REMOTE_USER}@${REMOTE_IP}:${REMOTE_DIR}/"
[ "${ENV_FILE}" = ".env.example" ] && ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "cp -n '${REMOTE_DIR}/.env.example' '${REMOTE_DIR}/.env' 2>/dev/null || true"
write_remote_runtime_env "$is_win"
echo "=== 5. 远端导入镜像并启动 Docker Compose 服务 ==="
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "cd '${REMOTE_DIR}' && docker load < '${TAR_NAME}' && rm -f '${TAR_NAME}' && docker compose up -d ${SERVICES} && sleep 3 && docker compose ps"
# 传输所有镜像归档包 (scp 落地名为 basename，丢掉本地 data/ 前缀)。
# 同时收集远端引用名 (纯文件名，与 docker load 的路径对齐)。
REMOTE_TAR_NAMES=()
for t in "${TAR_LIST[@]}"; do
scp -P "${REMOTE_PORT}" ${SSH_OPTS} "${t}" "${REMOTE_USER}@${REMOTE_IP}:${REMOTE_DIR}/"
REMOTE_TAR_NAMES+=("$(basename "${t}")")
done
scp -P "${REMOTE_PORT}" ${SSH_OPTS} docker-compose.yml "${REMOTE_USER}@${REMOTE_IP}:${REMOTE_DIR}/"
echo "=== 6. 清理本地临时归档包 ==="
rm -f "${TAR_NAME}"
echo "=== 5. 远端预设目录权限、导入镜像并启动 Docker Compose 服务 ==="
# 构造远端批量导入与清理命令 (逐个包 docker load 后删除)
remote_load_cmd=""
for rn in "${REMOTE_TAR_NAMES[@]}"; do
remote_load_cmd="${remote_load_cmd}docker load -i '${rn}' && rm -f '${rn}' && "
done
remote_load_cmd="${remote_load_cmd}true"
if [ "$is_win" = "true" ]; then
win_load_cmd=""
for rn in "${REMOTE_TAR_NAMES[@]}"; do
win_load_cmd="${win_load_cmd}docker load -i '${rn}'; Remove-Item -Force '${rn}' -ErrorAction SilentlyContinue; "
done
win_load_cmd="${win_load_cmd}Write-Output done"
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "powershell -Command \"Set-Location '${REMOTE_DIR}'; New-Item -ItemType Directory -Force -Path 'data/logs','data/seeds','data/result','data/node_work','assets','workflows' -ErrorAction SilentlyContinue; ${win_load_cmd}; docker compose up -d --force-recreate ${SERVICES}; Start-Sleep -Seconds 3; docker compose ps\""
else
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "cd '${REMOTE_DIR}' && mkdir -p ./data ./data/logs ./data/seeds ./data/result ./data/node_work ./assets ./workflows 2>/dev/null || true && (chmod -R 777 ./data 2>/dev/null || true; docker run --rm -v \"\$(pwd)\":/work alpine chown -R 65532:65532 /work/data 2>/dev/null || true) && ${remote_load_cmd} && docker compose up -d --force-recreate ${SERVICES} && sleep 3 && docker compose ps"
fi
if [ -z "${PREBUILT_TAR}" ]; then
echo "=== 6. 清理本地临时归档包 ==="
for t in "${TAR_LIST[@]}"; do
rm -f "${t}"
done
fi
echo -e "${GREEN}🎉 远程 Docker Compose [${ROLE_DESC}] 部署完成！${NC}"
if [[ "${ROLES}" == *"server"* ]]; then
echo -e "👉 远程面板访问地址: ${CYAN}http://${REMOTE_IP}:8090${NC}"
@@ -382,7 +877,7 @@ elif [ "${ENV_MODE}" = "local" ] && [ "${BACKEND_MODE}" = "systemd" ]; then
fi
TARGET_USER="${DCTS_DAEMON_USER:-${SUDO_USER:-$USER}}"
[ "$TARGET_USER" = "root" ] && [ $(id -u fmq 2>/dev/null || echo 0) -ne 0 ] && TARGET_USER="fmq"
[ "$TARGET_USER" = "root" ] && [ "$(id -u fmq 2>/dev/null || echo 0)" -ne 0 ] && TARGET_USER="fmq"
REAL_USER="${TARGET_USER}"
if [ ! -f "${WORK_DIR}/.env" ]; then
@@ -457,15 +952,13 @@ EOF
# ─── 4.4 远程 + Systemd ───────────────────────────────────────────────────────
elif [ "${ENV_MODE}" = "remote" ] && [ "${BACKEND_MODE}" = "systemd" ]; then
SSH_CONTROL_DIR="$HOME/.ssh/cm"
SSH_CONTROL_PATH="${SSH_CONTROL_DIR}/${REMOTE_USER}@${REMOTE_IP}:${REMOTE_PORT}-$$"
SSH_OPTS="-o ControlMaster=auto -o ControlPath=${SSH_CONTROL_PATH} -o ControlPersist=1800"
mkdir -p "${SSH_CONTROL_DIR}"
cleanup_cm() { ssh -p "${REMOTE_PORT}" -o ControlPath="${SSH_CONTROL_PATH}" "${REMOTE_USER}@${REMOTE_IP}" -O exit 2>/dev/null || true; rm -f "${SSH_CONTROL_PATH}" 2>/dev/null || true; }
trap cleanup_cm EXIT
setup_ssh_control
echo "=== 1. 建立 SSH 主连接复用 (SSH ControlMaster) ==="
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} -fN "${REMOTE_USER}@${REMOTE_IP}" || true
is_win=$(check_remote_is_windows)
if [ "$is_win" = "true" ]; then
echo -e "${RED}错误: Systemd 服务不支持 Windows 远端目标，请改用 Docker Compose 部署 (-b compose)。${NC}"
exit 1
fi
echo "=== 2. 检查架构与准备二进制文件 [${ROLES}] ==="
REMOTE_ARCH=$(ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "uname -m 2>/dev/null || echo unknown")
@@ -484,6 +977,7 @@ elif [ "${ENV_MODE}" = "remote" ] && [ "${BACKEND_MODE}" = "systemd" ]; then
echo "=== 3. 传输文件与静态资源 ==="
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "mkdir -p '${REMOTE_DIR}/scripts' '${REMOTE_DIR}/data' '${REMOTE_DIR}/logs'"
sync_remote_assets
sync_remote_workflows
for r in ${ROLES}; do
BIN_SRC="target/release/${r}"
@@ -493,10 +987,8 @@ elif [ "${ENV_MODE}" = "remote" ] && [ "${BACKEND_MODE}" = "systemd" ]; then
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "mv -f '${REMOTE_DIR}/${r}.tmp' '${REMOTE_DIR}/${r}' && chmod +x '${REMOTE_DIR}/${r}'"
done
ENV_FILE=".env"
[ ! -f ".env" ] && ENV_FILE=".env.example"
scp -P "${REMOTE_PORT}" ${SSH_OPTS} "${ENV_FILE}" scripts/deploy.sh "${REMOTE_USER}@${REMOTE_IP}:${REMOTE_DIR}/scripts/"
[ "${ENV_FILE}" = ".env.example" ] && ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "cp -n '${REMOTE_DIR}/scripts/.env.example' '${REMOTE_DIR}/.env' 2>/dev/null || true" || ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "cp -n '${REMOTE_DIR}/scripts/.env' '${REMOTE_DIR}/.env' 2>/dev/null || true"
write_remote_runtime_env "$is_win"
scp -P "${REMOTE_PORT}" ${SSH_OPTS} scripts/deploy.sh "${REMOTE_USER}@${REMOTE_IP}:${REMOTE_DIR}/scripts/"
echo "=== 4. 在远端配置并启动 Systemd 服务 ==="
ssh -t -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" "cd '${REMOTE_DIR}' && chmod +x scripts/deploy.sh && sudo ./scripts/deploy.sh -e local -b systemd -r ${ROLE_TARGET} install"
scripts/migrate_data_dirs.sh
Executable
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env bash
# 一次性数据目录迁移：results→seeds、archive→result（目录重命名后的历史数据搬迁）。
#
# 背景：data 目录重新规划后，两个子目录改名：
# data/results/ → data/seeds/ （server 种子库；原名 results 误导，实际只存种子）
# data/archive/ → data/result/ （node 完整计算结果归档；原名 archive 不够准确）
# 本脚本把已落盘的旧目录搬到新位置，并修正 DB 的 seeds.file_path 审计字段。
#
# 关键事实（已确认）：
# - download_seed 端点不读 seeds.file_path（从 seeds_dir 重拼路径），故 DB 迁移失败
# 不影响功能；但保持审计一致仍建议执行。
# - 迁移需在 server 停机时进行（mv 整目录 + 写 DB）。
#
# 用法：
# ./scripts/migrate_data_dirs.sh # dry-run，只看将做什么
# ./scripts/migrate_data_dirs.sh --apply # 实际执行
#
# 环境变量：
# DCTS_DATA_DIR 数据根目录（默认 data）
# DCTS_DB_PATH 主库路径（默认 data/dcts.db，相对 DCTS_DATA_DIR）
set -euo pipefail
DATA_DIR="${DCTS_DATA_DIR:-data}"
DB_PATH="${DCTS_DB_PATH:-${DCTS_DATA_DIR:-data}/dcts.db}"
APPLY=0
if [[ "${1:-}" == "--apply" ]]; then
APPLY=1
fi
run_or_echo() {
# dry-run 时打印命令，--apply 时执行
if [[ $APPLY -eq 1 ]]; then
eval "$1"
else
echo " [dry-run] $1"
fi
}
echo "数据根目录: $DATA_DIR"
echo "主库路径: $DB_PATH"
echo "模式: $([[ $APPLY -eq 1 ]] && echo ' APPLY（实际执行）' || echo ' dry-run（只预览）')"
echo "------------------------------------------------"
moved=0
# 1. results → seeds
OLD_RESULTS="${DATA_DIR}/results"
NEW_SEEDS="${DATA_DIR}/seeds"
if [[ -d "$OLD_RESULTS" ]]; then
echo "[1/2] 发现旧目录 $OLD_RESULTS"
if [[ -d "$NEW_SEEDS" ]]; then
echo " ⚠️ 目标 $NEW_SEEDS 已存在！不会自动合并，请手动处理后重跑。"
else
run_or_echo "mv '$OLD_RESULTS' '$NEW_SEEDS'"
moved=$((moved + 1))
echo " → results 重命名为 seeds"
fi
else
echo "[1/2] 旧目录 $OLD_RESULTS 不存在，跳过"
fi
# 2. archive → result
OLD_ARCHIVE="${DATA_DIR}/archive"
NEW_RESULT="${DATA_DIR}/result"
if [[ -d "$OLD_ARCHIVE" ]]; then
echo "[2/2] 发现旧目录 $OLD_ARCHIVE"
if [[ -d "$NEW_RESULT" ]]; then
echo " ⚠️ 目标 $NEW_RESULT 已存在！不会自动合并，请手动处理后重跑。"
else
run_or_echo "mv '$OLD_ARCHIVE' '$NEW_RESULT'"
moved=$((moved + 1))
echo " → archive 重命名为 result"
fi
else
echo "[2/2] 旧目录 $OLD_ARCHIVE 不存在，跳过"
fi
# 3. DB seeds.file_path 审计字段修正（非功能性，但保持一致）
echo "------------------------------------------------"
if [[ -f "$DB_PATH" ]] && command -v sqlite3 >/dev/null 2>&1; then
# 统计待更新的行数（results→seeds）
STMT_COUNT="SELECT COUNT(*) FROM seeds WHERE file_path LIKE '%/results/%';"
if [[ $APPLY -eq 1 ]]; then
cnt=$(sqlite3 "$DB_PATH" "$STMT_COUNT" 2>/dev/null || echo "0")
else
# dry-run：尝试只读，读不到就标"无法预览"
cnt=$(sqlite3 "file:${DB_PATH}?mode=ro" "$STMT_COUNT" 2>/dev/null || echo "?")
fi
echo "[3/3] DB seeds.file_path 含旧路径的行数: $cnt"
if [[ "$cnt" != "0" && "$cnt" != "?" ]]; then
# 把路径中的 /results/ 替换为 /seeds/
run_or_echo "sqlite3 \"$DB_PATH\" \"UPDATE seeds SET file_path = REPLACE(file_path, '/results/', '/seeds/') WHERE file_path LIKE '%/results/%';\""
echo " → 已修正 seeds.file_path 审计路径"
fi
else
echo "[3/3] 跳过 DB 迁移：$DB_PATH 不存在或无 sqlite3 命令"
echo " （download_seed 不读此列，不影响功能；可后续手动执行 UPDATE）"
fi
echo "------------------------------------------------"
if [[ $APPLY -eq 1 ]]; then
echo "✅ 完成：迁移了 $moved 个目录"
else
echo "✅ dry-run 完成。加 --apply 实际执行。"
echo " ⚠️ 执行前请先停掉 server（mv 整目录 + 写 DB 需停机）。"
fi
scripts/push_import_results.sh
Executable
+273
View File
@@ -0,0 +1,273 @@
#!/usr/bin/env bash
# ==============================================================================
# push_import_results.sh — 推送 import_results 工具到 Windows 计算节点并编译
# ==============================================================================
# 把 import_results 工具的源码推送到 deploy.env.d/<profile> 指定的 Windows 节点，
# 在该节点原生编译（Windows 上 cargo build），并打印确切的运行命令。
#
# 设计前提：
# - 旧版 Python run_grid.py 的结果在 Windows 机的 E:/fmq/grid（工具必须在该机运行才能读到）
# - 该机已部署 node（Docker Compose），data 目录在 E:/fmq/dcts/data
# - DCTS 服务端地址来自 profile 的 DCTS_SERVER_URL
# - Windows 机需已装 Rust 工具链（cargo/rustc）；脚本会自检并给出安装提示
#
# 用法:
# ./scripts/push_import_results.sh # 默认用 node-dckj-win-01.env
# ./scripts/push_import_results.sh -f deploy.env.d/other-node.env # 指定其他 profile
# ./scripts/push_import_results.sh --no-build # 只推源码不编译（远端已有缓存时加速）
# ./scripts/push_import_results.sh --archive-dir E:/fmq/dcts/data/result # 覆盖归档目录
#
# 运行（推完编译后，在 Windows 机上执行，或本脚本末尾会打印）:
# E:\fmq\dcts-src\target\release\import_results.exe \
# -d E:/fmq/grid -c E:/fmq/dcts-src/workflows/sdB_cno.yaml \
# -r E:/fmq/dcts/data/result -s http://100.66.1.2:8090 \
# -w sdB_cno -t <admin_token>
# ==============================================================================
set -eo pipefail
GREEN='\033[0;32m'
BLUE='\033[0;34m'
RED='\033[0;31m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m'
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
WORK_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
cd "${WORK_DIR}"
# 默认值（可被 profile 覆盖）
PROFILE_FILE="deploy.env.d/node-dckj-win-01.env"
REMOTE_USER="fmq"
REMOTE_IP="192.168.6.102"
REMOTE_PORT="22"
REMOTE_SRC_DIR="E:/fmq/dcts-src"
DCTS_SERVER_URL="http://100.66.1.2:8090"
RESULT_DIR="" # 空 → 用 profile 推导（默认 E:/fmq/dcts/data/result）
DO_BUILD=true
# SSH ControlMaster 复用目录（与 deploy.sh 一致，便于共享连接）
SSH_CONTROL_DIR="${SSH_CONTROL_DIR:-$HOME/.ssh/cm}"
# =============================================================================
# 解析命令行参数
# =============================================================================
while [[ $# -gt 0 ]]; do
case "$1" in
-f|--profile) PROFILE_FILE="$2"; shift 2 ;;
--no-build) DO_BUILD=false; shift ;;
--archive-dir) RESULT_DIR="$2"; shift 2 ;;
-h|--help)
sed -n '2,30p' "$0"
exit 0 ;;
*) echo -e "${RED}未知参数: $1${NC}"; exit 1 ;;
esac
done
# =============================================================================
# 加载 profile（SSH 连接信息 + 服务端地址）
# =============================================================================
if [ ! -f "$PROFILE_FILE" ]; then
echo -e "${RED}错误: 找不到 profile '${PROFILE_FILE}'${NC}"
exit 1
fi
echo -e "${CYAN}[i] 加载 profile: ${PROFILE_FILE}${NC}"
# shellcheck disable=SC1090
set -a; source "$PROFILE_FILE"; set +a
# profile 提供的变量优先（REMOTE_* / DCTS_SERVER_URL）
[ -n "${REMOTE_USER:-}" ] && REMOTE_USER="${REMOTE_USER}"
[ -n "${REMOTE_IP:-}" ] && REMOTE_IP="${REMOTE_IP}"
[ -n "${REMOTE_PORT:-}" ] && REMOTE_PORT="${REMOTE_PORT}"
# 源码目录默认放到部署目录旁（E:/fmq/dcts-src），与运行中的 node（E:/fmq/dcts）隔离
[ -n "${REMOTE_DIR:-}" ] && REMOTE_SRC_DIR="${REMOTE_DIR}-src"
# 结果目录：命令行 > 参数 > 默认（部署目录下的 data/result，node 的 DCTS_RESULT_DIR 卷映射到此）
if [ -z "${RESULT_DIR}" ]; then
RESULT_DIR="${REMOTE_DIR}/data/result"
fi
echo -e "${CYAN}📢 目标节点: ${YELLOW}${REMOTE_USER}@${REMOTE_IP}:${REMOTE_PORT}${NC}"
echo -e "${CYAN}📢 源码推送: ${YELLOW}${REMOTE_SRC_DIR}${NC}"
echo -e "${CYAN}📢 结果目录: ${YELLOW}${RESULT_DIR}${NC}"
echo -e "${CYAN}📢 服务端: ${YELLOW}${DCTS_SERVER_URL}${NC}"
echo ""
# =============================================================================
# 建立 SSH ControlMaster 连接复用（一次性输密码，后续 scp/ssh 复用）
# =============================================================================
mkdir -p "${SSH_CONTROL_DIR}"
SOCK="${SSH_CONTROL_DIR}/${REMOTE_USER}@${REMOTE_IP}:${REMOTE_PORT}"
SSH_OPTS="-o ControlMaster=auto -o ControlPath=${SOCK} -o ControlPersist=1800 -o StrictHostKeyChecking=no"
if ssh -p "${REMOTE_PORT}" -o ControlPath="${SOCK}" -O check "${REMOTE_USER}@${REMOTE_IP}" >/dev/null 2>&1; then
echo -e "${GREEN}[√] 复用已有 SSH 连接${NC}"
else
echo -e "${CYAN}=== 建立 SSH 连接（请输入 ${REMOTE_USER}@${REMOTE_IP} 的密码）===${NC}"
if ! ssh -p "${REMOTE_PORT}" ${SSH_OPTS} -fN "${REMOTE_USER}@${REMOTE_IP}"; then
echo -e "${RED}SSH 连接失败${NC}"
exit 1
fi
echo -e "${GREEN}[√] SSH 连接已建立${NC}"
fi
trap 'ssh -p "${REMOTE_PORT}" -o ControlPath="${SOCK}" "${REMOTE_USER}@${REMOTE_IP}" -O exit 2>/dev/null || true' EXIT
# =============================================================================
# 自检远端 Rust 工具链
# =============================================================================
echo -e "\n${CYAN}=== 检查远端 Rust 工具链 ===${NC}"
HAS_CARGO=$(ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" \
"powershell -NoProfile -Command \"if (Get-Command cargo -ErrorAction SilentlyContinue) { 'yes' } else { 'no' }\"" 2>/dev/null | tr -d '[:space:]')
if [ "$HAS_CARGO" != "yes" ]; then
echo -e "${RED}[✗] 远端 ${REMOTE_IP} 未检测到 cargo/rustc${NC}"
echo -e "${YELLOW}请先在该 Windows 机上安装 Rust 工具链：${NC}"
echo -e " 方式1（PowerShell，需联网）: winget install Rustlang.Rustup"
echo -e " 方式2（浏览器）: 访问 https://rustup.rs 下载 rustup-init.exe 并运行"
echo -e " 安装后重开 PowerShell 再跑本脚本。"
exit 1
fi
echo -e "${GREEN}[√] 远端已安装 Rust 工具链${NC}"
# =============================================================================
# 同步源码到远端（仅 git 跟踪文件，排除 target/data/assets 等大目录）
# =============================================================================
echo -e "\n${CYAN}=== 同步源码到 ${REMOTE_SRC_DIR} ===${NC}"
# 远端先建目录
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" \
"powershell -NoProfile -Command \"New-Item -ItemType Directory -Force -Path '${REMOTE_SRC_DIR}' | Out-Null\""
# 单文件 scp 辅助：失败时打印诊断并退出（不静默吞错）。
# Windows OpenSSH scp 对带正斜杠的远端路径需用 ':' 分隔主机，路径部分正斜杠 OK。
scp_one() {
local src="$1" dst="$2"
if ! scp -P "${REMOTE_PORT}" ${SSH_OPTS} "$src" "${REMOTE_USER}@${REMOTE_IP}:${dst}"; then
echo -e "${RED}[✗] 传输失败: ${src} -> ${dst}${NC}"
exit 1
fi
}
# 递归目录 scp：先建/清远端目标目录再传，避免 Windows scp -r 创建子目录失败。
scp_dir() {
local sub="$1"
[ -d "$sub" ] || return 0
# 远端先删除旧目录并重建空目录（确保 scp -r 能写入）
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" \
"powershell -NoProfile -Command \"Remove-Item -Recurse -Force '${REMOTE_SRC_DIR}/${sub}' -ErrorAction SilentlyContinue; New-Item -ItemType Directory -Force -Path '${REMOTE_SRC_DIR}/${sub}' | Out-Null\"" || true
echo -e " -> ${sub}/"
if ! scp -P "${REMOTE_PORT}" ${SSH_OPTS} -r "$sub" "${REMOTE_USER}@${REMOTE_IP}:${REMOTE_SRC_DIR}/"; then
echo -e "${RED}[✗] 传输目录失败: ${sub}${NC}"
echo -e "${YELLOW} 若反复失败，可在 Windows 机上手动 git clone 后仅同步增量，或检查磁盘空间/权限${NC}"
exit 1
fi
}
echo -e "${CYAN} -> 传输顶层文件 + crates/tools/workflows ...${NC}"
scp_one Cargo.toml "${REMOTE_SRC_DIR}/"
[ -f Cargo.lock ] && scp_one Cargo.lock "${REMOTE_SRC_DIR}/"
# import_results 依赖 common crate，二者都要同步；workflows 含配置（决定命名精度）。
scp_dir crates
scp_dir tools
scp_dir workflows
echo -e "${GREEN}[√] 源码同步完成${NC}"
# =============================================================================
# 远端编译 import_results（release）
# =============================================================================
if [ "$DO_BUILD" = true ]; then
echo -e "\n${CYAN}=== 远端编译 import_results (release) ===${NC}"
echo -e "${YELLOW} （首次编译会拉 crate 依赖，可能耗时数分钟，请耐心等待...）${NC}"
ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" \
"powershell -NoProfile -Command \"Set-Location '${REMOTE_SRC_DIR}'; cargo build --release -p import_results 2>&1; \$LASTEXITCODE\"" 2>&1 | tee /tmp/import_build.log | tail -20
BUILD_RC=$(tail -1 /tmp/import_build.log | tr -d '[:space:]')
rm -f /tmp/import_build.log
if [ "$BUILD_RC" != "0" ]; then
echo -e "${RED}[✗] 远端编译失败 (exit=${BUILD_RC})${NC}"
exit 1
fi
echo -e "${GREEN}[√] 编译成功${NC}"
else
echo -e "${YELLOW}[i] 跳过编译（--no-build）${NC}"
fi
# =============================================================================
# 在远端生成 run_import.bat（Windows 批处理），用户双击或在 cmd 里运行即可，无需手敲命令
# =============================================================================
EXE_PATH_WIN=$(echo "${REMOTE_SRC_DIR}/target/release/import_results.exe" | sed 's|/|\\|g')
CONFIG_PATH="${REMOTE_SRC_DIR}/workflows/sdB_cno.yaml"
WORK_DIR_WIN=$(echo "${REMOTE_SRC_DIR}" | sed 's|/|\\|g')
# .bat 内容：用 quoted heredoc 避免反引号/% 被 bash 解释，占位符再替换。
# .bat 用 set /p 交互读取 admin token（用户无需手动编辑文件），空回车则跳过入库只迁移 result。
BAT_CONTENT=$(cat <<'BATEOF'
@echo off
chcp 65001 >nul
setlocal
cd /d "__WORK_DIR__"
echo ============================================================
echo DCTS 历史计算结果导入工具
echo ============================================================
echo 旧数据目录 : E:\fmq\grid
echo workflow : __CONFIG__
echo 结果目录 : __RESULT__
echo 服务端 : __SERVER__
echo 工作流名 : sdB_cno
echo ============================================================
echo.
set /p ADMIN_TOKEN="请输入 Admin Token (DCTS_ADMIN_TOKEN，直接回车则只迁移 result 不入库): "
if "%ADMIN_TOKEN%"=="" (
echo [i] 未输入 token，仅迁移产物树到 result (入库会因无鉴权失败但不阻断迁移)
"__EXE__" -d E:/fmq/grid -c "__CONFIG__" -r "__RESULT__" -s __SERVER__ -w sdB_cno
) else (
"__EXE__" -d E:/fmq/grid -c "__CONFIG__" -r "__RESULT__" -s __SERVER__ -w sdB_cno -t "%ADMIN_TOKEN%"
)
echo.
echo ============================================================
echo 执行结束。按任意键退出...
pause >nul
BATEOF
)
BAT_CONTENT="${BAT_CONTENT//__WORK_DIR__/${WORK_DIR_WIN}}"
BAT_CONTENT="${BAT_CONTENT//__EXE__/${EXE_PATH_WIN}}"
BAT_CONTENT="${BAT_CONTENT//__CONFIG__/${CONFIG_PATH}}"
BAT_CONTENT="${BAT_CONTENT//__RESULT__/${RESULT_DIR}}"
BAT_CONTENT="${BAT_CONTENT//__SERVER__/${DCTS_SERVER_URL}}"
# 经 SSH stdin 管道写入远端 run_import.bat（UTF-8 with BOM 由 chcp 65001 处理中文）
BAT_REMOTE_PATH="${REMOTE_SRC_DIR}/run_import.bat"
echo -e "\n${CYAN}=== 在远端生成 run_import.bat ===${NC}"
printf '%s\r\n' "$BAT_CONTENT" | ssh -p "${REMOTE_PORT}" ${SSH_OPTS} "${REMOTE_USER}@${REMOTE_IP}" \
"powershell -NoProfile -Command \"\$input = [Console]::In.ReadToEnd(); [System.IO.File]::WriteAllText('${BAT_REMOTE_PATH}', \$input, [System.Text.Encoding]::UTF8)\"" \
&& echo -e "${GREEN}[√] 已生成: ${BAT_REMOTE_PATH}${NC}" \
|| { echo -e "${RED}[✗] 生成 run_import.bat 失败${NC}"; exit 1; }
# =============================================================================
# 输出最终提示
# =============================================================================
echo ""
echo -e "${BLUE}==============================================================${NC}"
echo -e "${GREEN}🎉 推送 + 编译完成！${NC}"
echo -e "${BLUE}==============================================================${NC}"
echo ""
echo -e "${CYAN}在 Windows 机 ${REMOTE_IP} 上启动导入（任选其一）：${NC}"
echo ""
echo -e " 方式1（推荐）：双击运行 ${YELLOW}${WORK_DIR_WIN}\\run_import.bat${NC}"
echo -e " 会交互提示输入 Admin Token（回车则只迁移 result 不入库）"
echo ""
echo -e " 方式2（cmd）： cd /d ${WORK_DIR_WIN} 然后 run_import.bat"
echo ""
echo -e "${YELLOW}参数（已内置在 .bat 里，无需手填）：${NC}"
echo -e " -d E:/fmq/grid 旧版 Python run_grid.py 的 results 根目录"
echo -e " -c ${CONFIG_PATH}"
echo -e " -r ${RESULT_DIR} 结果写入目标（node 容器卷映射处）"
echo -e " -s ${DCTS_SERVER_URL} 服务端地址（来自 profile）"
echo -e " -w sdB_cno 工作流名（必须等于将来要启动的工作流名）"
echo ""
echo -e "${YELLOW}⚠️ 注意：${NC}"
echo -e " - -w sdB_cno 必须匹配将来在 Dashboard 创建/启动的工作流名，否则 converged 标记被隔离看不到、导致重算"
echo -e " - 工具可重复运行（DB 幂等 + result 原子覆盖），网络中断后重跑 run_import.bat 即可补齐"
echo ""