feat(all): NaN 伪收敛否决与 nl_tight 能量回退、emflux 检验 4π 修正、nst 行宽与种子边界修复、gfATO 谱线表接通与网格加密 9216 点、阶段分项统计与跳板机部署
物理修复(400 失败点归因,见 docs/failed400_nan_pseudo_convergence_2026_08_17.md): - runner: 阶段 converged 后复查 fort.7,含 NaN/Inf 即否决(fort.9 全零伪收敛, 曾致 261 点误跳过 nl_direct 回退);否决阶段不产出种子,阻断污染传播 - runner: nl_tight 回退——仅能量边际失败时以 CHMAX 收紧 10× 从自身模型续迭代, 残差降幅 ~10×;execute_tlusty_stage 抽取供主链与回退共用 - conv_check: fort.14 为 Eddington 通量 Hλ,积分需乘 4π 再比 σTeff⁴ (旧版 ratio 稳定 0.0796=1/4π,全点系统性假阳性)+ 回归测试 - nst_writer: 单行超 80 字符被 TLUSTY 静默截断,IFALI/JALI/TRAD 等从未生效; 按 75 字符自动换行 - seed_finder: Teff 容忍度改含边界 <=,相邻 5000K 档恢复互为种子 + 回归测试 谱线表与网格: - 默认线表 gfVIS99 → gfATO(全波段 18-23000Å),TaskSpec.linelist 支持工作流 级覆盖,节点按需下载(进程互斥锁防并发重复下载 238MB) - sdB_cno Teff 加密至 5000K 步长,432 → 9216 点;tlusty/synspec 静态二进制更新 统计与部署: - grid 汇总改按 tlusty_status/synspec_status 分项计数,新增 tlusty_failed/ synspec_failed/synspec_pending,前端详情页双视图适配 - deploy/fetch_results 支持跳板机 ProxyJump 与 SSH 主连接复用,fetch 新增 --force; - Docker 构建支持 CARGO_MIRROR/USE_MIRRORS 国内镜像参数;移除 tools/ 拷贝 - 新增 tlusty-synspec-test skill 与 6 篇根因分析/验证文档
This commit is contained in:
+8
-2
@@ -216,12 +216,16 @@ batch_deploy_profiles() {
|
||||
[ -z "$r_port" ] && r_port="${REMOTE_PORT}"
|
||||
|
||||
local sock="${SSH_CONTROL_DIR}/${r_user}@${r_ip}:${r_port}"
|
||||
if ssh -p "${r_port}" -o ControlPath="${sock}" -O check "${r_user}@${r_ip}" >/dev/null 2>&1; then
|
||||
# 跳板机支持:profile 的 REMOTE_PROXY_JUMP(仅建立主连接时需要;-O check/exit 只操作本地 mux socket)
|
||||
local r_jump jump_opt=""
|
||||
r_jump=$(grep -E '^REMOTE_PROXY_JUMP=' "$p" 2>/dev/null | cut -d'=' -f2- | tr -d '"' | tr -d "'" || true)
|
||||
[ -n "${r_jump}" ] && jump_opt="-o ProxyJump=${r_jump}"
|
||||
if ssh -p "${r_port}" -o ControlPath="${sock}" ${jump_opt} -O check "${r_user}@${r_ip}" >/dev/null 2>&1; then
|
||||
echo " -> [${pre_idx}/${total}] ${r_user}@${r_ip}:${r_port} (连接已存在,复用)"
|
||||
continue
|
||||
fi
|
||||
echo " -> [${pre_idx}/${total}] 正在连接 ${r_user}@${r_ip}:${r_port} ..."
|
||||
ssh -p "${r_port}" -o ControlMaster=yes -o ControlPath="${sock}" -o ControlPersist=1800 -fN "${r_user}@${r_ip}" \
|
||||
ssh -p "${r_port}" -o ControlMaster=yes -o ControlPath="${sock}" -o ControlPersist=1800 ${jump_opt} -fN "${r_user}@${r_ip}" \
|
||||
&& echo -e " ${GREEN}[√] 连接已建立${NC}" \
|
||||
|| echo -e " ${YELLOW}[!] 连接失败,将在部署该节点时重试${NC}"
|
||||
done
|
||||
@@ -699,6 +703,8 @@ setup_ssh_control() {
|
||||
local socket_key="${REMOTE_USER}@${REMOTE_IP}:${REMOTE_PORT}"
|
||||
SSH_CONTROL_PATH="${SSH_CONTROL_DIR}/${socket_key}"
|
||||
SSH_OPTS="-o ControlMaster=auto -o ControlPath=${SSH_CONTROL_PATH} -o ControlPersist=1800"
|
||||
# 非直连节点:profile 提供 REMOTE_PROXY_JUMP 时经跳板机转发(对齐 fetch_results.sh)
|
||||
[ -n "${REMOTE_PROXY_JUMP:-}" ] && SSH_OPTS="${SSH_OPTS} -o ProxyJump=${REMOTE_PROXY_JUMP}"
|
||||
mkdir -p "${SSH_CONTROL_DIR}"
|
||||
|
||||
# 若已有活跃的主连接 (如批量部署父进程已建立),直接复用,避免重复认证。
|
||||
|
||||
+60
-16
@@ -14,6 +14,7 @@
|
||||
# ./scripts/fetch_results.sh -d /backup/dcts-salvage # 覆盖备份根目录
|
||||
# ./scripts/fetch_results.sh --with-work # 连带拉取 data/work 沙盒残留
|
||||
# ./scripts/fetch_results.sh --include-local # 把本机节点也复制进备份树
|
||||
# ./scripts/fetch_results.sh --force # 强制重拉:忽略本地已存在的同名目录,全部重新 scp 覆盖
|
||||
#
|
||||
# 传输策略(两类节点都走增量,重复运行只补差异、不重拷全量):
|
||||
# - 双方均可用 rsync → rsync 增量(断点续传、幂等,自动补新增/变更文件)
|
||||
@@ -47,6 +48,7 @@ ONLY_NODE=""
|
||||
WITH_WORK=false
|
||||
INCLUDE_LOCAL=false
|
||||
INTERACTIVE=false
|
||||
FORCE_REPULL=false
|
||||
HAD_ARGS=false
|
||||
SELECTED_PROFILES=()
|
||||
|
||||
@@ -61,6 +63,7 @@ while [[ $# -gt 0 ]]; do
|
||||
-i|--interactive) INTERACTIVE=true; shift ;;
|
||||
--with-work) WITH_WORK=true; shift ;;
|
||||
--include-local) INCLUDE_LOCAL=true; shift ;;
|
||||
--force|-f) FORCE_REPULL=true; shift ;;
|
||||
-h|--help)
|
||||
sed -n '2,32p' "$0"
|
||||
exit 0 ;;
|
||||
@@ -109,21 +112,25 @@ remote_list_dir() { # $1=user $2=ip $3=port $4=ssh_opts $5=远端绝对目录
|
||||
}
|
||||
|
||||
# scp 单个远端目录到本地,采用"先落地 .partial 再原子 mv"模式:
|
||||
# 1. 目标目录 ${dest}/${d} 已存在 → 视为已同步,跳过(保持原增量语义)
|
||||
# 1. 目标目录 ${dest}/${d} 已存在 → 视为已同步,跳过(保持原增量语义);
|
||||
# 但 force=true 时忽略该判断,强制重拉(用于同名目录内容已变更的场景)
|
||||
# 2. 否则 scp 远端目录到 ${dest}/${d}.partial
|
||||
# 3. scp 成功后 mv 为 ${dest}/${d}(同 dest 文件系统,原子语义)
|
||||
# 4. scp 失败 → 清理残留 .partial,return 非零,由调用方决定是否继续
|
||||
# 这样即便 scp 传输过程中断,也不会留下"伪完成"目录导致下次被跳过漏传。
|
||||
scp_dir() { # $1=u $2=ip $3=port $4=ssh_opts $5=远端根目录 $6=本地目标 $7=目录名
|
||||
local u="$1" ip="$2" port="$3" opts="$4" rdir="$5" dest="$6" d="$7"
|
||||
# 强制模式下只在 scp 成功后才清空旧目录再 mv,避免传输失败时丢失旧备份。
|
||||
scp_dir() { # $1=u $2=ip $3=port $4=ssh_opts $5=远端根目录 $6=本地目标 $7=目录名 $8=force
|
||||
local u="$1" ip="$2" port="$3" opts="$4" rdir="$5" dest="$6" d="$7" force="$8"
|
||||
local final="${dest}/${d}"
|
||||
if [ -e "${final}" ]; then
|
||||
if [ "${force}" != "true" ] && [ -e "${final}" ]; then
|
||||
return 0
|
||||
fi
|
||||
local tmp="${dest}/.${d}.partial"
|
||||
# 兜底清理可能的历史残留(上次中断留下的半成品)
|
||||
rm -rf "${tmp}"
|
||||
if scp -P "${port}" ${opts} -r "${u}@${ip}:${rdir}/${d}" "${tmp}"; then
|
||||
# 强制模式:先清旧目录再原子 mv(避免 mv 把新目录并入已存在的旧目录)
|
||||
[ "${force}" = "true" ] && rm -rf "${final}"
|
||||
mv "${tmp}" "${final}"
|
||||
else
|
||||
echo -e " ${RED}[!]${NC} scp 失败: ${d},清理残留 ${d}.partial"
|
||||
@@ -133,16 +140,32 @@ scp_dir() { # $1=u $2=ip $3=port $4=ssh_opts $5=远端根目录 $6=本地目标
|
||||
}
|
||||
|
||||
# 传输一个目录树(rsync 优先,退化 scp 目录级增量)
|
||||
sync_dir() { # $1=user $2=ip $3=port $4=远端绝对目录 $5=本地目标
|
||||
local u="$1" ip="$2" port="$3" rdir="$4" dest="$5"
|
||||
# $6=可选跳板 ([user@]host[:port],来自 profile 的 REMOTE_PROXY_JUMP):
|
||||
# 目标节点不与本机直连、需经中转节点(如 dckj-01 → dckj-02/03)时启用 ProxyJump。
|
||||
sync_dir() { # $1=user $2=ip $3=port $4=远端绝对目录 $5=本地目标 $6=jump
|
||||
local u="$1" ip="$2" port="$3" rdir="$4" dest="$5" jump="${6:-}"
|
||||
local ssh_opts="-o ControlMaster=auto -o ControlPath=${SSH_CONTROL_DIR}/cm-${u}@${ip}:${port} -o ControlPersist=1800"
|
||||
[ -n "${jump}" ] && ssh_opts="${ssh_opts} -o ProxyJump=${jump}"
|
||||
local jump_note=""
|
||||
[ -n "${jump}" ] && jump_note="(经跳板 ${jump})"
|
||||
|
||||
mkdir -p "${dest}"
|
||||
|
||||
# 连通性探测(BatchMode=no 允许交互输密码)。
|
||||
# 连通性与主连接建立(BatchMode=no 允许交互输密码)。
|
||||
# 先显式建立 ControlMaster 主连接(-fN 后台保活):跳板机/目标机的密码集中在此
|
||||
# 交互输入并各只输一次;成功后后续的探测/rsync/scp 全部复用 socket,不再要密码。
|
||||
# 若主连接已存在(如刚同步过其他目录),-O check 直接复用。
|
||||
if ! ssh -p "${port}" ${ssh_opts} -O check "${u}@${ip}" >/dev/null 2>&1; then
|
||||
echo -e " ${BLUE}[→]${NC} 建立 SSH 主连接${jump_note}(需分别输入跳板机/目标机密码)..."
|
||||
if ! ssh -p "${port}" ${ssh_opts} -fN "${u}@${ip}"; then
|
||||
echo -e " ${RED}[!]${NC} SSH 主连接建立失败: ${u}@${ip}${jump_note}(密码错误或链路不通),跳过本节点"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# 探测命令用 `echo ok` 而非 `true`:Windows OpenSSH 默认 shell 是 cmd/PowerShell,
|
||||
# 没有 Unix 的 true,会导致"密码正确但仍判连接失败"。
|
||||
if ! ssh -p "${port}" -o ConnectTimeout=8 ${ssh_opts} "${u}@${ip}" "echo ok" >/dev/null 2>&1; then
|
||||
if ! ssh -p "${port}" ${ssh_opts} "${u}@${ip}" "echo ok" >/dev/null 2>&1; then
|
||||
echo -e " ${RED}[!] 无法连接 ${u}@${ip},跳过本节点${NC}"
|
||||
return 1
|
||||
fi
|
||||
@@ -193,7 +216,7 @@ sync_dir() { # $1=user $2=ip $3=port $4=远端绝对目录 $5=本地目标
|
||||
echo -e " ${YELLOW}[!]${NC} 跳过不符合白名单的远端目录名: ${d}"
|
||||
continue
|
||||
fi
|
||||
if ! grep -Fxq -- "$d" <<<"${existing}"; then
|
||||
if [ "${FORCE_REPULL}" = "true" ] || ! grep -Fxq -- "$d" <<<"${existing}"; then
|
||||
missing="${missing}${d}"$'\n'
|
||||
fi
|
||||
done <<<"${remote_dirs}"
|
||||
@@ -201,9 +224,17 @@ sync_dir() { # $1=user $2=ip $3=port $4=远端绝对目录 $5=本地目标
|
||||
local total new
|
||||
total=$(printf '%s\n' "${remote_dirs}" | grep -c .)
|
||||
new=$(printf '%s' "${missing}" | grep -c .)
|
||||
echo -e " ${YELLOW}[→]${NC} scp 目录级增量: ${u}@${ip}:${rdir}/ 共 ${total} 个,缺失 ${new} 个 → ${dest}/"
|
||||
if [ "${FORCE_REPULL}" = "true" ]; then
|
||||
echo -e " ${YELLOW}[→]${NC} scp 强制重拉: ${u}@${ip}:${rdir}/ 共 ${total} 个,全部重拉 ${new} 个 → ${dest}/"
|
||||
else
|
||||
echo -e " ${YELLOW}[→]${NC} scp 目录级增量: ${u}@${ip}:${rdir}/ 共 ${total} 个,缺失 ${new} 个 → ${dest}/"
|
||||
fi
|
||||
if [ "${new}" -eq 0 ]; then
|
||||
echo -e " ${GREEN}[√]${NC} 本地已是最新,无新增目录"
|
||||
if [ "${FORCE_REPULL}" = "true" ]; then
|
||||
echo -e " ${GREEN}[√]${NC} 强制重拉完成,无远端目录"
|
||||
else
|
||||
echo -e " ${GREEN}[√]${NC} 本地已是最新,无新增目录"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
# scp 原子写入:先落地到 .partial 临时目录,成功后同文件系统原子 mv 为最终目录名。
|
||||
@@ -214,15 +245,19 @@ sync_dir() { # $1=user $2=ip $3=port $4=远端绝对目录 $5=本地目标
|
||||
rm -rf "${dest}"/.*.partial 2>/dev/null || true
|
||||
while IFS= read -r d; do
|
||||
[ -n "$d" ] || continue
|
||||
echo -e " ${YELLOW}[→]${NC} scp 新增: ${d}"
|
||||
scp_dir "${u}" "${ip}" "${port}" "${ssh_opts}" "${rdir}" "${dest}" "$d"
|
||||
if [ "${FORCE_REPULL}" = "true" ]; then
|
||||
echo -e " ${YELLOW}[→]${NC} scp 强制重拉: ${d}"
|
||||
else
|
||||
echo -e " ${YELLOW}[→]${NC} scp 新增: ${d}"
|
||||
fi
|
||||
scp_dir "${u}" "${ip}" "${port}" "${ssh_opts}" "${rdir}" "${dest}" "$d" "${FORCE_REPULL}"
|
||||
done <<<"${missing}"
|
||||
fi
|
||||
}
|
||||
|
||||
# 备份单个节点
|
||||
backup_node() { # $1 = profile 文件
|
||||
local p="$1" node_id role env_mode u ip port dir
|
||||
local p="$1" node_id role env_mode u ip port dir jump
|
||||
node_id=$(read_profile_var "$p" DCTS_NODE_ID)
|
||||
role=$(read_profile_var "$p" DEPLOY_ROLE node)
|
||||
env_mode=$(read_profile_var "$p" DEPLOY_ENV "")
|
||||
@@ -230,6 +265,7 @@ backup_node() { # $1 = profile 文件
|
||||
ip=$(read_profile_var "$p" REMOTE_IP "")
|
||||
port=$(read_profile_var "$p" REMOTE_PORT 22)
|
||||
dir=$(read_profile_var "$p" REMOTE_DIR "")
|
||||
jump=$(read_profile_var "$p" REMOTE_PROXY_JUMP "")
|
||||
|
||||
# 跳过 server 角色(server 的 result 不是计算产物,且 seeds 另行备份)
|
||||
if [ "${role}" = "server" ]; then
|
||||
@@ -239,6 +275,8 @@ backup_node() { # $1 = profile 文件
|
||||
# 本地节点:result 就在本机 ./data/result,默认不重复备份
|
||||
if [ "${env_mode}" = "local" ]; then
|
||||
if [ "${INCLUDE_LOCAL}" = "true" ]; then
|
||||
# 本机节点 ID 优先取根目录 .env(与节点注册、看板展示的 DCTS_NODE_ID 一致)
|
||||
[ -n "${node_id}" ] || node_id=$(grep -E "^DCTS_NODE_ID=" .env 2>/dev/null | head -n1 | cut -d'=' -f2- | tr -d '"' | tr -d "'")
|
||||
[ -n "${node_id}" ] || node_id="node-local"
|
||||
else
|
||||
echo -e "${YELLOW}== ${p##*/}: 本地节点(${node_id:-localhost}),result 已在 ./data/result,跳过(--include-local 可纳入备份树)${NC}"
|
||||
@@ -272,7 +310,7 @@ backup_node() { # $1 = profile 文件
|
||||
cp -a ./data/result/. "${dest_rc}/"
|
||||
fi
|
||||
else
|
||||
sync_dir "${u}" "${ip}" "${port}" "${dir}/data/result" "${dest_rc}" || return 1
|
||||
sync_dir "${u}" "${ip}" "${port}" "${dir}/data/result" "${dest_rc}" "${jump}" || return 1
|
||||
fi
|
||||
echo -e " ${GREEN}[√]${NC} result 已同步,共 $(ls -1 "${dest_rc}" 2>/dev/null | grep -c . || true) 个网格点子目录"
|
||||
|
||||
@@ -288,7 +326,7 @@ backup_node() { # $1 = profile 文件
|
||||
cp -a ./data/work/. "${dest_wk}/" 2>/dev/null || true
|
||||
fi
|
||||
else
|
||||
sync_dir "${u}" "${ip}" "${port}" "${dir}/data/work" "${dest_wk}" || true
|
||||
sync_dir "${u}" "${ip}" "${port}" "${dir}/data/work" "${dest_wk}" "${jump}" || true
|
||||
fi
|
||||
fi
|
||||
}
|
||||
@@ -365,6 +403,12 @@ interactive_select() {
|
||||
[[ "${WK_CHOICE,,}" == "y" || "${WK_CHOICE,,}" == "yes" ]] && WITH_WORK=true
|
||||
fi
|
||||
|
||||
# Windows(scp)节点是否强制重拉:本地即使已有同名目录也全部重新下载覆盖
|
||||
if [ "${FORCE_REPULL}" != "true" ]; then
|
||||
read -r -p "是否强制重拉(忽略本地已有同名目录,全部重新 scp 覆盖,仅对 Windows/scp 节点生效)? [y/N]: " FORCE_CHOICE
|
||||
[[ "${FORCE_CHOICE,,}" == "y" || "${FORCE_CHOICE,,}" == "yes" ]] && FORCE_REPULL=true
|
||||
fi
|
||||
|
||||
# 用户显式勾选本机节点 → 自动纳入备份树(否则 backup_node 会跳过本机)
|
||||
local p2 env2
|
||||
for p2 in "${SELECTED_PROFILES[@]}"; do
|
||||
|
||||
Reference in New Issue
Block a user