后端核心变更: - API 层: 新增 AppError 枚举统一错误类型,替代散落的 (StatusCode, String) - Agent 工具域: 重组为 astro/system/ 和 astro/research/ 两级域,新增 ProcessPaperTool 流水线工具 - 安全: 新增 SSRF 双层防护 (同步字符串级 + 异步 DNS 解析级),覆盖 IPv4/IPv6 私网段 - 弱密码检测: 扩展弱密码列表并增加最小长度检查 - LLM 客户端: 新增 ChatCompleter/Embedder trait,支持依赖注入与批量向量化 embed_batch - 批量处理: AssetBatch 从串行改为 Semaphore 并发池 (BATCH_CONCURRENCY=3) - 分块器: 重写为三阶段结构化管线 (章节解析→短节合并→带标题路径子块) - RAG: embedding 计算移出事务,RetrievalResult 新增 headings/section_index 字段 - 检索: ADS/arXiv 并行检索 (tokio::join!),去重改用 HashSet,本地库回填批量 IN 查询 - 天体查询: Sesame API 升级到 v4,新增视差误差/自行/视向速度/多波段测光字段 - 迁移: 14 个增量文件合并为单一 init.sql,支持 sqlx::migrate! 内存库集成测试 - 测试: circuit_breaker/hooks/task_board/session/memory/streaming_executor 新增修正 15+ 测试 前端架构重构: - 目录重组: features/ → pages/ + components/ + hooks/ 三层分离 - App.tsx 从 1181 行压缩至 ~174 行 (逻辑抽入 9 个自定义 Hook) - Agent 面板拆分为 AgentSessionSidebar/AgentMessageList/AgentInputArea 子组件 - 新增 GlobalDialog/PaperDetailModal/UncachedPaperModal 通用对话框组件 - 工具函数抽取: celestial.ts (天体坐标格式), paper.tsx (文献信息渲染)
376 lines
10 KiB
TypeScript
376 lines
10 KiB
TypeScript
// dashboard/src/hooks/useSyncState.ts
|
|
import { useState, useEffect, useRef } from 'react';
|
|
import axios from 'axios';
|
|
import type { SavedSyncQuery } from '../types';
|
|
|
|
export interface BatchStatus {
|
|
active: boolean;
|
|
total: number;
|
|
downloaded: number;
|
|
parsed: number;
|
|
download_failed: number;
|
|
parse_failed: number;
|
|
current_bibcode: string;
|
|
logs: string[];
|
|
action?: 'download' | 'parse' | 'translate' | 'embed' | 'target';
|
|
}
|
|
|
|
export interface HarvestStatus {
|
|
active: boolean;
|
|
query: string;
|
|
source: string;
|
|
synced: number;
|
|
total: number;
|
|
}
|
|
|
|
export function useSyncState() {
|
|
const [query, setQuery] = useState('');
|
|
const [source, setSource] = useState<'all' | 'ads' | 'arxiv'>('all');
|
|
const [limit, setLimit] = useState<number>(200);
|
|
const [estimating, setEstimating] = useState(false);
|
|
const [estimatedCount, setEstimatedCount] = useState<number | null>(null);
|
|
const [status, setStatus] = useState<HarvestStatus>({
|
|
active: false,
|
|
query: '',
|
|
source: '',
|
|
synced: 0,
|
|
total: 0,
|
|
});
|
|
const [errorMsg, setErrorMsg] = useState<string | null>(null);
|
|
const [syncQueries, setSyncQueries] = useState<SavedSyncQuery[]>([]);
|
|
const pollIntervalRef = useRef<any>(null);
|
|
|
|
// 批量下载与解析相关状态
|
|
const [targetPhase, setTargetPhase] = useState<'download' | 'parse' | 'translate' | 'embed' | 'target'>('download');
|
|
const [batchLimitCount, setBatchLimitCount] = useState<number>(100);
|
|
const [sortOrder, setSortOrder] = useState<'default' | 'pub_year_desc' | 'created_at_desc'>('default');
|
|
const [skipCompleted, setSkipCompleted] = useState<boolean>(true);
|
|
const [skipFailed, setSkipFailed] = useState<boolean>(false);
|
|
const [skipPrecedingFailed, setSkipPrecedingFailed] = useState<boolean>(false);
|
|
const [skipPrecedingUncompleted, setSkipPrecedingUncompleted] = useState<boolean>(false);
|
|
|
|
const [batchStatus, setBatchStatus] = useState<BatchStatus>({
|
|
active: false,
|
|
total: 0,
|
|
downloaded: 0,
|
|
parsed: 0,
|
|
download_failed: 0,
|
|
parse_failed: 0,
|
|
current_bibcode: '',
|
|
logs: [],
|
|
});
|
|
const [batchError, setBatchError] = useState<string | null>(null);
|
|
const batchPollIntervalRef = useRef<any>(null);
|
|
const logsContainerRef = useRef<HTMLDivElement | null>(null);
|
|
|
|
const [showBuilder, setShowBuilder] = useState(false);
|
|
const [rules, setRules] = useState<Array<{ field: string; op: string; val: string }>>([
|
|
{ field: 'all', op: 'AND', val: '' }
|
|
]);
|
|
|
|
// 当高级表单规则变化时,自动更新同步输入框的检索式
|
|
const updateQueryFromRules = (currentRules: typeof rules) => {
|
|
let qParts: string[] = [];
|
|
currentRules.forEach((rule, idx) => {
|
|
if (!rule.val.trim()) return;
|
|
let valStr = rule.val.trim();
|
|
if (valStr.includes(' ') && !valStr.startsWith('"') && !valStr.startsWith('(')) {
|
|
valStr = `"${valStr}"`;
|
|
}
|
|
|
|
let fieldPart = '';
|
|
if (rule.field !== 'all') {
|
|
fieldPart = `${rule.field}:${valStr}`;
|
|
} else {
|
|
fieldPart = valStr;
|
|
}
|
|
|
|
if (idx === 0) {
|
|
qParts.push(fieldPart);
|
|
} else {
|
|
qParts.push(`${rule.op} ${fieldPart}`);
|
|
}
|
|
});
|
|
setQuery(qParts.join(' '));
|
|
};
|
|
|
|
const handleAddRule = () => {
|
|
setRules(prev => [...prev, { field: 'all', op: 'AND', val: '' }]);
|
|
};
|
|
|
|
const handleRemoveRule = (idx: number) => {
|
|
const next = rules.filter((_, i) => i !== idx);
|
|
setRules(next);
|
|
updateQueryFromRules(next);
|
|
};
|
|
|
|
const handleRuleChange = (idx: number, key: 'field' | 'op' | 'val', value: string) => {
|
|
const next = rules.map((r, i) => i === idx ? { ...r, [key]: value } : r);
|
|
setRules(next);
|
|
updateQueryFromRules(next);
|
|
};
|
|
|
|
// 获取历史检索配置
|
|
const fetchSyncQueries = async () => {
|
|
try {
|
|
const res = await axios.get<SavedSyncQuery[]>(`/api/sync/queries?t=${Date.now()}`);
|
|
setSyncQueries(res.data);
|
|
} catch (e) {
|
|
console.error('获取检索配置列表失败', e);
|
|
}
|
|
};
|
|
|
|
const handleDeleteQuery = async (id: number) => {
|
|
try {
|
|
await axios.delete(`/api/sync/queries/${id}`);
|
|
fetchSyncQueries();
|
|
} catch (e) {
|
|
console.error('删除配置失败', e);
|
|
}
|
|
};
|
|
|
|
const handleReuseQuery = (sq: SavedSyncQuery) => {
|
|
setQuery(sq.query);
|
|
setSource(sq.source as any);
|
|
setLimit(sq.limit_count);
|
|
};
|
|
|
|
const handleQuickSync = async (sq: SavedSyncQuery) => {
|
|
setErrorMsg(null);
|
|
|
|
// 立即更新前端本地状态,提供即时的 UI 反馈并避免轮询竞态
|
|
setStatus(prev => ({
|
|
...prev,
|
|
active: true,
|
|
query: sq.query,
|
|
source: sq.source as any,
|
|
synced: 0,
|
|
total: sq.limit_count,
|
|
}));
|
|
startPolling();
|
|
|
|
try {
|
|
await axios.post('/api/sync/meta/run', {
|
|
q: sq.query,
|
|
source: sq.source,
|
|
limit: sq.limit_count,
|
|
});
|
|
fetchStatus();
|
|
setTimeout(fetchSyncQueries, 500);
|
|
} catch (e: any) {
|
|
console.error(e);
|
|
setErrorMsg(e.response?.data || '启动快速同步失败。');
|
|
fetchStatus();
|
|
}
|
|
};
|
|
|
|
// 获取当前的元数据同步状态
|
|
const fetchStatus = async () => {
|
|
try {
|
|
const res = await axios.get<HarvestStatus>(`/api/sync/meta/status?t=${Date.now()}`);
|
|
setStatus(res.data);
|
|
if (res.data.active) {
|
|
startPolling();
|
|
} else {
|
|
if (pollIntervalRef.current) {
|
|
clearInterval(pollIntervalRef.current);
|
|
pollIntervalRef.current = null;
|
|
fetchSyncQueries();
|
|
}
|
|
}
|
|
} catch (e) {
|
|
console.error('获取同步状态失败', e);
|
|
}
|
|
};
|
|
|
|
// 开始轮询元数据同步状态
|
|
const startPolling = () => {
|
|
if (pollIntervalRef.current) return;
|
|
pollIntervalRef.current = setInterval(fetchStatus, 1000);
|
|
};
|
|
|
|
useEffect(() => {
|
|
fetchStatus();
|
|
fetchSyncQueries();
|
|
|
|
return () => {
|
|
if (pollIntervalRef.current) {
|
|
clearInterval(pollIntervalRef.current);
|
|
}
|
|
};
|
|
}, []);
|
|
|
|
// 批量下载与解析相关的网络操作
|
|
const fetchBatchStatus = async () => {
|
|
try {
|
|
const res = await axios.get<BatchStatus>(`/api/batch/asset/status?t=${Date.now()}`);
|
|
setBatchStatus(res.data);
|
|
if (res.data.active) {
|
|
startBatchPolling();
|
|
} else if (batchPollIntervalRef.current) {
|
|
clearInterval(batchPollIntervalRef.current);
|
|
batchPollIntervalRef.current = null;
|
|
}
|
|
} catch (e) {
|
|
console.error('获取处理状态失败', e);
|
|
}
|
|
};
|
|
|
|
const startBatchPolling = () => {
|
|
if (batchPollIntervalRef.current) return;
|
|
batchPollIntervalRef.current = setInterval(fetchBatchStatus, 1000);
|
|
};
|
|
|
|
const handleStartBatch = async () => {
|
|
setBatchError(null);
|
|
try {
|
|
await axios.post('/api/batch/asset/run', {
|
|
target_phase: targetPhase,
|
|
limit_count: batchLimitCount,
|
|
sort_order: sortOrder,
|
|
skip_completed: skipCompleted,
|
|
skip_failed: skipFailed,
|
|
skip_preceding_failed: skipPrecedingFailed,
|
|
skip_preceding_uncompleted: skipPrecedingUncompleted,
|
|
});
|
|
fetchBatchStatus();
|
|
startBatchPolling();
|
|
} catch (e: any) {
|
|
console.error(e);
|
|
setBatchError(e.response?.data || '启动批量任务失败。');
|
|
}
|
|
};
|
|
|
|
const handleStopBatch = async () => {
|
|
try {
|
|
await axios.post('/api/batch/asset/stop');
|
|
fetchBatchStatus();
|
|
} catch (e: any) {
|
|
console.error(e);
|
|
setBatchError(e.response?.data || '停止任务失败。');
|
|
}
|
|
};
|
|
|
|
useEffect(() => {
|
|
fetchBatchStatus();
|
|
|
|
return () => {
|
|
if (batchPollIntervalRef.current) {
|
|
clearInterval(batchPollIntervalRef.current);
|
|
}
|
|
};
|
|
}, []);
|
|
|
|
// 日志终端自动滚动到底部
|
|
useEffect(() => {
|
|
const container = logsContainerRef.current;
|
|
if (container) {
|
|
const isCloseToBottom = container.scrollHeight - container.scrollTop - container.clientHeight < 80;
|
|
if (isCloseToBottom) {
|
|
container.scrollTop = container.scrollHeight;
|
|
}
|
|
}
|
|
}, [batchStatus.logs]);
|
|
|
|
// 估算文献总量
|
|
const handleEstimate = async () => {
|
|
if (!query.trim()) {
|
|
setErrorMsg('请输入检索关键词!');
|
|
return;
|
|
}
|
|
setErrorMsg(null);
|
|
setEstimating(true);
|
|
setEstimatedCount(null);
|
|
try {
|
|
const res = await axios.get<{ total: number }>('/api/sync/meta/count', {
|
|
params: { q: query.trim(), source }
|
|
});
|
|
setEstimatedCount(res.data.total);
|
|
} catch (e: any) {
|
|
console.error(e);
|
|
setErrorMsg(e.response?.data || '估算文献总量失败,请检查 API 密钥或网络。');
|
|
} finally {
|
|
setEstimating(false);
|
|
}
|
|
};
|
|
|
|
// 启动任务
|
|
const handleStartHarvest = async () => {
|
|
if (!query.trim()) {
|
|
setErrorMsg('请输入检索关键词!');
|
|
return;
|
|
}
|
|
setErrorMsg(null);
|
|
|
|
setStatus(prev => ({
|
|
...prev,
|
|
active: true,
|
|
query: query.trim(),
|
|
source,
|
|
synced: 0,
|
|
total: 0,
|
|
}));
|
|
startPolling();
|
|
|
|
try {
|
|
await axios.post('/api/sync/meta/run', {
|
|
q: query.trim(),
|
|
source,
|
|
limit: limit,
|
|
});
|
|
fetchStatus();
|
|
setTimeout(fetchSyncQueries, 500);
|
|
} catch (e: any) {
|
|
console.error(e);
|
|
setErrorMsg(e.response?.data || '启动元数据同步任务失败。');
|
|
fetchStatus();
|
|
}
|
|
};
|
|
|
|
return {
|
|
query,
|
|
setQuery,
|
|
source,
|
|
setSource,
|
|
limit,
|
|
setLimit,
|
|
estimating,
|
|
estimatedCount,
|
|
status,
|
|
errorMsg,
|
|
setErrorMsg,
|
|
syncQueries,
|
|
targetPhase,
|
|
setTargetPhase,
|
|
batchLimitCount,
|
|
setBatchLimitCount,
|
|
sortOrder,
|
|
setSortOrder,
|
|
skipCompleted,
|
|
setSkipCompleted,
|
|
skipFailed,
|
|
setSkipFailed,
|
|
skipPrecedingFailed,
|
|
setSkipPrecedingFailed,
|
|
skipPrecedingUncompleted,
|
|
setSkipPrecedingUncompleted,
|
|
batchStatus,
|
|
batchError,
|
|
setBatchError,
|
|
showBuilder,
|
|
setShowBuilder,
|
|
rules,
|
|
logsContainerRef,
|
|
handleAddRule,
|
|
handleRemoveRule,
|
|
handleRuleChange,
|
|
handleDeleteQuery,
|
|
handleReuseQuery,
|
|
handleQuickSync,
|
|
handleEstimate,
|
|
handleStartHarvest,
|
|
handleStartBatch,
|
|
handleStopBatch,
|
|
};
|
|
}
|