Asfmq
f885c0a4a8
- 服务层拆分：删除 api/helpers.rs，新增 citation/note/session/pipeline/paper/vision 独立服务模块
- Agent 工具精简：paper_content+paper_outline 合并为 paper.rs，图片分析逻辑下沉至 services/vision
- 并发模型升级：std::sync::{Mutex,RwLock} → tokio::sync::{Mutex,RwLock}，消除 async
上下文中的阻塞风险
- 客户端加固：HTTP 客户端统一超时配置、ADS 429 / arXiv 503 自动重试、构造函数返回 Result
- 启动安全：全局 panic hook 日志化、空密码拒绝启动、向量表维度不匹配需显式确认
- CLI 扩展：构建完整 AppState 复用服务层，新增 Content/Outline/Citations/Search/Process 子命令
- 前端：移动端汉堡菜单、侧栏滑出面板、引用星系触屏手势（单指拖拽/双指缩放）
407 lines
12 KiB
Rust
407 lines
12 KiB
Rust
// src/agent/memory/dedup.rs
|
||
//
|
||
// 记忆去重支持 — 参考 Claude Code memdir 提示词中的去重规则。
|
||
//
|
||
// 在保存新记忆前检查是否有可更新的现有条目，
|
||
// 构建现有记忆的 manifest 供 LLM 参考以减少重复写入。
|
||
|
||
use std::fs;
|
||
use std::path::Path;
|
||
|
||
use super::types::MemoryEntry;
|
||
|
||
/// 构建现有记忆的清单预览（供 LLM 了解已存在的内容）。
|
||
/// 在 save_memory 成功后注入到工具输出中。
|
||
pub fn build_manifest_preview(entries: &[MemoryEntry]) -> String {
|
||
if entries.is_empty() {
|
||
return "当前无其他记忆条目。".to_string();
|
||
}
|
||
|
||
let mut lines = vec!["当前记忆清单：".to_string()];
|
||
for entry in entries {
|
||
let type_label = match entry.memory_type {
|
||
super::types::MemoryType::User => "[偏好]",
|
||
super::types::MemoryType::Feedback => "[反馈]",
|
||
super::types::MemoryType::Project => "[项目]",
|
||
super::types::MemoryType::Reference => "[参考]",
|
||
};
|
||
lines.push(format!(
|
||
"- {} `{}` {}: {}",
|
||
type_label, entry.slug, entry.name, entry.description
|
||
));
|
||
}
|
||
lines.join("\n")
|
||
}
|
||
|
||
/// 检查 slug 是否在磁盘上已存在。
|
||
pub fn slug_exists(memory_dir: &Path, slug: &str) -> bool {
|
||
let file_path = memory_dir.join(format!("{}.md", slug));
|
||
file_path.exists()
|
||
}
|
||
|
||
/// 列出所有现有 slug（从磁盘直接读取，避免依赖 MemoryManager 状态）。
|
||
pub fn list_existing_slugs(memory_dir: &Path) -> Vec<String> {
|
||
let mut slugs = Vec::new();
|
||
if let Ok(entries) = fs::read_dir(memory_dir) {
|
||
for entry in entries.flatten() {
|
||
let path = entry.path();
|
||
if path.is_dir() {
|
||
continue;
|
||
}
|
||
if let Some(file_name) = path.file_name().and_then(|n| n.to_str()) {
|
||
if file_name == "MEMORY.md" || !file_name.ends_with(".md") {
|
||
continue;
|
||
}
|
||
if let Some(slug) = file_name.strip_suffix(".md") {
|
||
slugs.push(slug.to_string());
|
||
}
|
||
}
|
||
}
|
||
}
|
||
slugs.sort();
|
||
slugs
|
||
}
|
||
|
||
// ── Jaccard 相似度去重 ──
|
||
|
||
/// 计算两个字符串的 Jaccard 相似度（基于字符级 bigram）。
|
||
///
|
||
/// 使用 bigram 而非词级分词以正确处理中文（不依赖分词器）。
|
||
/// 值域 [0.0, 1.0]，阈值 ≥0.70 通常视为重复。
|
||
///
|
||
/// 参考 Martian-Engineering/agent-memory 的 70% Jaccard 门控。
|
||
pub fn jaccard_similarity(a: &str, b: &str) -> f64 {
|
||
let bigrams_a: std::collections::HashSet<String> = bigrams(a);
|
||
let bigrams_b: std::collections::HashSet<String> = bigrams(b);
|
||
|
||
if bigrams_a.is_empty() && bigrams_b.is_empty() {
|
||
return 1.0; // 两个空字符串完全相同
|
||
}
|
||
|
||
let intersection = bigrams_a.intersection(&bigrams_b).count();
|
||
let union = bigrams_a.union(&bigrams_b).count();
|
||
|
||
if union == 0 {
|
||
return 0.0;
|
||
}
|
||
intersection as f64 / union as f64
|
||
}
|
||
|
||
/// 提取字符串的字符级 bigram 集合。
|
||
fn bigrams(s: &str) -> std::collections::HashSet<String> {
|
||
let chars: Vec<char> = s.chars().collect();
|
||
let mut set = std::collections::HashSet::new();
|
||
if chars.len() < 2 {
|
||
// 单字符内容：将单字符本身作为 bigram
|
||
if !chars.is_empty() {
|
||
set.insert(chars[0].to_string());
|
||
}
|
||
return set;
|
||
}
|
||
for window in chars.windows(2) {
|
||
set.insert(format!("{}{}", window[0], window[1]));
|
||
}
|
||
set
|
||
}
|
||
|
||
/// 检查新内容与现有记忆是否高度重复。
|
||
/// 返回重复的 slug，或在无重复时返回 None。
|
||
pub fn find_duplicate_by_content(
|
||
new_content: &str,
|
||
existing_entries: &[MemoryEntry],
|
||
threshold: f64,
|
||
) -> Option<String> {
|
||
for entry in existing_entries {
|
||
if !entry.status.is_active() {
|
||
continue;
|
||
}
|
||
let sim = jaccard_similarity(new_content, &entry.content);
|
||
if sim >= threshold {
|
||
return Some(entry.slug.clone());
|
||
}
|
||
}
|
||
None
|
||
}
|
||
|
||
// ── 写入时内容质量门控 ──
|
||
|
||
/// 内容质量检查结果
|
||
#[derive(Debug, PartialEq, Eq)]
|
||
pub enum QualityCheck {
|
||
/// 通过质量检查
|
||
Accept,
|
||
/// 太短：有效字符不足
|
||
TooShort(usize),
|
||
/// 瞬时状态描述
|
||
TransientState,
|
||
/// 模糊语言
|
||
VagueLanguage(String),
|
||
/// 纯代码片段
|
||
CodePattern,
|
||
}
|
||
|
||
/// 瞬时状态关键词（中文 + 英文）
|
||
const TRANSIENT_PATTERNS: &[&str] = &[
|
||
"正在做",
|
||
"正在写",
|
||
"正在调试",
|
||
"正在看",
|
||
"准备做",
|
||
"is working on",
|
||
"currently",
|
||
"right now",
|
||
"at the moment",
|
||
];
|
||
|
||
/// 模糊语言关键词
|
||
const VAGUE_PATTERNS: &[(&str, &str)] = &[
|
||
("maybe", "可能"),
|
||
("probably", "大概"),
|
||
("perhaps", "也许"),
|
||
("might be", "或许"),
|
||
("似乎", "似乎"),
|
||
("好像", "好像"),
|
||
];
|
||
|
||
/// 代码模式检测（纯代码片段不应作为记忆）
|
||
const CODE_PATTERNS: &[&str] = &[
|
||
"fn ",
|
||
"impl ",
|
||
"struct ",
|
||
"pub fn",
|
||
"use crate",
|
||
"function ",
|
||
"const ",
|
||
"let mut",
|
||
"&mut",
|
||
"import {",
|
||
"from \"",
|
||
"export ",
|
||
];
|
||
|
||
/// 最小内容长度（有效字符）。
|
||
/// 中文信息密度高，10 字即可表达完整语义。
|
||
const MIN_CONTENT_CHARS: usize = 10;
|
||
|
||
/// 检查内容质量（写入时门控）。
|
||
///
|
||
/// 仅返回警告 — 不强制拒绝，由 LLM 最终决定。
|
||
/// 参考 OpenClaw claw-mem 写入时门控 + agent-memory 写入规则。
|
||
pub fn check_content_quality(content: &str) -> QualityCheck {
|
||
let trimmed = content.trim();
|
||
|
||
// 1. 长度检查
|
||
let char_count = trimmed.chars().count();
|
||
if char_count < MIN_CONTENT_CHARS {
|
||
return QualityCheck::TooShort(char_count);
|
||
}
|
||
|
||
// 2. 瞬时状态检查
|
||
let lower = trimmed.to_lowercase();
|
||
for pattern in TRANSIENT_PATTERNS {
|
||
if lower.contains(pattern) {
|
||
return QualityCheck::TransientState;
|
||
}
|
||
}
|
||
|
||
// 3. 模糊语言检查
|
||
for (en, zh) in VAGUE_PATTERNS {
|
||
if lower.contains(en) || lower.contains(zh) {
|
||
return QualityCheck::VagueLanguage(if lower.contains(en) {
|
||
en.to_string()
|
||
} else {
|
||
zh.to_string()
|
||
});
|
||
}
|
||
}
|
||
|
||
// 4. 代码模式检查
|
||
for pattern in CODE_PATTERNS {
|
||
if trimmed.contains(pattern) {
|
||
return QualityCheck::CodePattern;
|
||
}
|
||
}
|
||
|
||
QualityCheck::Accept
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use std::path::PathBuf;
|
||
|
||
fn make_entry(slug: &str, name: &str, desc: &str) -> MemoryEntry {
|
||
MemoryEntry {
|
||
slug: slug.to_string(),
|
||
name: name.to_string(),
|
||
description: desc.to_string(),
|
||
memory_type: super::super::types::MemoryType::User,
|
||
mtime: 1000,
|
||
content: desc.to_string(),
|
||
path: PathBuf::from(slug),
|
||
status: super::super::types::MemoryStatus::Active,
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn test_manifest_preview_empty() {
|
||
let preview = build_manifest_preview(&[]);
|
||
assert!(preview.contains("无其他记忆条目"));
|
||
}
|
||
|
||
#[test]
|
||
fn test_manifest_preview_with_entries() {
|
||
let entries = vec![
|
||
make_entry("user-role", "用户角色", "数据科学家"),
|
||
make_entry("feedback-tests", "测试反馈", "不要 mock 数据库"),
|
||
];
|
||
let preview = build_manifest_preview(&entries);
|
||
assert!(preview.contains("user-role"));
|
||
assert!(preview.contains("feedback-tests"));
|
||
assert!(preview.contains("数据科学家"));
|
||
assert!(preview.contains("不要 mock 数据库"));
|
||
}
|
||
|
||
#[test]
|
||
fn test_slug_exists_true() {
|
||
let dir = std::env::temp_dir().join("astro_memory_test_dedup");
|
||
fs::create_dir_all(&dir).unwrap();
|
||
fs::write(dir.join("existing.md"), "test").unwrap();
|
||
assert!(slug_exists(&dir, "existing"));
|
||
fs::remove_dir_all(&dir).unwrap();
|
||
}
|
||
|
||
#[test]
|
||
fn test_slug_exists_false() {
|
||
let dir = std::env::temp_dir().join("astro_memory_test_dedup_nonexist");
|
||
assert!(!slug_exists(&dir, "nonexistent"));
|
||
}
|
||
|
||
#[test]
|
||
fn test_list_existing_slugs() {
|
||
let dir = std::env::temp_dir().join("astro_memory_test_list_slugs");
|
||
fs::create_dir_all(&dir).unwrap();
|
||
fs::write(dir.join("alpha.md"), "a").unwrap();
|
||
fs::write(dir.join("beta.md"), "b").unwrap();
|
||
fs::write(dir.join("MEMORY.md"), "index").unwrap();
|
||
|
||
let slugs = list_existing_slugs(&dir);
|
||
assert!(slugs.contains(&"alpha".to_string()));
|
||
assert!(slugs.contains(&"beta".to_string()));
|
||
assert!(!slugs.contains(&"MEMORY".to_string()));
|
||
|
||
fs::remove_dir_all(&dir).unwrap();
|
||
}
|
||
|
||
// ── Jaccard 相似度测试 ──
|
||
|
||
#[test]
|
||
fn test_jaccard_identical() {
|
||
let sim = jaccard_similarity("hello world", "hello world");
|
||
assert!((sim - 1.0).abs() < 0.01, "完全相同应为 1.0，实际 {}", sim);
|
||
}
|
||
|
||
#[test]
|
||
fn test_jaccard_completely_different() {
|
||
let sim = jaccard_similarity("hello world", "abc xyz");
|
||
assert!(sim < 0.3, "完全不同应较低，实际 {}", sim);
|
||
}
|
||
|
||
#[test]
|
||
fn test_jaccard_high_overlap() {
|
||
let sim = jaccard_similarity(
|
||
"用户偏好使用 Rust 开发后端服务",
|
||
"用户偏好使用 Rust 开发后端",
|
||
);
|
||
assert!(sim > 0.5, "高重叠应 >0.5，实际 {}", sim);
|
||
}
|
||
|
||
#[test]
|
||
fn test_jaccard_chinese_bigram() {
|
||
let sim = jaccard_similarity("天体物理学研究", "天体物理研究");
|
||
assert!(sim > 0.5, "中文 bigram 应能正确匹配，实际 {}", sim);
|
||
}
|
||
|
||
// ── 内容质量检查测试 ──
|
||
|
||
#[test]
|
||
fn test_quality_too_short() {
|
||
assert_eq!(check_content_quality("太短"), QualityCheck::TooShort(2));
|
||
// 刚好 10 个中文字符（可通过最低长度）
|
||
let ten = "一二三四五六七八九十";
|
||
assert_eq!(char_count(ten), 10);
|
||
assert_eq!(check_content_quality(ten), QualityCheck::Accept);
|
||
}
|
||
|
||
fn char_count(s: &str) -> usize {
|
||
s.chars().count()
|
||
}
|
||
|
||
#[test]
|
||
fn test_quality_transient_state() {
|
||
assert_eq!(
|
||
check_content_quality("用户正在调试登录模块的问题"),
|
||
QualityCheck::TransientState
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_quality_vague_language() {
|
||
assert_eq!(
|
||
check_content_quality("可能需要在后续版本中优化"),
|
||
QualityCheck::VagueLanguage("可能".to_string())
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_quality_code_pattern() {
|
||
assert_eq!(
|
||
check_content_quality("fn main() { println!(\"hello\"); }"),
|
||
QualityCheck::CodePattern
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_quality_accept_good_content() {
|
||
assert_eq!(
|
||
check_content_quality(
|
||
"用户是天体物理学家，主要研究星系演化。偏好使用 Kim 的径向速度拟合方法。"
|
||
),
|
||
QualityCheck::Accept
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_find_duplicate_by_content_detects_high_overlap() {
|
||
let base = "用户偏好使用 Rust 开发后端服务";
|
||
let entries = vec![make_entry("memory-a", "A", base)];
|
||
let dup = find_duplicate_by_content("用户偏好使用 Rust 开发后端系统", &entries, 0.40);
|
||
assert!(dup.is_some(), "高重叠内容应检测为重复");
|
||
}
|
||
|
||
#[test]
|
||
fn test_find_duplicate_rejects_low_overlap() {
|
||
let entries = vec![make_entry("a", "A", "用户偏好使用 Rust 开发后端")];
|
||
let dup = find_duplicate_by_content("天体物理学中星系演化研究的最新进展", &entries, 0.40);
|
||
assert!(dup.is_none(), "低重叠内容不应检测为重复");
|
||
}
|
||
|
||
#[test]
|
||
fn test_find_duplicate_skips_historical() {
|
||
let entries = vec![MemoryEntry {
|
||
slug: "historical-one".to_string(),
|
||
name: "历史记忆".to_string(),
|
||
description: "已过时".to_string(),
|
||
memory_type: super::super::types::MemoryType::User,
|
||
mtime: 1000,
|
||
content: "用户偏好使用 Rust 开发后端".to_string(),
|
||
path: PathBuf::from("historical-one.md"),
|
||
status: super::super::types::MemoryStatus::Historical {
|
||
superseded_by: Some("new-one".to_string()),
|
||
},
|
||
}];
|
||
// historical 应被跳过，不匹配
|
||
assert!(find_duplicate_by_content("用户偏好使用 Rust 开发后端", &entries, 0.6,).is_none());
|
||
}
|
||
}
|