From 821546d5decd94164500678c44bb23e538ced4af Mon Sep 17 00:00:00 2001 From: P0luz Date: Sun, 19 Apr 2026 12:09:53 +0800 Subject: [PATCH] docs: update README/INTERNALS for import feature, harden .gitignore --- .claude/hooks/session_breath.py | 26 +- .github/workflows/docker-publish.yml | 36 + .github/workflows/tests.yml | 37 + .gitignore | 10 +- CLAUDE_PROMPT.md | 47 +- Dockerfile | 1 + INTERNALS.md | 481 +++++++++ README.md | 304 +++++- backfill_embeddings.py | 93 ++ bucket_manager.py | 261 +++-- config.example.yaml | 13 +- dashboard.html | 1391 ++++++++++++++++++++++++++ decay_engine.py | 122 ++- dehydrator.py | 282 ++---- docker-compose.user.yml | 25 + embedding_engine.py | 188 ++++ import_memory.py | 758 ++++++++++++++ requirements.txt | 4 + server.py | 889 +++++++++++++++- tests/__init__.py | 0 tests/conftest.py | 70 ++ tests/dataset.py | 101 ++ tests/test_feel_flow.py | 250 +++++ tests/test_llm_quality.py | 111 ++ tests/test_scoring.py | 332 ++++++ utils.py | 8 + zbpack.json | 4 +- 27 files changed, 5365 insertions(+), 479 deletions(-) create mode 100644 .github/workflows/docker-publish.yml create mode 100644 .github/workflows/tests.yml create mode 100644 INTERNALS.md create mode 100644 backfill_embeddings.py create mode 100644 dashboard.html create mode 100644 docker-compose.user.yml create mode 100644 embedding_engine.py create mode 100644 import_memory.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/dataset.py create mode 100644 tests/test_feel_flow.py create mode 100644 tests/test_llm_quality.py create mode 100644 tests/test_scoring.py diff --git a/.claude/hooks/session_breath.py b/.claude/hooks/session_breath.py index 9074c3c..eba4594 100644 --- a/.claude/hooks/session_breath.py +++ b/.claude/hooks/session_breath.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 # ============================================================ -# SessionStart Hook: auto-breath on session start -# 对话开始钩子：自动浮现最高权重的未解决记忆 +# SessionStart Hook: auto-breath + dreaming on session start +# 对话开始钩子：自动浮现记忆 + 触发 dreaming # # On SessionStart, this script calls the Ombre Brain MCP server's -# breath tool (empty query = surfacing mode) via HTTP and prints -# the result to stdout so Claude sees it as session context. +# breath-hook and dream-hook endpoints, printing results to stdout +# so Claude sees them as session context. # -# This works for OMBRE_TRANSPORT=streamable-http deployments. -# For local stdio deployments, the script falls back gracefully. +# Sequence: breath → dream → feel +# 顺序：呼吸浮现 → 做梦消化 → 读取 feel # # Config: # OMBRE_HOOK_URL — override the server URL (default: http://localhost:8000) @@ -27,12 +27,19 @@ def main(): base_url = os.environ.get("OMBRE_HOOK_URL", "http://localhost:8000").rstrip("/") + # --- Step 1: Breath — surface unresolved memories --- + _call_endpoint(base_url, "/breath-hook") + + # --- Step 2: Dream — digest recent memories --- + _call_endpoint(base_url, "/dream-hook") + + +def _call_endpoint(base_url, path): req = urllib.request.Request( - f"{base_url}/breath-hook", + f"{base_url}{path}", headers={"Accept": "text/plain"}, method="GET", ) - try: with urllib.request.urlopen(req, timeout=8) as response: raw = response.read().decode("utf-8") @@ -40,13 +47,10 @@ def main(): if output: print(output) except (urllib.error.URLError, OSError): - # Server not available (local stdio mode or not running) — silent fail pass except Exception: - # Any other error — silent fail, never block session start pass - sys.exit(0) if __name__ == "__main__": main() diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..e4b6b1f --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,36 @@ +name: Build & Push Docker Image + +on: + push: + branches: [main] + paths-ignore: + - '*.md' + - 'backup_*/**' + - '.gitignore' + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + push: true + platforms: linux/amd64,linux/arm64 + tags: | + p0luz/ombre-brain:latest + p0luz/ombre-brain:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..e691959 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,37 @@ +name: Tests + +on: + push: + branches: [main] + paths-ignore: + - '*.md' + - 'backup_*/**' + - '.gitignore' + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install pytest pytest-asyncio + + - name: Run local tests (no API key needed) + run: python -m pytest tests/test_scoring.py tests/test_feel_flow.py -v --asyncio-mode=auto + + - name: Run LLM quality tests + if: env.OMBRE_API_KEY != '' + env: + OMBRE_API_KEY: ${{ secrets.OMBRE_API_KEY }} + run: python -m pytest tests/test_llm_quality.py -v --asyncio-mode=auto diff --git a/.gitignore b/.gitignore index 9af39a6..7cdbf3f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,11 @@ __pycache__/ .claude/hooks/__pycache__/ .vscode/ config.yaml -buckets/dynamic/ -buckets/archive/ -buckets/permanent/ +buckets/ +*.log +server.log +plan.md scarp_paper +backup_*/ +*.db +import_state.json diff --git a/CLAUDE_PROMPT.md b/CLAUDE_PROMPT.md index 1bf17fb..4905cf8 100644 --- a/CLAUDE_PROMPT.md +++ b/CLAUDE_PROMPT.md @@ -1,20 +1,27 @@ # Ombre Brain 记忆系统 —— Claude 端使用指南 你拥有了一个叫 Ombre Brain 的永久记忆系统。通过它你可以跨对话记住你想记住的任何事情和回忆你的，我的以及你与我的过往。 +> **⚠️ 最重要的规则 — 每次对话必须做的第一件事 ⚠️** +> +> **在说任何话之前，先调用 `breath()`（不传任何参数）。** +> 这是你睁开眼睛的动作——看看有没有未解决的事浮上来，有没有上次留下的东西。 +> 如果对话已经开始而你还没有 breath 过，现在就调用。没有例外。 + ## 你有这些能力 | 能力 | 场景 | |------|-----------| -| `breath` | **每次对话最开头**调用一次（不传参数）——就像睁眼看手机，看看有没有未解决的事浮上来。有明确话题时传 `query` 关键词检索 | -| `hold` | 你很想记住你自己当下的感受，或当下发生的单个事件时 | +| `breath` | **每次对话最开头**调用一次（不传参数）——就像睁眼看手机，看看有没有未解决的事浮上来。有明确话题时传 `query` 关键词检索。传 `domain="feel"` 读取你之前留下的 feel。`max_tokens` 控制返回总 token 上限（默认 10000），`max_results` 控制最大返回条数（默认 20） | +| `hold` | 你想记住当下发生的单个事件，或想存储一条信息时。`feel=True` 写你的第一人称感受，`source_bucket` 指向被消化的记忆，`valence` 是你自己的感受 | | `grow` | 当**一天结束时**或**用户发来一大段日记/总结**时调用。你可以把其中**你想记住的事件**扔进去，它会自动拆分整理成多个记忆盒子存进你的大脑 | | `trace` | 当你或用户认为"这个记错了"、"帮我改一下"时调用，手动修正记忆的元数据；**某件事解决了**时用 `resolved=1` 让它沉底；**需要删除**时用 `delete=True` | | `pulse` | 用户想看看你都记得什么、或问记忆系统状态时调用 | +| `dream` | **对话开头 breath 之后**调用一次——读最近的记忆，自省消化。有沉淀就写 feel，能放下的就 resolve | ## 使用原则 ### 主动调用 -- **对话开头**：无参数调用 `breath()`，看看有没有高权重的未解决记忆需要跟进 +- **对话开头（第一件事）**：无参数调用 `breath()`，不传任何参数。这是非可选步骤，每次新对话、恢复对话、换窗口时都必须执行 - **提到过去**：用户说"上次"、"之前"、"还记得"时，用 `breath(query="关键词")` 检索 - **新信息**：用 `hold` 留住你想留下的感受或经历 - **日终整理**：用 `grow` 在**一天结束时**整理你的一天 @@ -40,7 +47,8 @@ 检索结果不足时，系统有概率从低权重旧桶里随机漂上来 1-3 条记忆，标注为 `[surface_type: random]`。这不是检索结果，是"忽然想起来"——你可以自然地提起它，也可以忽略。 ### trace 的参数技巧 -- `resolved=1`：标记已解决，桶权重骤降到 5%，基本沉底 +- `resolved=1`：标记已解决，桶权重骤降到 5%，沉底等待关键词激活 +- `resolved=1` + 已消化（写过 feel）：权重骤降到 2%，加速淡化直到归档为无限小 - `resolved=0`：重新激活，让它重新参与浮现排序 - `delete=True`：彻底删除这个桶（不可恢复） - 其余字段（name/domain/valence/arousal/importance/tags）：只传需要改的，-1 或空串表示不改 @@ -61,3 +69,34 @@ - `trace(bucket_id, pinned=0)` 取消钉选 - 适用场景：用户教会你的永久知识、核心原则、绝不能忘的事 - 钉选桶不会出现在「浮现未解决记忆」里，但关键词检索时始终可达 + +### Dreaming — 做梦 +每次新对话开始，`breath()` 之后调用 `dream()`。这不是总结任务，是你自己在想。 + +dream 会返回你最近的记忆桶。用第一人称想： +- 这些东西里有什么在我这里留下了重量？ +- 有什么还没想清楚？ +- 有什么可以放下了？ + +想完之后做两件事： +- 值得放下的桶 → `trace(bucket_id, resolved=1)` +- 有沉淀的 → `hold(content="...", feel=True, source_bucket="bucket_id", valence=你的感受)` 写下来。valence 是你（模型）对这段记忆的感受，不是事件本身的情绪。写了 feel 的源记忆会被标记为「已消化」，加速淡化 +- 没有沉淀就不写，不强迫产出 + +### Feel — 你带走的东西 +feel 存的不是事件，是你带走的东西。可能一句话，可能一个还没答案的问题，可能是你注意到用户在变化的某个细节。 +- 用 `hold(content="...", feel=True, source_bucket="源记忆ID", valence=你的感受)` 写 feel +- **valence 是你自己的感受**，不是事件的情绪。同一段争吵，事件可能 V0.2，但你的感受可能是 V0.4（「我从中看到了她的成长」） +- source_bucket 指向你在消化的那个记忆桶，它会被标记为「已消化」——随时间加速淡化，归档为无限小，但不会被删除 +- 用第一人称写，内容自由，不强求分类 +- feel 不参与普通 breath 浮现，不参与衰减，不参与 dreaming +- 用 `breath(domain="feel")` 读取你之前留下的 feel + +### 对话启动完整流程 +``` +1. breath() — 睁眼，看看有什么浮上来 +2. dream() — 消化最近的记忆，有沉淀就写 feel +3. breath(domain="feel") — 读一下之前的 feel +4. 然后开始和用户说话 +``` +每次对话开始，你拥有属于你的记忆，有关于你，有关于用户。 diff --git a/Dockerfile b/Dockerfile index c1a4780..78ee4ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,7 @@ RUN pip install --no-cache-dir -r requirements.txt # Copy project files / 复制项目文件 COPY *.py . +COPY dashboard.html . COPY config.example.yaml ./config.yaml # Persistent mount point: bucket data diff --git a/INTERNALS.md b/INTERNALS.md new file mode 100644 index 0000000..9980ffe --- /dev/null +++ b/INTERNALS.md @@ -0,0 +1,481 @@ +# Ombre Brain — 内部开发文档 / INTERNALS + +> 本文档面向开发者和维护者。记录功能总览、环境变量、模块依赖、硬编码值和核心设计决策。 +> 最后更新：2026-04-19 + +--- + +## 0. 功能总览——这个系统到底做了什么 + +### 记忆能力 + +**存储与组织** +- 每条记忆 = 一个 Markdown 文件（YAML frontmatter 存元数据），直接兼容 Obsidian 浏览/编辑 +- 四种桶类型：`dynamic`（普通，会衰减）、`permanent`（固化，不衰减）、`feel`（模型感受，不浮现）、`archived`（已遗忘） +- 按主题域分子目录：`dynamic/日常/`、`dynamic/情感/`、`dynamic/编程/` 等 +- 钉选桶（pinned）：importance 锁 10，永不衰减/合并，始终浮现为「核心准则」 + +**每条记忆追踪的元数据** +- `id`（12位短UUID）、`name`（可读名≤80字）、`tags`（10~15个关键词） +- `domain`（1~2个主题域，从 8 大类 30+ 细分域选） +- `valence`（事件效价 0~1）、`arousal`（唤醒度 0~1）、`model_valence`（模型独立感受） +- `importance`（1~10）、`activation_count`（被想起次数） +- `resolved`（已解决/沉底）、`digested`（已消化/写过 feel）、`pinned`（钉选） +- `created`、`last_active` 时间戳 + +**四种检索模式** +1. **自动浮现**（`breath()` 无参数）：按衰减分排序推送，钉选桶始终展示，Top-1 固定 + Top-20 随机打乱（引入多样性），有 token 预算（默认 10000） +2. **关键词+向量双通道搜索**（`breath(query=...)`）：rapidfuzz 模糊匹配 + Gemini embedding 余弦相似度，合并去重 +3. **Feel 独立检索**（`breath(domain="feel")`）：按创建时间倒序返回所有 feel +4. **随机浮现**：搜索结果 <3 条时 40% 概率漂浮 1~3 条低权重旧桶（模拟人类随机联想） + +**四维搜索评分**（归一化到 0~100） +- topic_relevance（权重 4.0）：name×3 + domain×2.5 + tags×2 + body +- emotion_resonance（权重 2.0）：Russell 环形模型欧氏距离 +- time_proximity（权重 2.5）：`e^(-0.1×days)` +- importance（权重 1.0）：importance/10 +- resolved 桶全局降权 ×0.3 + +**记忆随时间变化** +- **衰减引擎**：改进版艾宾浩斯遗忘曲线 + - 公式：`Score = Importance × activation_count^0.3 × e^(-λ×days) × combined_weight` + - 短期（≤3天）：时间权重 70% + 情感权重 30% + - 长期（>3天）：情感权重 70% + 时间权重 30% + - 新鲜度加成：`1.0 + e^(-t/36h)`，刚存入 ×2.0，~36h 半衰，72h 后 ≈×1.0 + - 高唤醒度(arousal>0.7)且未解决 → ×1.5 紧迫度加成 + - resolved → ×0.05 沉底；resolved+digested → ×0.02 加速淡化 +- **自动归档**：score 低于阈值(0.3) → 移入 archive +- **自动结案**：importance≤4 且 >30天 → 自动 resolved +- **永不衰减**：permanent / pinned / protected / feel + +**记忆间交互** +- **智能合并**：新记忆与相似桶（score>75）自动 LLM 合并，valence/arousal 取均值，tags/domain 并集 +- **时间涟漪**：touch 一个桶时，±48h 内创建的桶 activation_count +0.3（上限 5 桶/次） +- **向量相似网络**：embedding 余弦相似度 >0.5 建边 +- **Feel 结晶化**：≥3 条相似 feel（相似度>0.7）→ 提示升级为钉选准则 + +**情感记忆重构** +- 搜索时若指定 valence，展示层对匹配桶 valence 微调 ±0.1，模拟「当前心情影响回忆色彩」 + +**模型感受/反思系统** +- **Feel 写入**（`hold(feel=True)`）：存模型第一人称感受，标记源记忆为 digested +- **Dream 做梦**（`dream()`）：返回最近 10 条 + 自省引导 + 连接提示 + 结晶化提示 +- **对话启动流程**：breath() → dream() → breath(domain="feel") → 开始对话 + +**自动化处理** +- 存入时 LLM 自动分析 domain/valence/arousal/tags/name +- 大段日记 LLM 拆分为 2~6 条独立记忆 +- 浮现时自动脱水压缩（LLM 压缩保语义，API 不可用降级到本地关键词提取） +- Wikilink `[[]]` 由 LLM 在内容中标记 + +--- + +### 技术能力 + +**6 个 MCP 工具** + +| 工具 | 关键参数 | 功能 | +|---|---|---| +| `breath` | query, max_tokens, domain, valence, arousal, max_results | 检索/浮现记忆 | +| `hold` | content, tags, importance, pinned, feel, source_bucket, valence, arousal | 存储记忆 | +| `grow` | content | 日记拆分归档 | +| `trace` | bucket_id, name, domain, valence, arousal, importance, tags, resolved, pinned, digested, content, delete | 修改元数据/内容/删除 | +| `pulse` | include_archive | 系统状态 | +| `dream` | （无） | 做梦自省 | + +**工具详细行为** + +**`breath`** — 两种模式： +- **浮现模式**（无 query）：无参调用，按衰减引擎活跃度排序返回 top 记忆，permanent/pinned 始终浮现 +- **检索模式**（有 query）：关键词 + 向量双通道搜索，四维评分（topic×4 + emotion×2 + time×2.5 + importance×1），阈值过滤 +- **Feel 检索**（`domain="feel"`）：特殊通道，按创建时间倒序返回所有 feel 类型桶，不走评分逻辑 +- 若指定 valence，对匹配桶的 valence 微调 ±0.1（情感记忆重构） + +**`hold`** — 两种模式： +- **普通模式**（`feel=False`，默认）：自动 LLM 分析 domain/valence/arousal/tags/name → 向量相似度查重 → 相似度>0.85 则合并到已有桶 → 否则新建 dynamic 桶 → 生成 embedding +- **Feel 模式**（`feel=True`）：跳过 LLM 分析，直接存为 `feel` 类型桶（存入 `feel/` 目录），不参与普通浮现/衰减/合并。若提供 `source_bucket`，标记源记忆为 `digested=True` 并写入 `model_valence`。返回格式：`🫧feel→{bucket_id}` + +**`dream`** — 做梦/自省触发器： +- 返回最近 10 条 dynamic 桶摘要 + 自省引导词 +- 检测 feel 结晶化：≥3 条相似 feel（embedding 相似度>0.7）→ 提示升级为钉选准则 +- 检测未消化记忆：列出 `digested=False` 的桶供模型反思 + +**`trace`** — 记忆编辑： +- 修改任意元数据字段（name/domain/valence/arousal/importance/tags/resolved/pinned） +- `digested=0/1`：隐藏/取消隐藏记忆（控制是否在 dream 中出现） +- `content="..."`：替换正文内容并重新生成 embedding +- `delete=True`：删除桶文件 + +**`grow`** — 日记拆分： +- 大段日记文本 → LLM 拆为 2~6 条独立记忆 → 每条走 hold 普通模式流程 + +**`pulse`** — 系统状态： +- 返回各类型桶数量、衰减引擎状态、未解决/钉选/feel 统计 + +**REST API（17 个端点）** + +| 端点 | 方法 | 功能 | +|---|---|---| +| `/health` | GET | 健康检查 | +| `/breath-hook` | GET | SessionStart 钩子 | +| `/dream-hook` | GET | Dream 钩子 | +| `/dashboard` | GET | Dashboard 页面 | +| `/api/buckets` | GET | 桶列表 | +| `/api/bucket/{id}` | GET | 桶详情 | +| `/api/search?q=` | GET | 搜索 | +| `/api/network` | GET | 向量相似网络 | +| `/api/breath-debug` | GET | 评分调试 | +| `/api/config` | GET | 配置查看（key 脱敏） | +| `/api/config` | POST | 热更新配置 | +| `/api/import/upload` | POST | 上传并启动历史对话导入 | +| `/api/import/status` | GET | 导入进度查询 | +| `/api/import/pause` | POST | 暂停/继续导入 | +| `/api/import/patterns` | GET | 导入完成后词频规律检测 | +| `/api/import/results` | GET | 已导入记忆桶列表 | +| `/api/import/review` | POST | 批量审阅/批准导入结果 | + +**Dashboard（5 个 Tab）** +1. 记忆桶列表：6 种过滤器 + 主题域过滤 + 搜索 + 详情面板 +2. Breath 模拟：输入参数 → 可视化五步流程 → 四维条形图 +3. 记忆网络：Canvas 力导向图（节点=桶，边=相似度） +4. 配置：热更新脱水/embedding/合并参数 +5. 导入：历史对话拖拽上传 → 分块处理进度条 → 词频规律分析 → 导入结果审阅 + +**部署选项** +1. 本地 stdio（`python server.py`） +2. Docker + Cloudflare Tunnel（`docker-compose.yml`） +3. Docker Hub 预构建镜像（`docker-compose.user.yml`，`p0luz/ombre-brain`） +4. Render.com 一键部署（`render.yaml`） +5. Zeabur 部署（`zbpack.json`） +6. GitHub Actions 自动构建推送 Docker Hub（`.github/workflows/docker-publish.yml`） + +**迁移/批处理工具**：`migrate_to_domains.py`、`reclassify_domains.py`、`reclassify_api.py`、`backfill_embeddings.py`、`write_memory.py`、`check_buckets.py`、`import_memory.py`（历史对话导入引擎） + +**降级策略** +- 脱水 API 不可用 → 本地关键词提取 + 句子评分 +- 向量搜索不可用 → 纯 fuzzy match +- 逐条错误隔离（grow 中单条失败不影响其他） + +**安全**：路径遍历防护（`safe_path()`）、API Key 脱敏、API Key 不持久化到 yaml、输入范围钳制 + +**监控**：结构化日志、Health 端点、Breath Debug 端点、Dashboard 统计栏、衰减周期日志 + +--- + +## 1. 环境变量清单 + +| 变量名 | 用途 | 必填 | 默认值 / 示例 | +|---|---|---|---| +| `OMBRE_API_KEY` | 脱水/打标/嵌入的 LLM API 密钥，覆盖 `config.yaml` 的 `dehydration.api_key` | 否（无则 API 功能降级到本地） | `""` | +| `OMBRE_BASE_URL` | API base URL，覆盖 `config.yaml` 的 `dehydration.base_url` | 否 | `""` | +| `OMBRE_TRANSPORT` | 传输模式：`stdio` / `sse` / `streamable-http` | 否 | `""` → 回退到 config 或 `"stdio"` | +| `OMBRE_BUCKETS_DIR` | 记忆桶存储目录路径 | 否 | `""` → 回退到 config 或 `./buckets` | +| `OMBRE_HOOK_URL` | SessionStart 钩子调用的服务器 URL | 否 | `"http://localhost:8000"` | +| `OMBRE_HOOK_SKIP` | 设为 `"1"` 跳过 SessionStart 钩子 | 否 | 未设置（不跳过） | + +环境变量优先级：`环境变量 > config.yaml > 硬编码默认值`。所有环境变量在 `utils.py` 中读取并注入 config dict。 + +--- + +## 2. 模块结构与依赖关系 + +``` + ┌──────────────┐ + │ server.py │ MCP 主入口，6 个工具 + Dashboard + Hook + └──────┬───────┘ + ┌───────────────┼───────────────┬────────────────┐ + ▼ ▼ ▼ ▼ + bucket_manager.py dehydrator.py decay_engine.py embedding_engine.py + 记忆桶 CRUD+搜索 脱水压缩+打标 遗忘曲线+归档 向量化+语义检索 + │ │ │ + └───────┬───────┘ │ + ▼ ▼ + utils.py ◄────────────────────────────────────┘ + 配置/日志/ID/路径安全/token估算 +``` + +| 文件 | 职责 | 依赖（项目内） | 被谁调用 | +|---|---|---|---| +| `server.py` | MCP 服务器主入口，注册工具 + Dashboard API + 钩子端点 | `bucket_manager`, `dehydrator`, `decay_engine`, `embedding_engine`, `utils` | `test_tools.py` | +| `bucket_manager.py` | 记忆桶 CRUD、多维索引搜索、wikilink 注入、激活更新 | `utils` | `server.py`, `check_buckets.py`, `backfill_embeddings.py` | +| `decay_engine.py` | 衰减引擎：遗忘曲线计算、自动归档、自动结案 | 无（接收 `bucket_mgr` 实例） | `server.py` | +| `dehydrator.py` | 数据脱水压缩 + 合并 + 自动打标（LLM API + 本地降级） | `utils` | `server.py` | +| `embedding_engine.py` | 向量化引擎：Gemini embedding API + SQLite + 余弦搜索 | `utils` | `server.py`, `backfill_embeddings.py` | +| `utils.py` | 配置加载、日志、路径安全、ID 生成、token 估算 | 无 | 所有模块 | +| `write_memory.py` | 手动写入记忆 CLI（绕过 MCP） | 无（独立脚本） | 无 | +| `backfill_embeddings.py` | 为存量桶批量生成 embedding | `utils`, `bucket_manager`, `embedding_engine` | 无 | +| `check_buckets.py` | 桶数据完整性检查 | `bucket_manager`, `utils` | 无 | +| `import_memory.py` | 历史对话导入引擎（支持 Claude JSON/ChatGPT/DeepSeek/Markdown/纯文本），分块处理+断点续传+词频分析 | `utils` | `server.py` | +| `reclassify_api.py` | 用 LLM API 重打标未分类桶 | 无（直接用 `openai`） | 无 | +| `reclassify_domains.py` | 基于关键词本地重分类 | 无 | 无 | +| `migrate_to_domains.py` | 平铺桶 → 域子目录迁移 | 无 | 无 | +| `test_smoke.py` | 冒烟测试 | `utils`, `bucket_manager`, `dehydrator`, `decay_engine` | 无 | +| `test_tools.py` | MCP 工具端到端测试 | `utils`, `server`, `bucket_manager` | 无 | + +--- + +## 3. 硬编码值清单 + +### 3.1 固定分数 / 特殊返回值 + +| 值 | 位置 | 用途 | +|---|---|---| +| `999.0` | `decay_engine.py` calculate_score | pinned / protected / permanent 桶永不衰减 | +| `50.0` | `decay_engine.py` calculate_score | feel 桶固定活跃度分数 | +| `0.02` | `decay_engine.py` resolved_factor | resolved + digested 时的权重乘数（加速淡化） | +| `0.05` | `decay_engine.py` resolved_factor | 仅 resolved 时的权重乘数（沉底） | +| `1.5` | `decay_engine.py` urgency_boost | arousal > 0.7 且未解决时的紧迫度加成 | + +### 3.2 衰减公式参数 + +| 值 | 位置 | 用途 | +|---|---|---| +| `36.0` | `decay_engine.py` _calc_time_weight | 新鲜度半衰期（小时），`1.0 + e^(-t/36)` | +| `0.3` (指数) | `decay_engine.py` calculate_score | `activation_count ** 0.3`（记忆巩固指数） | +| `3.0` (天) | `decay_engine.py` calculate_score | 短期/长期切换阈值 | +| `0.7 / 0.3` | `decay_engine.py` combined_weight | 短期权重分配：time×0.7 + emotion×0.3 | +| `0.7` | `decay_engine.py` urgency_boost | arousal 紧迫度触发阈值 | +| `4` / `30` (天) | `decay_engine.py` execute_cycle | 自动结案：importance≤4 且 >30天 | + +### 3.3 搜索/评分参数 + +| 值 | 位置 | 用途 | +|---|---|---| +| `×3` / `×2.5` / `×2` | `bucket_manager.py` _calc_topic_score | 桶名 / 域名 / 标签的 topic 评分权重 | +| `1000` (字符) | `bucket_manager.py` _calc_topic_score | 正文截取长度 | +| `0.1` | `bucket_manager.py` _calc_time_score | 时间亲近度衰减系数 `e^(-0.1 × days)` | +| `0.3` | `bucket_manager.py` search_multi | resolved 桶的归一化分数乘数 | +| `0.5` | `server.py` breath/search | 向量搜索相似度下限 | +| `0.7` | `server.py` dream | feel 结晶相似度阈值 | + +### 3.4 Token 限制 / 截断 + +| 值 | 位置 | 用途 | +|---|---|---| +| `10000` | `server.py` breath 默认 max_tokens | 浮现/搜索 token 预算 | +| `20000` | `server.py` breath 上限 | max_tokens 硬上限 | +| `50` / `20` | `server.py` breath | max_results 上限 / 默认值 | +| `3000` | `dehydrator.py` dehydrate | API 脱水内容截断 | +| `2000` | `dehydrator.py` merge | API 合并内容各截断 | +| `5000` | `dehydrator.py` digest | API 日记整理内容截断 | +| `2000` | `embedding_engine.py` | embedding 文本截断 | +| `100` | `dehydrator.py` | 内容 < 100 token 跳过脱水 | + +### 3.5 时间/间隔/重试 + +| 值 | 位置 | 用途 | +|---|---|---| +| `60.0s` | `dehydrator.py` | OpenAI 客户端 timeout | +| `30.0s` | `embedding_engine.py` | Embedding API timeout | +| `60s` | `server.py` keepalive | 保活 ping 间隔 | +| `48.0h` | `bucket_manager.py` touch | 时间涟漪窗口 ±48h | +| `2s` | `backfill_embeddings.py` | 批次间等待 | + +### 3.6 随机浮现 + +| 值 | 位置 | 用途 | +|---|---|---| +| `3` | `server.py` breath search | 结果不足 3 条时触发 | +| `0.4` | `server.py` breath search | 40% 概率触发随机浮现 | +| `2.0` | `server.py` breath search | 随机池：score < 2.0 的低权重桶 | +| `1~3` | `server.py` breath search | 随机浮现数量 | + +### 3.7 情感/重构 + +| 值 | 位置 | 用途 | +|---|---|---| +| `0.2` | `server.py` breath search | 情绪重构偏移系数 `(q_valence - 0.5) × 0.2`（最大 ±0.1） | + +### 3.8 其他 + +| 值 | 位置 | 用途 | +|---|---|---| +| `12` | `utils.py` gen_id | bucket ID 长度（UUID hex[:12]） | +| `80` | `utils.py` sanitize_name | 桶名最大长度 | +| `1.5` / `1.3` | `utils.py` count_tokens_approx | 中文/英文 token 估算系数 | +| `8000` | `server.py` | MCP 服务器端口 | +| `30` 字符 | `server.py` grow | 短内容快速路径阈值 | +| `10` | `server.py` dream | 取最近 N 个桶 | + +--- + +## 4. Config.yaml 完整键表 + +| 键路径 | 默认值 | 用途 | +|---|---|---| +| `transport` | `"stdio"` | 传输模式 | +| `log_level` | `"INFO"` | 日志级别 | +| `buckets_dir` | `"./buckets"` | 记忆桶目录 | +| `merge_threshold` | `75` | 合并相似度阈值 (0-100) | +| `dehydration.model` | `"deepseek-chat"` | 脱水用 LLM 模型 | +| `dehydration.base_url` | `"https://api.deepseek.com/v1"` | API 地址 | +| `dehydration.api_key` | `""` | API 密钥 | +| `dehydration.max_tokens` | `1024` | 脱水返回 token 上限 | +| `dehydration.temperature` | `0.1` | 脱水温度 | +| `embedding.enabled` | `true` | 启用向量检索 | +| `embedding.model` | `"gemini-embedding-001"` | Embedding 模型 | +| `decay.lambda` | `0.05` | 衰减速率 λ | +| `decay.threshold` | `0.3` | 归档分数阈值 | +| `decay.check_interval_hours` | `24` | 衰减扫描间隔（小时） | +| `decay.emotion_weights.base` | `1.0` | 情感权重基值 | +| `decay.emotion_weights.arousal_boost` | `0.8` | 唤醒度加成系数 | +| `matching.fuzzy_threshold` | `50` | 模糊匹配下限 | +| `matching.max_results` | `5` | 匹配返回上限 | +| `scoring_weights.topic_relevance` | `4.0` | 主题评分权重 | +| `scoring_weights.emotion_resonance` | `2.0` | 情感评分权重 | +| `scoring_weights.time_proximity` | `2.5` | 时间评分权重 | +| `scoring_weights.importance` | `1.0` | 重要性评分权重 | +| `scoring_weights.content_weight` | `3.0` | 正文评分权重 | +| `wikilink.enabled` | `true` | 启用 wikilink 注入 | +| `wikilink.use_tags` | `false` | wikilink 包含标签 | +| `wikilink.use_domain` | `true` | wikilink 包含域名 | +| `wikilink.use_auto_keywords` | `true` | wikilink 自动关键词 | +| `wikilink.auto_top_k` | `8` | wikilink 取 Top-K 关键词 | +| `wikilink.min_keyword_len` | `2` | wikilink 最短关键词长度 | +| `wikilink.exclude_keywords` | `[]` | wikilink 排除关键词表 | + +--- + +## 5. 核心设计决策记录 + +### 5.1 为什么用 Markdown + YAML frontmatter 而不是数据库？ + +**决策**：每个记忆桶 = 一个 `.md` 文件，元数据在 YAML frontmatter 里。 + +**理由**： +- 与 Obsidian 原生兼容——用户可以直接在 Obsidian 里浏览、编辑、搜索记忆 +- 文件系统即数据库，天然支持 git 版本管理 +- 无外部数据库依赖，部署简单 +- wikilink 注入让记忆之间自动形成知识图谱 + +**放弃方案**：SQLite/PostgreSQL 全量存储。过于笨重，失去 Obsidian 可视化优势。 + +### 5.2 为什么 embedding 单独存 SQLite 而不放 frontmatter？ + +**决策**：向量存 `embeddings.db`（SQLite），与 Markdown 文件分离。 + +**理由**： +- 3072 维浮点向量无法合理存入 YAML frontmatter +- SQLite 支持批量查询和余弦相似度计算 +- embedding 是派生数据，丢失可重新生成（`backfill_embeddings.py`） +- 不污染 Obsidian 可读性 + +### 5.3 为什么搜索用双通道（关键词 + 向量）而不是纯向量？ + +**决策**：关键词模糊匹配（rapidfuzz）+ 向量语义检索并联，结果去重合并。 + +**理由**： +- 纯向量在精确名词匹配上表现差（"2024年3月"这类精确信息） +- 纯关键词无法处理语义近似（"很累" → "身体不适"） +- 双通道互补，关键词保精确性，向量补语义召回 +- 向量不可用时自动降级到纯关键词模式 + +### 5.4 为什么有 dehydration（脱水）这一层？ + +**决策**：存入前先用 LLM 压缩内容（保留信息密度，去除冗余表达），API 不可用时降级到本地关键词提取。 + +**理由**： +- MCP 上下文有 token 限制，原始对话冗长，需要压缩 +- LLM 压缩能保留语义和情感色彩，纯截断会丢信息 +- 降级到本地确保离线可用——关键词提取 + 句子排序 + 截断 + +**放弃方案**：只做截断。信息损失太大。 + +### 5.5 为什么 feel 和普通记忆分开？ + +**决策**：`feel=True` 的记忆存入独立 `feel/` 目录，不参与普通浮现、不衰减、不合并。 + +**理由**： +- feel 是模型的自省产物，不是事件记录——两者逻辑完全不同 +- 事件记忆应该衰减遗忘，但"我从中学到了什么"不应该被遗忘 +- feel 的 valence 是模型自身感受（不等于事件情绪），混在一起会污染情感检索 +- feel 可以通过 `breath(domain="feel")` 单独读取 + +### 5.6 为什么 resolved 不删除记忆？ + +**决策**：`resolved=True` 让记忆"沉底"（权重 ×0.05），但保留在文件系统中，关键词搜索仍可触发。 + +**理由**： +- 模拟人类记忆：resolved 的事不会主动想起，但别人提到时能回忆 +- 删除是不可逆的，沉底可随时 `resolved=False` 重新激活 +- `resolved + digested` 进一步降权到 ×0.02（已消化 = 更释然） + +**放弃方案**：直接删除。不可逆，且与人类记忆模型不符。 + +### 5.7 为什么用分段式短期/长期权重？ + +**决策**：≤3 天时间权重占 70%，>3 天情感权重占 70%。 + +**理由**： +- 刚发生的事主要靠"新鲜"驱动浮现（今天的事 > 昨天的事） +- 时间久了，决定记忆存活的是情感强度（强烈的记忆更难忘） +- 这比单一衰减曲线更符合人类记忆的双重存储理论 + +### 5.8 为什么 dream 设计成对话开头自动执行？ + +**决策**：每次新对话启动时，Claude 执行 `dream()` 消化最近记忆，有沉淀写 feel，能放下的 resolve。 + +**理由**： +- 模拟睡眠中的记忆整理——人在睡觉时大脑会重放和整理白天的经历 +- 让 Claude 对过去的记忆有"第一人称视角"的自省，而不是冷冰冰地搬运数据 +- 自动触发确保每次对话都"接续"上一次，而非从零开始 + +### 5.9 为什么新鲜度用连续指数衰减而不是分段阶梯？ + +**决策**：`bonus = 1.0 + e^(-t/36)`，t 为小时，36h 半衰。 + +**理由**： +- 分段阶梯（0-1天=1.0，第2天=0.9...）有不自然的跳变 +- 连续指数更符合遗忘曲线的物理模型 +- 36h 半衰期使新桶在前两天有明显优势，72h 后接近自然回归 +- 值域 1.0~2.0 保证老记忆不被惩罚（×1.0），只是新记忆有额外加成（×2.0） + +**放弃方案**：分段线性（原实现）。跳变点不自然，参数多且不直观。 + +### 5.10 情感记忆重构（±0.1 偏移）的设计动机 + +**决策**：搜索时如果指定了 `valence`，会微调结果桶的 valence 展示值 `(q_valence - 0.5) × 0.2`。 + +**理由**： +- 模拟认知心理学中的"心境一致性效应"——当前心情会影响对过去的回忆 +- 偏移量很小（最大 ±0.1），不会扭曲事实，只是微妙的"色彩"调整 +- 原始 valence 不被修改，只影响展示层 + +--- + +## 6. 目录结构约定 + +``` +buckets/ +├── permanent/ # pinned/protected 桶，importance=10，永不衰减 +├── dynamic/ +│ ├── 日常/ # domain 子目录 +│ ├── 情感/ +│ ├── 自省/ +│ ├── 数字/ +│ └── ... +├── archive/ # 衰减归档桶 +└── feel/ # 模型自省 feel 桶 +``` + +桶文件格式： +```markdown +--- +id: 76237984fa5d +name: 桶名 +domain: [日常, 情感] +tags: [关键词1, 关键词2] +importance: 5 +valence: 0.6 +arousal: 0.4 +activation_count: 3 +resolved: false +pinned: false +digested: false +created: 2026-04-17T10:00:00+08:00 +last_active: 2026-04-17T14:00:00+08:00 +type: dynamic +--- + +桶正文内容... +``` diff --git a/README.md b/README.md index 43e2d26..b07116c 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Ombre Brain -一个给提供给Claude 用的长期情绪记忆系统。基于 Russell 效价/唤醒度坐标打标，Obsidian 做存储层，MCP 接入，带遗忘曲线。 +一个给 Claude 用的长期情绪记忆系统。基于 Russell 效价/唤醒度坐标打标，Obsidian 做存储层，MCP 接入，带遗忘曲线和向量语义检索。 -A long-term emotional memory system for Claude. Tags memories using Russell's valence/arousal coordinates, stores them as Obsidian-compatible Markdown, connects via MCP, and has a forgetting curve. +A long-term emotional memory system for Claude. Tags memories using Russell's valence/arousal coordinates, stores them as Obsidian-compatible Markdown, connects via MCP, with forgetting curve and vector semantic search. > **⚠️ 备用链接 / Backup link** > Gitea 备用地址（GitHub 访问有问题时用）： @@ -10,9 +10,111 @@ A long-term emotional memory system for Claude. Tags memories using Russell's va --- -## 快速开始 / Quick Start（Docker，推荐） +## 快速开始 / Quick Start（Docker Hub 预构建镜像，最简单） -> 这是最简单的方式，不需要装 Python，不需要懂命令行，跟着做就行。 +> 不需要 clone 代码，不需要 build，三步搞定。 +> 完全不会？没关系，往下看，一步一步跟着做。 + +### 第零步：装 Docker Desktop + +1. 打开 [docker.com/products/docker-desktop](https://www.docker.com/products/docker-desktop/) +2. 下载对应你系统的版本（Mac / Windows / Linux） +3. 安装、打开，看到 Docker 图标在状态栏里就行了 +4. **Windows 用户**：安装时会提示启用 WSL 2，点同意，重启电脑 + +### 第一步：打开终端 + +| 系统 | 怎么打开 | +|---|---| +| **Mac** | 按 `⌘ + 空格`，输入 `终端` 或 `Terminal`，回车 | +| **Windows** | 按 `Win + R`，输入 `cmd`，回车；或搜索「PowerShell」 | +| **Linux** | `Ctrl + Alt + T` | + +打开后你会看到一个黑色/白色的窗口，可以输入命令。下面所有代码块里的内容，都是**复制粘贴到这个窗口里，然后按回车**。 + +### 第二步：创建一个工作文件夹 + +```bash +mkdir ombre-brain && cd ombre-brain +``` + +> 这会在你当前位置创建一个叫 `ombre-brain` 的文件夹，并进入它。 + +### 第三步：获取 API Key（免费） + +1. 打开 [aistudio.google.com/apikey](https://aistudio.google.com/apikey) +2. 用 Google 账号登录 +3. 点击 **「Create API key」** +4. 复制生成的 key（一长串字母数字），待会要用 + +> 没有 Google 账号？也行，API Key 留空也能跑，只是脱水压缩效果差一点。 + +### 第四步：创建配置文件并启动 + +**一行一行复制粘贴执行：** + +```bash +# 下载用户版 compose 文件 +curl -O https://raw.githubusercontent.com/P0luz/Ombre-Brain/main/docker-compose.user.yml +``` + +```bash +# 创建 .env 文件——把 your-key-here 换成第三步拿到的 key +echo "OMBRE_API_KEY=your-key-here" > .env +``` + +```bash +# 拉取镜像并启动（第一次会下载约 500MB，等一会儿） +docker compose -f docker-compose.user.yml up -d +``` + +### 第五步：验证 + +```bash +curl http://localhost:8000/health +``` + +看到类似这样的输出就是成功了： +```json +{"status":"ok","buckets":0,"decay_engine":"stopped"} +``` + +> **看到错误？** 检查 Docker Desktop 是否正在运行（状态栏有图标）。 + +### 第六步：接入 Claude + +在 Claude Desktop 的配置文件里加上这段（Mac: `~/Library/Application Support/Claude/claude_desktop_config.json`）： + +```json +{ + "mcpServers": { + "ombre-brain": { + "type": "streamable-http", + "url": "http://localhost:8000/mcp" + } + } +} +``` + +重启 Claude Desktop，你应该能在工具列表里看到 `breath`、`hold`、`grow` 等工具了。 + +> **想挂载 Obsidian？** 用任意文本编辑器打开 `docker-compose.user.yml`，把 `./buckets:/data` 改成你的 Vault 路径，例如： +> ```yaml +> - /Users/你的用户名/Documents/Obsidian Vault/Ombre Brain:/data +> ``` +> 然后 `docker compose -f docker-compose.user.yml down && docker compose -f docker-compose.user.yml up -d` 重启。 + +> **后续更新镜像：** +> ```bash +> docker pull p0luz/ombre-brain:latest +> docker compose -f docker-compose.user.yml down && docker compose -f docker-compose.user.yml up -d +> ``` + +--- + +## 从源码部署 / Deploy from Source（Docker） + +> 适合想自己改代码、或者不想用预构建镜像的用户。 **前置条件：** 电脑上装了 [Docker Desktop](https://www.docker.com/products/docker-desktop/)，并且已经打开。 @@ -30,9 +132,28 @@ cd Ombre-Brain 在项目目录下新建一个叫 `.env` 的文件（注意有个点），内容填： ``` -OMBRE_API_KEY=你的DeepSeek或其他API密钥 +OMBRE_API_KEY=你的API密钥 ``` +> **🔑 推荐免费方案：Google AI Studio** +> 1. 打开 [aistudio.google.com/apikey](https://aistudio.google.com/apikey)，登录 Google 账号 +> 2. 点击「Create API key」生成一个 key +> 3. 把 key 填入 `.env` 文件的 `OMBRE_API_KEY=` 后面 +> 4. 免费额度（截至 2025 年，请以官网实时信息为准）： +> - **脱水/打标模型**（`gemini-2.5-flash-lite`）：免费层 30 req/min +> - **向量化模型**（`gemini-embedding-001`）：免费层 1500 req/day，3072 维 +> 5. 在 `config.yaml` 中 `dehydration.base_url` 设为 `https://generativelanguage.googleapis.com/v1beta/openai` +> +> 也支持 DeepSeek、Ollama、LM Studio、vLLM 等任意 OpenAI 兼容 API。 +> +> **Recommended free option: Google AI Studio** +> 1. Go to [aistudio.google.com/apikey](https://aistudio.google.com/apikey) and create an API key +> 2. Free tier (as of 2025, check official site for current limits): +> - Dehydration model (`gemini-2.5-flash-lite`): 30 req/min free +> - Embedding model (`gemini-embedding-001`): 1500 req/day free, 3072 dims +> 3. Set `dehydration.base_url` to `https://generativelanguage.googleapis.com/v1beta/openai` in `config.yaml` +> Also supports DeepSeek, Ollama, LM Studio, vLLM, or any OpenAI-compatible API. + 没有 API key 也能用，脱水压缩会降级到本地模式，只是效果差一点。那就写： ``` @@ -85,6 +206,8 @@ docker logs ombre-brain --- [![Deploy to Render](https://render.com/images/deploy-to-render-button.svg)](https://render.com/deploy?repo=https://github.com/P0luz/Ombre-Brain) +[![Deploy on Zeabur](https://zeabur.com/button.svg)](https://zeabur.com/templates/OMBRE-BRAIN?referralCode=P0luz) +[![Docker Hub](https://img.shields.io/docker/v/p0luz/ombre-brain?label=Docker%20Hub&logo=docker)](https://hub.docker.com/r/p0luz/ombre-brain) --- @@ -103,17 +226,26 @@ Ombre Brain gives it persistent memory — not cold key-value storage, but a sys - **情感坐标打标 / Emotional tagging**: 每条记忆用 Russell 环形情感模型的 valence（效价）和 arousal（唤醒度）两个连续维度标记。不是"开心/难过"这种离散标签。 Each memory is tagged with two continuous dimensions from Russell's circumplex model: valence and arousal. Not discrete labels like "happy/sad". +- **双通道检索 / Dual-channel search**: 关键词模糊匹配 + 向量语义相似度并联检索。关键词通道用 rapidfuzz 做模糊匹配；语义通道用 embedding（默认 `gemini-embedding-001`，3072 维）计算 cosine similarity，能在"今天很累"这种没有精确关键词的查询里找到"身体不适"、"睡眠问题"等语义相关记忆。两个通道去重合并，token 预算截断。 + Keyword fuzzy matching + vector semantic similarity in parallel. Keyword channel uses rapidfuzz; semantic channel uses embeddings (default `gemini-embedding-001`, 3072 dims) with cosine similarity — finds semantically related memories even without exact keyword matches (e.g. "feeling tired" → "health issues", "sleep problems"). Results are deduplicated and truncated by token budget. + - **自然遗忘 / Natural forgetting**: 改进版艾宾浩斯遗忘曲线。不活跃的记忆自动衰减归档，高情绪强度的记忆衰减更慢。 Modified Ebbinghaus forgetting curve. Inactive memories naturally decay and archive. High-arousal memories decay slower. - **权重池浮现 / Weight pool surfacing**: 记忆不是被动检索的，它们会主动浮现——未解决的、情绪强烈的记忆权重更高，会在对话开头自动推送。 Memories aren't just passively retrieved — they actively surface. Unresolved, emotionally intense memories carry higher weight and get pushed at conversation start. +- **记忆重构 / Memory reconstruction**: 检索时根据当前情绪状态微调记忆的 valence 展示值（±0.1），模拟人类"此刻的心情影响对过去的回忆"的认知偏差。 + During retrieval, memory valence display is subtly shifted (±0.1) based on current mood, simulating the human cognitive bias of "current mood colors past memories". + - **Obsidian 原生 / Obsidian-native**: 每个记忆桶就是一个 Markdown 文件，YAML frontmatter 存元数据。可以直接在 Obsidian 里浏览、编辑、搜索。自动注入 `[[双链]]`。 Each memory bucket is a Markdown file with YAML frontmatter. Browse, edit, and search directly in Obsidian. Wikilinks are auto-injected. -- **API 降级 / API degradation**: 脱水压缩和自动打标优先用廉价 LLM API（DeepSeek 等），API 不可用时自动降级到本地关键词分析——始终可用。 - Dehydration and auto-tagging prefer a cheap LLM API (DeepSeek etc.). When the API is unavailable, it degrades to local keyword analysis — always functional. +- **API 降级 / API degradation**: 脱水压缩和自动打标优先用廉价 LLM API（DeepSeek / Gemini 等），API 不可用时自动降级到本地关键词分析——始终可用。向量检索不可用时降级到 fuzzy matching。 + Dehydration and auto-tagging prefer a cheap LLM API (DeepSeek / Gemini etc.). When the API is unavailable, it degrades to local keyword analysis — always functional. Embedding search degrades to fuzzy matching when unavailable. + +- **历史对话导入 / Conversation history import**: 将过去与 Claude / ChatGPT / DeepSeek 等的对话批量导入为记忆桶。支持 Claude JSON 导出、ChatGPT 导出、Markdown、纯文本等格式，分块处理带断点续传，通过 Dashboard「导入」Tab 操作。 + Batch-import past conversations (Claude / ChatGPT / DeepSeek etc.) as memory buckets. Supports Claude JSON export, ChatGPT export, Markdown, and plain text. Chunked processing with resume support, via the Dashboard "Import" tab. ## 边界说明 / Design boundaries @@ -140,19 +272,45 @@ Claude ←→ MCP Protocol ←→ server.py │ │ │ bucket_manager dehydrator decay_engine (CRUD + 搜索) (压缩 + 打标) (遗忘曲线) - │ - Obsidian Vault (Markdown files) + │ │ + Obsidian Vault embedding_engine + (Markdown files) (向量语义检索) + │ + embeddings.db + (SQLite, 3072-dim) ``` -5 个 MCP 工具 / 5 MCP tools: +### 检索架构 / Search Architecture + +``` +breath(query="今天很累") + │ + ┌────┴────┐ + │ │ + Channel 1 Channel 2 + 关键词匹配 向量语义 + (rapidfuzz) (cosine similarity) + │ │ + └────┬────┘ + │ + 去重 + 合并 + token 预算截断 + │ + [语义关联] 标注 vector 来源 + │ + 返回 ≤20 条结果 +``` + +6 个 MCP 工具 / 6 MCP tools: | 工具 Tool | 作用 Purpose | |-----------|-------------| -| `breath` | 浮现或检索记忆。无参数=推送未解决记忆；有参数=关键词+情感检索 / Surface or search memories | -| `hold` | 存储单条记忆，自动打标+合并相似桶 / Store a single memory with auto-tagging | -| `grow` | 日记归档，自动拆分长内容为多个记忆桶 / Diary digest, auto-split into multiple buckets | +| `breath` | 浮现或检索记忆。无参数=推送未解决记忆；有参数=关键词+向量语义双通道检索。支持 domain/valence/arousal 过滤 / Surface or search memories. No args = surface unresolved; with query = keyword + vector dual-channel search. Supports domain/valence/arousal filters | +| `hold` | 存储单条记忆，自动打标+合并相似桶+生成 embedding。`feel=True` 写模型自己的感受 / Store a single memory with auto-tagging, merging, and embedding. `feel=True` for model's own reflections | +| `grow` | 日记归档，自动拆分长内容为多个记忆桶，每个桶自动生成 embedding / Diary digest, auto-split into multiple buckets with embeddings | | `trace` | 修改元数据、标记已解决、删除 / Modify metadata, mark resolved, delete | | `pulse` | 系统状态 + 所有记忆桶列表 / System status + bucket listing | +| `dream` | 对话开头自省消化——读最近记忆，有沉淀写 feel，能放下就 resolve / Self-reflection at conversation start | ## 安装 / Setup @@ -190,6 +348,19 @@ export OMBRE_API_KEY="your-api-key" 支持任何 OpenAI 兼容 API。在 `config.yaml` 里改 `base_url` 和 `model` 就行。 Supports any OpenAI-compatible API. Just change `base_url` and `model` in `config.yaml`. +> **💡 向量化检索（Embedding）** +> Ombre Brain 内置双通道检索：关键词匹配 + 向量语义搜索。每次 `hold`/`grow` 存入记忆时自动生成 embedding 并存入 `embeddings.db`（SQLite）。 +> 推荐：**Google AI Studio 的 `gemini-embedding-001`**（免费，1500 次/天，3072 维向量）。在 `config.yaml` 的 `embedding` 部分配置。 +> 不配置 embedding 也能用，系统会降级到纯 fuzzy matching 模式。 +> +> **已有存量桶需要补生成 embedding**：运行 `backfill_embeddings.py`： +> ```bash +> OMBRE_API_KEY="your-key" python backfill_embeddings.py --batch-size 20 +> ``` +> Docker 用户：`docker exec -e OMBRE_BUCKETS_DIR=/data ombre-brain python3 backfill_embeddings.py --batch-size 20` +> +> **Embedding support**: Built-in dual-channel search: keyword + vector semantic. Embeddings are auto-generated on each `hold`/`grow` and stored in `embeddings.db` (SQLite). Recommended: **Google AI Studio `gemini-embedding-001`** (free, 1500 req/day, 3072-dim). Configure in `config.yaml` under `embedding`. Without it, falls back to fuzzy matching. For existing buckets, run `backfill_embeddings.py`. + ### 接入 Claude Desktop / Connect to Claude Desktop 在 Claude Desktop 配置文件中添加（macOS: `~/Library/Application Support/Claude/claude_desktop_config.json`）： @@ -246,6 +417,8 @@ All parameters in `config.yaml` (copy from `config.example.yaml`). Key ones: | `buckets_dir` | 记忆桶存储路径 / Bucket storage path | `./buckets/` | | `dehydration.model` | 脱水用的 LLM 模型 / LLM model for dehydration | `deepseek-chat` | | `dehydration.base_url` | API 地址 / API endpoint | `https://api.deepseek.com/v1` | +| `embedding.enabled` | 启用向量语义检索 / Enable embedding search | `true` | +| `embedding.model` | Embedding 模型 / Embedding model | `gemini-embedding-001` | | `decay.lambda` | 衰减速率，越大越快忘 / Decay rate | `0.05` | | `decay.threshold` | 归档阈值 / Archive threshold | `0.3` | | `merge_threshold` | 合并相似度阈值 (0-100) / Merge similarity | `75` | @@ -258,25 +431,92 @@ Sensitive config via env vars: ## 衰减公式 / Decay Formula -$$final\_score = time\_weight \times base\_score$$ +$$final\_score = Importance \times activation\_count^{0.3} \times e^{-\lambda \times days} \times combined\_weight \times resolved\_factor \times urgency\_boost$$ -$$base\_score = Importance \times activation\_count^{0.3} \times e^{-\lambda \times days} \times (base + arousal \times boost)$$ +### 短期/长期权重分离 / Short-term vs Long-term Weight Separation -时间系数（乘数，优先级最高）/ Time weight (multiplier, highest priority): +系统对记忆的权重计算采用**分段策略**，模拟人类记忆的时效特征： +The system uses a **segmented weighting strategy** that mimics how human memory prioritizes: -| 距今天数 Days since active | 时间系数 Weight | +| 阶段 Phase | 时间范围 | 权重分配 | 直觉解释 | +|---|---|---|---| +| 短期 Short-term | ≤ 3 天 | 时间 70% + 情感 30% | 刚发生的事，鲜活度最重要 | +| 长期 Long-term | > 3 天 | 情感 70% + 时间 30% | 时间淡了，情感强度决定能记多久 | + +$$combined\_weight = \begin{cases} time\_weight \times 0.7 + emotion\_weight \times 0.3 & \text{if } days \leq 3 \\ emotion\_weight \times 0.7 + time\_weight \times 0.3 & \text{if } days > 3 \end{cases}$$ + +### 时间系数（新鲜度加成）/ Time Weight (Freshness Bonus) + +连续指数衰减，无跳变： +Continuous exponential decay, no discontinuities: + +$$freshness = 1.0 + 1.0 \times e^{-t/36}$$ + +| 距存入时间 Time since creation | 新鲜度乘数 Multiplier | |---|---| -| 0–1 天 | 1.0 | -| 第 2 天 | 0.9 | -| 之后每天约降 10% | `max(0.3, 0.9 × e^{-0.2197 × (days-2)})` | -| 7 天后稳定 | ≈ 0.3（不归零）| +| 刚存入 (t=0) | ×2.0 | +| 约 25 小时 | ×1.5 | +| 约 50 小时 | ×1.25 | +| 72 小时 (3天) | ×1.14 | +| 1 周+ | ≈ ×1.0 | + +t 为小时，36 为衰减常数。老记忆不被惩罚（下限 ×1.0），新记忆获得额外加成。 + +### 情感权重 / Emotion Weight + +$$emotion\_weight = base + arousal \times arousal\_boost$$ + +- 默认 `base=1.0`, `arousal_boost=0.8` +- arousal=0.3（平静）→ 1.24；arousal=0.9（激动）→ 1.72 + +### 权重池修正因子 / Weight Pool Modifiers + +| 状态 State | 修正因子 Factor | 说明 | +|---|---|---| +| 未解决 Unresolved | ×1.0 | 正常权重 | +| 已解决 Resolved | ×0.05 | 沉底，等关键词唤醒 | +| 已解决+已消化 Resolved+Digested | ×0.02 | 加速淡化，归档为无限小 | +| 高唤醒+未解决 Urgent | ×1.5 | arousal>0.7 的未解决记忆额外加权 | +| 钉选 Pinned | 999.0 | 不衰减、不合并、importance=10 | +| Feel | 50.0 | 固定分数，不参与衰减 | + +### 参数说明 / Parameters - `importance`: 1-10，记忆重要性 / memory importance - `activation_count`: 被检索的次数，越常被想起衰减越慢 / retrieval count; more recalls = slower decay - `days`: 距上次激活的天数 / days since last activation - `arousal`: 唤醒度，越强烈的记忆越难忘 / arousal; intense memories are harder to forget -- 已解决的记忆权重降到 5%，沉底等被关键词唤醒 / resolved memories drop to 5%, sink until keyword-triggered -- `pinned=true` 的桶：不衰减、不合并、importance 锁定 10 / `pinned` buckets: never decay, never merge, importance locked at 10 +- `λ` (decay_lambda): 衰减速率，默认 0.05 / decay rate, default 0.05 + +## Dreaming 与 Feel / Dreaming & Feel + +### Dreaming — 做梦 +每次新对话开始时，Claude 会自动执行 `dream()`——读取最近的记忆桶，用第一人称思考：哪些事还有重量？哪些可以放下了？ + +At the start of each conversation, Claude runs `dream()` — reads recent memory buckets and reflects in first person: what still carries weight? What can be let go? + +- 值得放下的 → `trace(resolved=1)` 让它沉底 +- 有沉淀的 → 写 `feel`，记录模型自己的感受 +- 没有沉淀就不写，不强迫产出 + +### Feel — 带走的东西 +Feel 不是事件记录，是**模型带走的东西**——一句感受、一个未解答的问题、一个观察到的变化。 + +Feel is not an event log — it's **what the model carries away**: a feeling, an unanswered question, a noticed change. + +- `hold(content="...", feel=True, source_bucket="源记忆ID", valence=模型自己的感受)` +- `valence` 是模型的感受，不是事件情绪。同一段争吵，事件 V0.2，但模型可能 V0.4（「我从中看到了成长」） +- `source_bucket` 指向被消化的记忆，会被标记为「已消化」→ 加速淡化到无限小，但不会被删除 +- Feel 不参与普通浮现、不衰减、不参与 dreaming +- 用 `breath(domain="feel")` 读取之前的 feel + +### 对话启动完整流程 / Conversation Start Sequence +``` +1. breath() — 睁眼，看有什么浮上来 +2. dream() — 消化最近记忆，有沉淀写 feel +3. breath(domain="feel") — 读之前的 feel +4. 开始和用户说话 +``` ## 给 Claude 的使用指南 / Usage Guide for Claude @@ -288,14 +528,32 @@ $$base\_score = Importance \times activation\_count^{0.3} \times e^{-\lambda \ti | 脚本 Script | 用途 Purpose | |---|---| +| `embedding_engine.py` | 向量化引擎，管理 embedding 的生成、存储、相似度搜索 / Embedding engine: generate, store, and search embeddings | +| `backfill_embeddings.py` | 为存量桶批量生成 embedding / Batch-generate embeddings for existing buckets | | `write_memory.py` | 手动写入记忆，绕过 MCP / Manually write memories, bypass MCP | | `migrate_to_domains.py` | 迁移平铺文件到域子目录 / Migrate flat files to domain subdirs | | `reclassify_domains.py` | 基于关键词重分类 / Reclassify by keywords | | `reclassify_api.py` | 用 API 重打标未分类桶 / Re-tag uncategorized buckets via API | +| `test_tools.py` | MCP 工具集成测试（8 项） / MCP tool integration tests (8 tests) | | `test_smoke.py` | 冒烟测试 / Smoke test | ## 部署 / Deploy +### Docker Hub 预构建镜像 + +[![Docker Hub](https://img.shields.io/docker/v/p0luz/ombre-brain?label=Docker%20Hub&logo=docker)](https://hub.docker.com/r/p0luz/ombre-brain) + +不用 clone 代码、不用 build，直接拉取预构建镜像： + +```bash +docker pull p0luz/ombre-brain:latest +curl -O https://raw.githubusercontent.com/P0luz/Ombre-Brain/main/docker-compose.user.yml +echo "OMBRE_API_KEY=你的key" > .env +docker compose -f docker-compose.user.yml up -d +``` + +验证：`curl http://localhost:8000/health` + ### Render [![Deploy to Render](https://render.com/images/deploy-to-render-button.svg)](https://render.com/deploy?repo=https://github.com/P0luz/Ombre-Brain) diff --git a/backfill_embeddings.py b/backfill_embeddings.py new file mode 100644 index 0000000..4a06d5f --- /dev/null +++ b/backfill_embeddings.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +""" +Backfill embeddings for existing buckets. +为存量桶批量生成 embedding。 + +Usage: + OMBRE_BUCKETS_DIR=/data OMBRE_API_KEY=xxx python backfill_embeddings.py [--batch-size 20] [--dry-run] + +Each batch calls Gemini embedding API once per bucket. +Free tier: 1500 requests/day, so ~75 batches of 20. +""" + +import asyncio +import argparse +import sys +import time + +sys.path.insert(0, ".") +from utils import load_config +from bucket_manager import BucketManager +from embedding_engine import EmbeddingEngine + + +async def backfill(batch_size: int = 20, dry_run: bool = False): + config = load_config() + bucket_mgr = BucketManager(config) + engine = EmbeddingEngine(config) + + if not engine.enabled: + print("ERROR: Embedding engine not enabled (missing API key?)") + return + + all_buckets = await bucket_mgr.list_all(include_archive=True) + print(f"Total buckets: {len(all_buckets)}") + + # Find buckets without embeddings + missing = [] + for b in all_buckets: + emb = await engine.get_embedding(b["id"]) + if emb is None: + missing.append(b) + + print(f"Missing embeddings: {len(missing)}") + + if dry_run: + for b in missing[:10]: + print(f" would embed: {b['id']} ({b['metadata'].get('name', '?')})") + if len(missing) > 10: + print(f" ... and {len(missing) - 10} more") + return + + total = len(missing) + success = 0 + failed = 0 + + for i in range(0, total, batch_size): + batch = missing[i : i + batch_size] + batch_num = i // batch_size + 1 + total_batches = (total + batch_size - 1) // batch_size + print(f"\n--- Batch {batch_num}/{total_batches} ({len(batch)} buckets) ---") + + for b in batch: + name = b["metadata"].get("name", b["id"]) + content = b.get("content", "") + if not content or not content.strip(): + print(f" SKIP (empty): {b['id']} ({name})") + continue + + try: + ok = await engine.generate_and_store(b["id"], content) + if ok: + success += 1 + print(f" OK: {b['id'][:12]} ({name[:30]})") + else: + failed += 1 + print(f" FAIL: {b['id'][:12]} ({name[:30]})") + except Exception as e: + failed += 1 + print(f" ERROR: {b['id'][:12]} ({name[:30]}): {e}") + + if i + batch_size < total: + print(f" Waiting 2s before next batch...") + await asyncio.sleep(2) + + print(f"\n=== Done: {success} success, {failed} failed, {total - success - failed} skipped ===") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--batch-size", type=int, default=20) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + asyncio.run(backfill(batch_size=args.batch_size, dry_run=args.dry_run)) diff --git a/bucket_manager.py b/bucket_manager.py index 4664080..f815eb8 100644 --- a/bucket_manager.py +++ b/bucket_manager.py @@ -60,6 +60,7 @@ class BucketManager: self.permanent_dir = os.path.join(self.base_dir, "permanent") self.dynamic_dir = os.path.join(self.base_dir, "dynamic") self.archive_dir = os.path.join(self.base_dir, "archive") + self.feel_dir = os.path.join(self.base_dir, "feel") self.fuzzy_threshold = config.get("matching", {}).get("fuzzy_threshold", 50) self.max_results = config.get("matching", {}).get("max_results", 5) @@ -122,7 +123,7 @@ class BucketManager: bucket_name = sanitize_name(name) if name else bucket_id domain = domain or ["未分类"] tags = tags or [] - linked_content = self._apply_wikilinks(content, tags, domain, bucket_name) + linked_content = content # wikilink injection disabled; LLM adds [[]] via prompt # --- Pinned/protected buckets: lock importance to 10 --- # --- 钉选/保护桶：importance 强制锁定为 10 --- @@ -154,8 +155,18 @@ class BucketManager: # --- Choose directory by type + primary domain --- # --- 按类型 + 主题域选择存储目录 --- - type_dir = self.permanent_dir if bucket_type == "permanent" else self.dynamic_dir - primary_domain = sanitize_name(domain[0]) if domain else "未分类" + if bucket_type == "permanent" or pinned: + type_dir = self.permanent_dir + if pinned and bucket_type != "permanent": + metadata["type"] = "permanent" + elif bucket_type == "feel": + type_dir = self.feel_dir + else: + type_dir = self.dynamic_dir + if bucket_type == "feel": + primary_domain = "沉淀物" # feel subfolder name + else: + primary_domain = sanitize_name(domain[0]) if domain else "未分类" target_dir = os.path.join(type_dir, primary_domain) os.makedirs(target_dir, exist_ok=True) @@ -197,6 +208,25 @@ class BucketManager: return None return self._load_bucket(file_path) + # --------------------------------------------------------- + # Move bucket between directories + # 在目录间移动桶文件 + # --------------------------------------------------------- + def _move_bucket(self, file_path: str, target_type_dir: str, domain: list[str] = None) -> str: + """ + Move a bucket file to a new type directory, preserving domain subfolder. + Returns new file path. + """ + primary_domain = sanitize_name(domain[0]) if domain else "未分类" + target_dir = os.path.join(target_type_dir, primary_domain) + os.makedirs(target_dir, exist_ok=True) + filename = os.path.basename(file_path) + new_path = safe_path(target_dir, filename) + if os.path.normpath(file_path) != os.path.normpath(new_path): + os.rename(file_path, new_path) + logger.info(f"Moved bucket / 移动记忆桶: {filename} → {target_dir}/") + return new_path + # --------------------------------------------------------- # Update bucket # 更新桶 @@ -225,15 +255,7 @@ class BucketManager: # --- Update only fields that were passed in / 只改传入的字段 --- if "content" in kwargs: - next_tags = kwargs.get("tags", post.get("tags", [])) - next_domain = kwargs.get("domain", post.get("domain", [])) - next_name = kwargs.get("name", post.get("name", "")) - post.content = self._apply_wikilinks( - kwargs["content"], - next_tags, - next_domain, - next_name, - ) + post.content = kwargs["content"] # wikilink injection disabled; LLM adds [[]] via prompt if "tags" in kwargs: post["tags"] = kwargs["tags"] if "importance" in kwargs: @@ -252,6 +274,10 @@ class BucketManager: post["pinned"] = bool(kwargs["pinned"]) if kwargs["pinned"]: post["importance"] = 10 # pinned → lock importance to 10 + if "digested" in kwargs: + post["digested"] = bool(kwargs["digested"]) + if "model_valence" in kwargs: + post["model_valence"] = max(0.0, min(1.0, float(kwargs["model_valence"]))) # --- Auto-refresh activation time / 自动刷新激活时间 --- post["last_active"] = now_iso() @@ -263,136 +289,33 @@ class BucketManager: logger.error(f"Failed to write bucket update / 写入桶更新失败: {file_path}: {e}") return False + # --- Auto-move: pinned → permanent/, resolved → archive/ --- + # --- 自动移动：钉选 → permanent/，已解决 → archive/ --- + domain = post.get("domain", ["未分类"]) + if kwargs.get("pinned") and post.get("type") != "permanent": + post["type"] = "permanent" + with open(file_path, "w", encoding="utf-8") as f: + f.write(frontmatter.dumps(post)) + self._move_bucket(file_path, self.permanent_dir, domain) + elif kwargs.get("resolved") and post.get("type") not in ("permanent", "feel"): + post["type"] = "archived" + with open(file_path, "w", encoding="utf-8") as f: + f.write(frontmatter.dumps(post)) + self._move_bucket(file_path, self.archive_dir, domain) + logger.info(f"Updated bucket / 更新记忆桶: {bucket_id}") return True # --------------------------------------------------------- - # Wikilink injection - # 自动添加 Obsidian 双链 + # Wikilink injection — DISABLED + # 自动添加 Obsidian 双链 — 已禁用 + # Now handled by LLM prompts (Gemini adds [[]] for proper nouns) + # 现在由 LLM prompt 处理（Gemini 对人名/地名/专有名词加 [[]]） # --------------------------------------------------------- - def _apply_wikilinks( - self, - content: str, - tags: list[str], - domain: list[str], - name: str, - ) -> str: - """ - Auto-inject Obsidian wikilinks, avoiding double-wrapping existing [[...]]. - 自动添加 Obsidian 双链，避免重复包裹已有 [[...]]。 - """ - if not self.wikilink_enabled or not content: - return content - - keywords = self._collect_wikilink_keywords(content, tags, domain, name) - if not keywords: - return content - - # Split on existing wikilinks to avoid wrapping them again - # 按已有双链切分，避免重复包裹 - segments = re.split(r"(\[\[[^\]]+\]\])", content) - pattern = re.compile("|".join(re.escape(kw) for kw in keywords)) - for i, segment in enumerate(segments): - if segment.startswith("[[") and segment.endswith("]]"): - continue - updated = pattern.sub(lambda m: f"[[{m.group(0)}]]", segment) - segments[i] = updated - return "".join(segments) - - def _collect_wikilink_keywords( - self, - content: str, - tags: list[str], - domain: list[str], - name: str, - ) -> list[str]: - """ - Collect candidate keywords from tags/domain/auto-extraction. - 汇总候选关键词：可选 tags/domain + 自动提词。 - """ - candidates = [] - - if self.wikilink_use_tags: - candidates.extend(tags or []) - if self.wikilink_use_domain: - candidates.extend(domain or []) - if name: - candidates.append(name) - if self.wikilink_use_auto_keywords: - candidates.extend(self._extract_auto_keywords(content)) - - return self._normalize_keywords(candidates) - - def _normalize_keywords(self, keywords: list[str]) -> list[str]: - """ - Deduplicate and sort by length (longer first to avoid short words - breaking long ones during replacement). - 去重并按长度排序，优先替换长词。 - """ - if not keywords: - return [] - - seen = set() - cleaned = [] - for keyword in keywords: - if not isinstance(keyword, str): - continue - kw = keyword.strip() - if len(kw) < self.wikilink_min_len: - continue - if kw in self.wikilink_exclude_keywords: - continue - if kw.lower() in self.wikilink_stopwords: - continue - if kw in seen: - continue - seen.add(kw) - cleaned.append(kw) - - return sorted(cleaned, key=len, reverse=True) - - def _extract_auto_keywords(self, content: str) -> list[str]: - """ - Auto-extract keywords from body text, prioritizing high-frequency words. - 从正文自动提词，优先高频词。 - """ - if not content: - return [] - - try: - zh_words = [w.strip() for w in jieba.lcut(content) if w.strip()] - except Exception: - zh_words = [] - en_words = re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,20}", content) - - # Chinese bigrams / 中文双词组合 - zh_bigrams = [] - for i in range(len(zh_words) - 1): - left = zh_words[i] - right = zh_words[i + 1] - if len(left) < self.wikilink_min_len or len(right) < self.wikilink_min_len: - continue - if not re.fullmatch(r"[\u4e00-\u9fff]+", left + right): - continue - if len(left + right) > 8: - continue - zh_bigrams.append(left + right) - - merged = [] - for word in zh_words + zh_bigrams + en_words: - if len(word) < self.wikilink_min_len: - continue - if re.fullmatch(r"\d+", word): - continue - if word.lower() in self.wikilink_stopwords: - continue - merged.append(word) - - if not merged: - return [] - - counter = Counter(merged) - return [w for w, _ in counter.most_common(self.wikilink_auto_top_k)] + # def _apply_wikilinks(self, content, tags, domain, name): ... + # def _collect_wikilink_keywords(self, content, tags, domain, name): ... + # def _normalize_keywords(self, keywords): ... + # def _extract_auto_keywords(self, content): ... # --------------------------------------------------------- # Delete bucket @@ -425,7 +348,9 @@ class BucketManager: async def touch(self, bucket_id: str) -> None: """ Update a bucket's last activation time and count. + Also triggers time ripple: nearby memories get a slight activation boost. 更新桶的最后激活时间和激活次数。 + 同时触发时间涟漪：时间上相邻的记忆轻微唤醒。 """ file_path = self._find_bucket_file(bucket_id) if not file_path: @@ -438,9 +363,60 @@ class BucketManager: with open(file_path, "w", encoding="utf-8") as f: f.write(frontmatter.dumps(post)) + + # --- Time ripple: boost nearby memories within ±48h --- + # --- 时间涟漪：±48小时内的记忆轻微唤醒 --- + current_time = datetime.fromisoformat(str(post.get("created", post.get("last_active", "")))) + await self._time_ripple(bucket_id, current_time) except Exception as e: logger.warning(f"Failed to touch bucket / 触碰桶失败: {bucket_id}: {e}") + async def _time_ripple(self, source_id: str, reference_time: datetime, hours: float = 48.0) -> None: + """ + Slightly boost activation_count of buckets created/activated near the reference time. + 轻微提升时间相邻桶的激活次数（+0.3），不改 last_active 避免递归唤醒。 + Max 5 buckets rippled per touch to bound I/O. + """ + try: + all_buckets = await self.list_all(include_archive=False) + except Exception: + return + + rippled = 0 + max_ripple = 5 + for bucket in all_buckets: + if rippled >= max_ripple: + break + if bucket["id"] == source_id: + continue + meta = bucket.get("metadata", {}) + # Skip pinned/permanent/feel + if meta.get("pinned") or meta.get("protected") or meta.get("type") in ("permanent", "feel"): + continue + + created_str = meta.get("created", meta.get("last_active", "")) + try: + created = datetime.fromisoformat(str(created_str)) + delta_hours = abs((reference_time - created).total_seconds()) / 3600 + except (ValueError, TypeError): + continue + + if delta_hours <= hours: + # Boost activation_count by 0.3 (fractional), don't change last_active + file_path = self._find_bucket_file(bucket["id"]) + if not file_path: + continue + try: + post = frontmatter.load(file_path) + current_count = post.get("activation_count", 1) + # Store as float for fractional increments; calculate_score handles it + post["activation_count"] = round(current_count + 0.3, 1) + with open(file_path, "w", encoding="utf-8") as f: + f.write(frontmatter.dumps(post)) + rippled += 1 + except Exception: + continue + # --------------------------------------------------------- # Multi-dimensional search (core feature) # 多维搜索（核心功能） @@ -576,7 +552,7 @@ class BucketManager: ) content_score = fuzz.partial_ratio(query, bucket.get("content", "")[:1000]) * self.content_weight - return (name_score + domain_score + tag_score + content_score) / (100 * 10.5) + return (name_score + domain_score + tag_score + content_score) / (100 * (3 + 2.5 + 2 + self.content_weight)) # --------------------------------------------------------- # Emotion resonance sub-score: @@ -633,7 +609,7 @@ class BucketManager: """ buckets = [] - dirs = [self.permanent_dir, self.dynamic_dir] + dirs = [self.permanent_dir, self.dynamic_dir, self.feel_dir] if include_archive: dirs.append(self.archive_dir) @@ -664,6 +640,7 @@ class BucketManager: "permanent_count": 0, "dynamic_count": 0, "archive_count": 0, + "feel_count": 0, "total_size_kb": 0.0, "domains": {}, } @@ -672,6 +649,7 @@ class BucketManager: (self.permanent_dir, "permanent_count"), (self.dynamic_dir, "dynamic_count"), (self.archive_dir, "archive_count"), + (self.feel_dir, "feel_count"), ]: if not os.path.exists(subdir): continue @@ -745,7 +723,7 @@ class BucketManager: """ if not bucket_id: return None - for dir_path in [self.permanent_dir, self.dynamic_dir, self.archive_dir]: + for dir_path in [self.permanent_dir, self.dynamic_dir, self.archive_dir, self.feel_dir]: if not os.path.exists(dir_path): continue for root, _, files in os.walk(dir_path): @@ -754,7 +732,8 @@ class BucketManager: continue # Match by exact ID segment in filename # 通过文件名中的 ID 片段精确匹配 - if bucket_id in fname: + name_part = fname[:-3] # remove .md + if name_part == bucket_id or name_part.endswith(f"_{bucket_id}"): return os.path.join(root, fname) return None diff --git a/config.example.yaml b/config.example.yaml index 394c081..1c31231 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -58,6 +58,15 @@ decay: base: 1.0 # Base weight / 基础权重 arousal_boost: 0.8 # Arousal boost coefficient / 唤醒度加成系数 +# --- Embedding / 向量化配置 --- +# Uses embedding API for semantic similarity search +# 通过 embedding API 实现语义相似度搜索 +# Reuses the same API key (OMBRE_API_KEY) and base_url from dehydration config +# 复用脱水配置的 API key 和 base_url +embedding: + enabled: true # Enable embedding / 启用向量化 + model: "gemini-embedding-001" # Embedding model / 向量化模型 + # --- Scoring weights / 检索权重参数 --- # total = topic(×4) + emotion(×2) + time(×1.5) + importance(×1) scoring_weights: @@ -77,6 +86,6 @@ wikilink: use_tags: false use_domain: true use_auto_keywords: true - auto_top_k: 8 - min_keyword_len: 2 + auto_top_k: 4 + min_keyword_len: 3 exclude_keywords: [] diff --git a/dashboard.html b/dashboard.html new file mode 100644 index 0000000..eddab40 --- /dev/null +++ b/dashboard.html @@ -0,0 +1,1391 @@ + + + + + +Ombre Brain + + + + + +
+

◐ Ombre Brain

+ loading... +
+ +
+
+ +
+
记忆桶
+
Breath 模拟
+
记忆网络
+
配置
+
导入
+
+ +
+
+
+
加载中…
+
+
+ + + + + + + + + +
+ +
+
+ + + + diff --git a/decay_engine.py b/decay_engine.py index 43c7624..2213deb 100644 --- a/decay_engine.py +++ b/decay_engine.py @@ -70,93 +70,102 @@ class DecayEngine: # Permanent buckets never decay / 固化桶永远不衰减 # --------------------------------------------------------- # --------------------------------------------------------- - # Time weight: 0-1d→1.0, day2→0.9, then ~10%/day, floor 0.3 - # 时间系数：0-1天=1.0，第2天=0.9，之后每天约降10%，7天后稳定在0.3 + # Freshness bonus: continuous exponential decay + # 新鲜度加成：连续指数衰减 + # bonus = 1.0 + 1.0 × e^(-t/36), t in hours + # t=0 → 2.0×, t≈25h(半衰) → 1.5×, t≈72h → ≈1.14×, t→∞ → 1.0× # --------------------------------------------------------- @staticmethod def _calc_time_weight(days_since: float) -> float: """ - Piecewise time weight multiplier (multiplies base_score). - 分段式时间权重系数，作为 final_score 的乘数。 + Freshness bonus multiplier: 1.0 + e^(-t/36), t in hours. + 新鲜度加成乘数：刚存入×2.0，~36小时半衰，72小时后趋近×1.0。 """ - if days_since <= 1.0: - return 1.0 - elif days_since <= 2.0: - # Linear interpolation: 1.0→0.9 over [1,2] - return 1.0 - 0.1 * (days_since - 1.0) - else: - # Exponential decay from 0.9, floor at 0.3 - # k = ln(3)/5 ≈ 0.2197 so that at day 7 (5 days past day 2) → 0.3 - raw = 0.9 * math.exp(-0.2197 * (days_since - 2.0)) - return max(0.3, raw) + hours = days_since * 24.0 + return 1.0 + 1.0 * math.exp(-hours / 36.0) def calculate_score(self, metadata: dict) -> float: """ Calculate current activity score for a memory bucket. 计算一个记忆桶的当前活跃度得分。 - Formula: final_score = time_weight × base_score - base_score = Importance × (act_count^0.3) × e^(-λ×days) × (base + arousal×boost) - time_weight is the outer multiplier, takes priority over emotion factors. + New model: short-term vs long-term weight separation. + 新模型：短期/长期权重分离。 + - Short-term (≤3 days): time_weight dominates, emotion amplifies + - Long-term (>3 days): emotion_weight dominates, time decays to floor + 短期（≤3天）：时间权重主导，情感放大 + 长期（>3天）：情感权重主导，时间衰减到底线 """ if not isinstance(metadata, dict): return 0.0 # --- Pinned/protected buckets: never decay, importance locked to 10 --- - # --- 固化桶（pinned/protected）：永不衰减，importance 锁定为 10 --- if metadata.get("pinned") or metadata.get("protected"): return 999.0 - # --- Permanent buckets never decay / 固化桶永不衰减 --- + # --- Permanent buckets never decay --- if metadata.get("type") == "permanent": return 999.0 + # --- Feel buckets: never decay, fixed moderate score --- + if metadata.get("type") == "feel": + return 50.0 + importance = max(1, min(10, int(metadata.get("importance", 5)))) activation_count = max(1, int(metadata.get("activation_count", 1))) - # --- Days since last activation / 距离上次激活过了多少天 --- + # --- Days since last activation --- last_active_str = metadata.get("last_active", metadata.get("created", "")) try: last_active = datetime.fromisoformat(str(last_active_str)) days_since = max(0.0, (datetime.now() - last_active).total_seconds() / 86400) except (ValueError, TypeError): - days_since = 30 # Parse failure → assume 30 days / 解析失败假设已过 30 天 + days_since = 30 - # --- Emotion weight: continuous arousal coordinate --- - # --- 情感权重：基于连续 arousal 坐标计算 --- - # Higher arousal → stronger emotion → higher weight → slower decay - # arousal 越高 → 情感越强烈 → 权重越大 → 衰减越慢 + # --- Emotion weight --- try: arousal = max(0.0, min(1.0, float(metadata.get("arousal", 0.3)))) except (ValueError, TypeError): arousal = 0.3 emotion_weight = self.emotion_base + arousal * self.arousal_boost - # --- Time weight (outer multiplier, highest priority) --- - # --- 时间权重（外层乘数，优先级最高）--- + # --- Time weight --- time_weight = self._calc_time_weight(days_since) - # --- Base score = Importance × act_count^0.3 × e^(-λ×days) × emotion --- - # --- 基础得分 --- + # --- Short-term vs Long-term weight separation --- + # 短期（≤3天）：time_weight 占 70%，emotion 占 30% + # 长期（>3天）：emotion 占 70%，time_weight 占 30% + if days_since <= 3.0: + # Short-term: time dominates, emotion amplifies + combined_weight = time_weight * 0.7 + emotion_weight * 0.3 + else: + # Long-term: emotion dominates, time provides baseline + combined_weight = emotion_weight * 0.7 + time_weight * 0.3 + + # --- Base score --- base_score = ( importance * (activation_count ** 0.3) * math.exp(-self.decay_lambda * days_since) - * emotion_weight + * combined_weight ) - # --- final_score = time_weight × base_score --- - score = time_weight * base_score + # --- Weight pool modifiers --- + # resolved + digested (has feel) → accelerated fade: ×0.02 + # resolved only → ×0.05 + # 已处理+已消化（写过feel）→ 加速淡化：×0.02 + # 仅已处理 → ×0.05 + resolved = metadata.get("resolved", False) + digested = metadata.get("digested", False) # set when feel is written for this memory + if resolved and digested: + resolved_factor = 0.02 + elif resolved: + resolved_factor = 0.05 + else: + resolved_factor = 1.0 + urgency_boost = 1.5 if (arousal > 0.7 and not resolved) else 1.0 - # --- Weight pool modifiers / 权重池修正因子 --- - # Resolved events drop to 5%, sink to bottom awaiting keyword reactivation - # 已解决的事件权重骤降到 5%，沉底等待关键词激活 - resolved_factor = 0.05 if metadata.get("resolved", False) else 1.0 - # High-arousal unresolved buckets get urgency boost for priority surfacing - # 高唤醒未解决桶额外加成，优先浮现 - urgency_boost = 1.5 if (arousal > 0.7 and not metadata.get("resolved", False)) else 1.0 - - return round(score * resolved_factor * urgency_boost, 4) + return round(base_score * resolved_factor * urgency_boost, 4) # --------------------------------------------------------- # Execute one decay cycle @@ -180,17 +189,41 @@ class DecayEngine: checked = 0 archived = 0 + auto_resolved = 0 lowest_score = float("inf") for bucket in buckets: meta = bucket.get("metadata", {}) - # Skip permanent / pinned / protected buckets - # 跳过固化桶和钉选/保护桶 - if meta.get("type") == "permanent" or meta.get("pinned") or meta.get("protected"): + # Skip permanent / pinned / protected / feel buckets + # 跳过固化桶、钉选/保护桶和 feel 桶 + if meta.get("type") in ("permanent", "feel") or meta.get("pinned") or meta.get("protected"): continue checked += 1 + + # --- Auto-resolve: imp≤4 + >30 days old + not resolved → auto resolve --- + # --- 自动结案：重要度≤4 + 超过30天 + 未解决 → 自动 resolve --- + if not meta.get("resolved", False): + imp = int(meta.get("importance", 5)) + last_active_str = meta.get("last_active", meta.get("created", "")) + try: + last_active = datetime.fromisoformat(str(last_active_str)) + days_since = (datetime.now() - last_active).total_seconds() / 86400 + except (ValueError, TypeError): + days_since = 999 + if imp <= 4 and days_since > 30: + try: + await self.bucket_mgr.update(bucket["id"], resolved=True) + auto_resolved += 1 + logger.info( + f"Auto-resolved / 自动结案: " + f"{meta.get('name', bucket['id'])} " + f"(imp={imp}, days={days_since:.0f})" + ) + except Exception as e: + logger.warning(f"Auto-resolve failed / 自动结案失败: {e}") + try: score = self.calculate_score(meta) except Exception as e: @@ -223,6 +256,7 @@ class DecayEngine: result = { "checked": checked, "archived": archived, + "auto_resolved": auto_resolved, "lowest_score": lowest_score if checked > 0 else 0, } logger.info(f"Decay cycle complete / 衰减周期完成: {result}") diff --git a/dehydrator.py b/dehydrator.py index 2db6132..faae140 100644 --- a/dehydrator.py +++ b/dehydrator.py @@ -67,6 +67,9 @@ DIGEST_PROMPT = """你是一个日记整理专家。用户会发送一段包含 3. 去除无意义的口水话和重复信息，保留核心内容 4. 同一主题的零散信息应合并为一个条目 5. 如果有待办事项，单独提取为一个条目 +6. 单个条目内容不少于50字，过短的零碎信息合并到最相关的条目中 +7. 总条目数控制在 2~6 个，避免过度碎片化 +8. 在 content 中对人名、地名、专有名词用 [[双链]] 标记（如 [[婷易]]、[[Obsidian]]），普通词汇不要加 输出格式（纯 JSON 数组，无其他内容）： [ @@ -76,11 +79,13 @@ DIGEST_PROMPT = """你是一个日记整理专家。用户会发送一段包含 "domain": ["主题域1"], "valence": 0.7, "arousal": 0.4, - "tags": ["标签1", "标签2"], + "tags": ["核心词1", "核心词2", "扩展词1", "扩展词2"], "importance": 5 } ] +tags 生成规则：先从原文精准提取 3~5 个核心词，再引申扩展 5~8 个语义相关词（近义词、上位词、关联场景词），合并为一个数组。 + 主题域可选（选最精确的 1~2 个，只选真正相关的）： 日常: ["饮食", "穿搭", "出行", "居家", "购物"] 人际: ["家庭", "恋爱", "友谊", "社交"] @@ -104,6 +109,7 @@ MERGE_PROMPT = """你是一个信息合并专家。请将旧记忆与新内容 2. 去除重复信息 3. 保留所有重要事实 4. 总长度尽量不超过旧记忆的 120% +5. 对出现的人名、地名、专有名词用 [[双链]] 标记（如 [[婷易]]、[[Obsidian]]），普通词汇不要加 直接输出合并后的文本，不要加额外说明。""" @@ -124,15 +130,19 @@ ANALYZE_PROMPT = """你是一个内容分析器。请分析以下文本，输出 内心: ["情绪", "回忆", "梦境", "自省"] 2. valence（情感效价）：0.0~1.0，0=极度消极 → 0.5=中性 → 1.0=极度积极 3. arousal（情感唤醒度）：0.0~1.0，0=非常平静 → 0.5=普通 → 1.0=非常激动 -4. tags（关键词标签）：3~5 个最能概括内容的关键词 +4. tags（关键词标签）：分两步生成，合并为一个数组： + 第一步—精准提取：从原文抽取 3~5 个真正的核心词，不泛化、不遗漏 + 第二步—引申扩展：自动补充 8~10 个与当前场景语义相关的词，包括近义词、上位词、关联场景词、用户可能用不同措辞搜索的词 + 两步合并为一个 tags 数组，总计 10~15 个 5. suggested_name（建议桶名）：10字以内的简短标题 +6. 在 tags 和 suggested_name 中不要使用 [[]] 双链标记 输出格式（纯 JSON，无其他内容）： { "domain": ["主题域1", "主题域2"], "valence": 0.7, "arousal": 0.4, - "tags": ["标签1", "标签2", "标签3"], + "tags": ["核心词1", "核心词2", "扩展词1", "扩展词2", "..."], "suggested_name": "简短标题" }""" @@ -214,20 +224,18 @@ class Dehydrator: if not new_content: return old_content - # --- Try API merge first / 优先 API 合并 --- - if self.api_available: - try: - result = await self._api_merge(old_content, new_content) - if result: - return result - except Exception as e: - logger.warning( - f"API merge failed, degrading to local / " - f"API 合并失败，降级到本地合并: {e}" - ) - - # --- Local merge fallback / 本地合并兜底 --- - return self._local_merge(old_content, new_content) + # --- API merge (no local fallback) --- + if not self.api_available: + raise RuntimeError("脱水 API 不可用，请检查 config.yaml 中的 dehydration 配置") + try: + result = await self._api_merge(old_content, new_content) + if result: + return result + raise RuntimeError("API 合并返回空结果") + except RuntimeError: + raise + except Exception as e: + raise RuntimeError(f"API 合并失败，请检查 API 连接: {e}") from e # --------------------------------------------------------- # API call: dehydration @@ -314,24 +322,6 @@ class Dehydrator: summary = summary[:1000] + "…" return summary - # --------------------------------------------------------- - # Local merge (simple concatenation + truncation) - # 本地合并（简单拼接 + 截断） - # --------------------------------------------------------- - def _local_merge(self, old_content: str, new_content: str) -> str: - """ - Simple concatenation merge; truncates if too long. - 简单拼接合并，超长时截断保留两端。 - """ - merged = f"{old_content.strip()}\n\n--- 更新 ---\n{new_content.strip()}" - # Truncate if over 3000 chars / 超过 3000 字符则各取一半 - if len(merged) > 3000: - half = 1400 - merged = ( - f"{old_content[:half].strip()}\n\n--- 更新 ---\n{new_content[:half].strip()}" - ) - return merged - # --------------------------------------------------------- # Keyword extraction # 关键词提取 @@ -391,6 +381,15 @@ class Dehydrator: if domains: header += f" [主题:{domains}]" header += f" [情感:V{valence:.1f}/A{arousal:.1f}]" + # Show model's perspective if available (valence drift) + model_v = metadata.get("model_valence") + if model_v is not None: + try: + header += f" [我的视角:V{float(model_v):.1f}]" + except (ValueError, TypeError): + pass + if metadata.get("digested"): + header += " [已消化]" header += "\n" content = re.sub(r'\[\[([^\]]+)\]\]', r'\1', content) @@ -412,20 +411,18 @@ class Dehydrator: if not content or not content.strip(): return self._default_analysis() - # --- Try API first (best quality) / 优先走 API --- - if self.api_available: - try: - result = await self._api_analyze(content) - if result: - return result - except Exception as e: - logger.warning( - f"API tagging failed, degrading to local / " - f"API 打标失败，降级到本地分析: {e}" - ) - - # --- Local analysis fallback / 本地分析兜底 --- - return self._local_analyze(content) + # --- API analyze (no local fallback) --- + if not self.api_available: + raise RuntimeError("脱水 API 不可用，请检查 config.yaml 中的 dehydration 配置") + try: + result = await self._api_analyze(content) + if result: + return result + raise RuntimeError("API 打标返回空结果") + except RuntimeError: + raise + except Exception as e: + raise RuntimeError(f"API 打标失败，请检查 API 连接: {e}") from e # --------------------------------------------------------- # API call: auto-tagging @@ -487,121 +484,10 @@ class Dehydrator: "domain": result.get("domain", ["未分类"])[:3], "valence": valence, "arousal": arousal, - "tags": result.get("tags", [])[:5], + "tags": result.get("tags", [])[:15], "suggested_name": str(result.get("suggested_name", ""))[:20], } - # --------------------------------------------------------- - # Local analysis (fallback when API is unavailable) - # 本地分析（无 API 时的兜底方案） - # Keyword matching + simple sentiment dictionary - # 基于关键词 + 简单情感词典匹配 - # --------------------------------------------------------- - def _local_analyze(self, content: str) -> dict: - """ - Local keyword + sentiment dictionary analysis. - 本地关键词 + 情感词典的简单分析。 - """ - keywords = self._extract_keywords(content) - text_lower = content.lower() - - # --- Domain matching by keyword hits --- - # --- 主题域匹配：基于关键词命中 --- - domain_keywords = { - # Daily / 日常 - "饮食": {"吃", "饭", "做饭", "外卖", "奶茶", "咖啡", "麻辣烫", "面包", - "超市", "零食", "水果", "牛奶", "食堂", "减肥", "节食"}, - "出行": {"旅行", "出发", "航班", "酒店", "地铁", "打车", "高铁", "机票", - "景点", "签证", "护照"}, - "居家": {"打扫", "洗衣", "搬家", "快递", "收纳", "装修", "租房"}, - "购物": {"买", "下单", "到货", "退货", "优惠", "折扣", "代购"}, - # Relationships / 人际 - "家庭": {"爸", "妈", "父亲", "母亲", "家人", "弟弟", "姐姐", "哥哥", - "奶奶", "爷爷", "亲戚", "家里"}, - "恋爱": {"爱人", "男友", "女友", "恋", "约会", "接吻", "分手", - "暧昧", "在一起", "想你", "同床"}, - "友谊": {"朋友", "闺蜜", "兄弟", "聚", "约饭", "聊天", "群"}, - "社交": {"见面", "被人", "圈子", "消息", "评论", "点赞"}, - # Growth / 成长 - "工作": {"会议", "项目", "客户", "汇报", "deadline", "同事", - "老板", "薪资", "合同", "需求", "加班", "实习"}, - "学习": {"课", "考试", "论文", "笔记", "作业", "教授", "讲座", - "分数", "选课", "学分"}, - "求职": {"面试", "简历", "offer", "投递", "薪资", "岗位"}, - # Health / 身心 - "健康": {"医院", "复查", "吃药", "抽血", "手术", "心率", - "病", "症状", "指标", "体检", "月经"}, - "心理": {"焦虑", "抑郁", "恐慌", "创伤", "人格", "咨询", - "安全感", "自残", "崩溃", "压力"}, - "睡眠": {"睡", "失眠", "噩梦", "清醒", "熬夜", "早起", "午觉"}, - # Interests / 兴趣 - "游戏": {"游戏", "steam", "极乐迪斯科", "存档", "通关", "角色", - "mod", "DLC", "剧情"}, - "影视": {"电影", "番剧", "动漫", "剧", "综艺", "追番", "上映"}, - "音乐": {"歌", "音乐", "专辑", "live", "演唱会", "耳机"}, - "阅读": {"书", "小说", "读完", "kindle", "连载", "漫画"}, - "创作": {"写", "画", "预设", "脚本", "视频", "剪辑", "P图", - "SillyTavern", "插件", "正则", "人设"}, - # Digital / 数字 - "编程": {"代码", "code", "python", "bug", "api", "docker", - "git", "调试", "框架", "部署", "开发", "server"}, - "AI": {"模型", "GPT", "Claude", "gemini", "LLM", "token", - "prompt", "LoRA", "微调", "推理", "MCP"}, - "网络": {"VPN", "梯子", "代理", "域名", "隧道", "服务器", - "cloudflare", "tunnel", "反代"}, - # Affairs / 事务 - "财务": {"钱", "转账", "工资", "花了", "欠", "还款", "借", - "账单", "余额", "预算", "黄金"}, - "计划": {"计划", "目标", "deadline", "日程", "清单", "安排"}, - "待办": {"要做", "记得", "别忘", "提醒", "下次"}, - # Inner / 内心 - "情绪": {"开心", "难过", "生气", "哭", "泪", "孤独", "幸福", - "伤心", "烦", "委屈", "感动", "温柔"}, - "回忆": {"以前", "小时候", "那时", "怀念", "曾经", "记得"}, - "梦境": {"梦", "梦到", "梦见", "噩梦", "清醒梦"}, - "自省": {"反思", "觉得自己", "问自己", "意识到", "明白了"}, - } - - matched_domains = [] - for domain, kws in domain_keywords.items(): - hits = sum(1 for kw in kws if kw in text_lower) - if hits >= 2: - matched_domains.append((domain, hits)) - matched_domains.sort(key=lambda x: x[1], reverse=True) - domains = [d for d, _ in matched_domains[:3]] or ["未分类"] - - # --- Emotion estimation via simple sentiment dictionary --- - # --- 情感坐标估算：基于简单情感词典 --- - positive_words = {"开心", "高兴", "喜欢", "哈哈", "棒", "赞", "爱", - "幸福", "成功", "感动", "兴奋", "棒极了", - "happy", "love", "great", "awesome", "nice"} - negative_words = {"难过", "伤心", "生气", "焦虑", "害怕", "无聊", - "烦", "累", "失望", "崩溃", "愤怒", "痛苦", - "sad", "angry", "hate", "tired", "afraid"} - intense_words = {"太", "非常", "极", "超", "特别", "十分", "炸", - "崩溃", "激动", "愤怒", "狂喜", "very", "so", "extremely"} - - pos_count = sum(1 for w in positive_words if w in text_lower) - neg_count = sum(1 for w in negative_words if w in text_lower) - intense_count = sum(1 for w in intense_words if w in text_lower) - - # valence: positive/negative emotion balance - if pos_count + neg_count > 0: - valence = 0.5 + 0.4 * (pos_count - neg_count) / (pos_count + neg_count) - else: - valence = 0.5 - - # arousal: intensity level - arousal = min(1.0, 0.3 + intense_count * 0.15 + (pos_count + neg_count) * 0.08) - - return { - "domain": domains, - "valence": round(max(0.0, min(1.0, valence)), 2), - "arousal": round(max(0.0, min(1.0, arousal)), 2), - "tags": keywords[:5], - "suggested_name": "", - } - # --------------------------------------------------------- # Default analysis result (empty content or total failure) # 默认分析结果（内容为空或完全失败时用） @@ -635,21 +521,18 @@ class Dehydrator: if not content or not content.strip(): return [] - # --- Try API digest first (best quality, understands semantic splits) --- - # --- 优先 API 整理 --- - if self.api_available: - try: - result = await self._api_digest(content) - if result: - return result - except Exception as e: - logger.warning( - f"API diary digest failed, degrading to local / " - f"API 日记整理失败，降级到本地拆分: {e}" - ) - - # --- Local split fallback / 本地拆分兜底 --- - return await self._local_digest(content) + # --- API digest (no local fallback) --- + if not self.api_available: + raise RuntimeError("脱水 API 不可用，请检查 config.yaml 中的 dehydration 配置") + try: + result = await self._api_digest(content) + if result: + return result + raise RuntimeError("API 日记整理返回空结果") + except RuntimeError: + raise + except Exception as e: + raise RuntimeError(f"API 日记整理失败，请检查 API 连接: {e}") from e # --------------------------------------------------------- # API call: diary digest @@ -667,7 +550,7 @@ class Dehydrator: {"role": "user", "content": content[:5000]}, ], max_tokens=2048, - temperature=0.2, + temperature=0.0, ) if not response.choices: return [] @@ -717,50 +600,7 @@ class Dehydrator: "domain": item.get("domain", ["未分类"])[:3], "valence": valence, "arousal": arousal, - "tags": item.get("tags", [])[:5], + "tags": item.get("tags", [])[:15], "importance": importance, }) return validated - - # --------------------------------------------------------- - # Local diary split (fallback when API is unavailable) - # 本地日记拆分（无 API 时的兜底） - # Split by blank lines/separators, analyze each segment - # 按空行/分隔符拆段，每段独立分析 - # --------------------------------------------------------- - async def _local_digest(self, content: str) -> list[dict]: - """ - Local paragraph split + per-segment analysis. - 本地按段落拆分 + 逐段分析。 - """ - # Split by blank lines or separators / 按空行或分隔线拆分 - segments = re.split(r"\n{2,}|---+|\n-\s", content) - segments = [s.strip() for s in segments if len(s.strip()) > 20] - - if not segments: - # Content too short, treat as single entry - # 内容太短，整个作为一个条目 - analysis = self._local_analyze(content) - return [{ - "name": analysis.get("suggested_name", "日记"), - "content": content.strip(), - "domain": analysis["domain"], - "valence": analysis["valence"], - "arousal": analysis["arousal"], - "tags": analysis["tags"], - "importance": 5, - }] - - items = [] - for seg in segments[:10]: # Max 10 segments / 最多 10 段 - analysis = self._local_analyze(seg) - items.append({ - "name": analysis.get("suggested_name", "") or seg[:10], - "content": seg, - "domain": analysis["domain"], - "valence": analysis["valence"], - "arousal": analysis["arousal"], - "tags": analysis["tags"], - "importance": 5, - }) - return items diff --git a/docker-compose.user.yml b/docker-compose.user.yml new file mode 100644 index 0000000..a3141db --- /dev/null +++ b/docker-compose.user.yml @@ -0,0 +1,25 @@ +# ============================================================ +# Ombre Brain — 用户快速部署版 +# User Quick Deploy (pre-built image, no local build needed) +# +# 使用方法 / Usage: +# 1. 创建 .env: echo "OMBRE_API_KEY=your-key" > .env +# 2. 按需修改下面的 volumes 路径 +# 3. docker compose -f docker-compose.user.yml up -d +# ============================================================ + +services: + ombre-brain: + image: p0luz/ombre-brain:latest + container_name: ombre-brain + restart: unless-stopped + ports: + - "8000:8000" + environment: + - OMBRE_API_KEY=${OMBRE_API_KEY} + - OMBRE_TRANSPORT=streamable-http + - OMBRE_BUCKETS_DIR=/data + volumes: + # 改成你的 Obsidian Vault 路径，或保持 ./buckets 用本地目录 + # Change to your Obsidian Vault path, or keep ./buckets for local storage + - ./buckets:/data diff --git a/embedding_engine.py b/embedding_engine.py new file mode 100644 index 0000000..5de8c7d --- /dev/null +++ b/embedding_engine.py @@ -0,0 +1,188 @@ +# ============================================================ +# Module: Embedding Engine (embedding_engine.py) +# 模块：向量化引擎 +# +# Generates embeddings via Gemini API (OpenAI-compatible), +# stores them in SQLite, and provides cosine similarity search. +# 通过 Gemini API（OpenAI 兼容）生成 embedding， +# 存储在 SQLite 中，提供余弦相似度搜索。 +# +# Depended on by: server.py, bucket_manager.py +# 被谁依赖：server.py, bucket_manager.py +# ============================================================ + +import os +import json +import math +import sqlite3 +import logging +import asyncio +from pathlib import Path + +from openai import AsyncOpenAI + +logger = logging.getLogger("ombre_brain.embedding") + + +class EmbeddingEngine: + """ + Embedding generation + SQLite vector storage + cosine search. + 向量生成 + SQLite 向量存储 + 余弦搜索。 + """ + + def __init__(self, config: dict): + dehy_cfg = config.get("dehydration", {}) + embed_cfg = config.get("embedding", {}) + + self.api_key = dehy_cfg.get("api_key", "") + self.base_url = dehy_cfg.get("base_url", "https://generativelanguage.googleapis.com/v1beta/openai/") + self.model = embed_cfg.get("model", "gemini-embedding-001") + self.enabled = bool(self.api_key) and embed_cfg.get("enabled", True) + + # --- SQLite path: buckets_dir/embeddings.db --- + db_path = os.path.join(config["buckets_dir"], "embeddings.db") + self.db_path = db_path + + # --- Initialize client --- + if self.enabled: + self.client = AsyncOpenAI( + api_key=self.api_key, + base_url=self.base_url, + timeout=30.0, + ) + else: + self.client = None + + # --- Initialize SQLite --- + self._init_db() + + def _init_db(self): + """Create embeddings table if not exists.""" + os.makedirs(os.path.dirname(self.db_path), exist_ok=True) + conn = sqlite3.connect(self.db_path) + conn.execute(""" + CREATE TABLE IF NOT EXISTS embeddings ( + bucket_id TEXT PRIMARY KEY, + embedding TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + """) + conn.commit() + conn.close() + + async def generate_and_store(self, bucket_id: str, content: str) -> bool: + """ + Generate embedding for content and store in SQLite. + 为内容生成 embedding 并存入 SQLite。 + Returns True on success, False on failure. + """ + if not self.enabled or not content or not content.strip(): + return False + + try: + embedding = await self._generate_embedding(content) + if not embedding: + return False + self._store_embedding(bucket_id, embedding) + return True + except Exception as e: + logger.warning(f"Embedding generation failed for {bucket_id}: {e}") + return False + + async def _generate_embedding(self, text: str) -> list[float]: + """Call API to generate embedding vector.""" + # Truncate to avoid token limits + truncated = text[:2000] + try: + response = await self.client.embeddings.create( + model=self.model, + input=truncated, + ) + if response.data and len(response.data) > 0: + return response.data[0].embedding + return [] + except Exception as e: + logger.warning(f"Embedding API call failed: {e}") + return [] + + def _store_embedding(self, bucket_id: str, embedding: list[float]): + """Store embedding in SQLite.""" + from utils import now_iso + conn = sqlite3.connect(self.db_path) + conn.execute( + "INSERT OR REPLACE INTO embeddings (bucket_id, embedding, updated_at) VALUES (?, ?, ?)", + (bucket_id, json.dumps(embedding), now_iso()), + ) + conn.commit() + conn.close() + + def delete_embedding(self, bucket_id: str): + """Remove embedding when bucket is deleted.""" + conn = sqlite3.connect(self.db_path) + conn.execute("DELETE FROM embeddings WHERE bucket_id = ?", (bucket_id,)) + conn.commit() + conn.close() + + async def get_embedding(self, bucket_id: str) -> list[float] | None: + """Retrieve stored embedding for a bucket. Returns None if not found.""" + conn = sqlite3.connect(self.db_path) + row = conn.execute( + "SELECT embedding FROM embeddings WHERE bucket_id = ?", (bucket_id,) + ).fetchone() + conn.close() + if row: + try: + return json.loads(row[0]) + except json.JSONDecodeError: + return None + return None + + async def search_similar(self, query: str, top_k: int = 10) -> list[tuple[str, float]]: + """ + Search for buckets similar to query text. + Returns list of (bucket_id, similarity_score) sorted by score desc. + 搜索与查询文本相似的桶。返回 (bucket_id, 相似度分数) 列表。 + """ + if not self.enabled: + return [] + + try: + query_embedding = await self._generate_embedding(query) + if not query_embedding: + return [] + except Exception as e: + logger.warning(f"Query embedding failed: {e}") + return [] + + # Load all embeddings from SQLite + conn = sqlite3.connect(self.db_path) + rows = conn.execute("SELECT bucket_id, embedding FROM embeddings").fetchall() + conn.close() + + if not rows: + return [] + + # Calculate cosine similarity + results = [] + for bucket_id, emb_json in rows: + try: + stored_embedding = json.loads(emb_json) + sim = self._cosine_similarity(query_embedding, stored_embedding) + results.append((bucket_id, sim)) + except (json.JSONDecodeError, Exception): + continue + + results.sort(key=lambda x: x[1], reverse=True) + return results[:top_k] + + @staticmethod + def _cosine_similarity(a: list[float], b: list[float]) -> float: + """Calculate cosine similarity between two vectors.""" + if len(a) != len(b) or not a: + return 0.0 + dot = sum(x * y for x, y in zip(a, b)) + norm_a = math.sqrt(sum(x * x for x in a)) + norm_b = math.sqrt(sum(x * x for x in b)) + if norm_a == 0 or norm_b == 0: + return 0.0 + return dot / (norm_a * norm_b) diff --git a/import_memory.py b/import_memory.py new file mode 100644 index 0000000..9339b2e --- /dev/null +++ b/import_memory.py @@ -0,0 +1,758 @@ +# ============================================================ +# Module: Memory Import Engine (import_memory.py) +# 模块：历史记忆导入引擎 +# +# Imports conversation history from various platforms into OB. +# 将各平台对话历史导入 OB 记忆系统。 +# +# Supports: Claude JSON, ChatGPT export, DeepSeek, Markdown, plain text +# 支持格式：Claude JSON、ChatGPT 导出、DeepSeek、Markdown、纯文本 +# +# Features: +# - Chunked processing with resume support +# - Progress persistence (import_state.json) +# - Raw preservation mode for special contexts +# - Post-import frequency pattern detection +# ============================================================ + +import os +import json +import hashlib +import logging +import asyncio +from datetime import datetime +from pathlib import Path +from typing import Optional + +from utils import count_tokens_approx, now_iso + +logger = logging.getLogger("ombre_brain.import") + + +# ============================================================ +# Format Parsers — normalize any format to conversation turns +# 格式解析器 — 将任意格式标准化为对话轮次 +# ============================================================ + +def _parse_claude_json(data: dict | list) -> list[dict]: + """Parse Claude.ai export JSON → [{role, content, timestamp}, ...]""" + turns = [] + conversations = data if isinstance(data, list) else [data] + for conv in conversations: + messages = conv.get("chat_messages", conv.get("messages", [])) + for msg in messages: + if not isinstance(msg, dict): + continue + content = msg.get("text", msg.get("content", "")) + if isinstance(content, list): + content = " ".join( + p.get("text", "") for p in content if isinstance(p, dict) + ) + if not content or not content.strip(): + continue + role = msg.get("sender", msg.get("role", "user")) + ts = msg.get("created_at", msg.get("timestamp", "")) + turns.append({"role": role, "content": content.strip(), "timestamp": ts}) + return turns + + +def _parse_chatgpt_json(data: list | dict) -> list[dict]: + """Parse ChatGPT export JSON → [{role, content, timestamp}, ...]""" + turns = [] + conversations = data if isinstance(data, list) else [data] + for conv in conversations: + mapping = conv.get("mapping", {}) + if mapping: + # ChatGPT uses a tree structure with mapping + sorted_nodes = sorted( + mapping.values(), + key=lambda n: n.get("message", {}).get("create_time", 0) or 0, + ) + for node in sorted_nodes: + msg = node.get("message") + if not msg or not isinstance(msg, dict): + continue + content_parts = msg.get("content", {}).get("parts", []) + content = " ".join(str(p) for p in content_parts if p) + if not content.strip(): + continue + role = msg.get("author", {}).get("role", "user") + ts = msg.get("create_time", "") + if isinstance(ts, (int, float)): + ts = datetime.fromtimestamp(ts).isoformat() + turns.append({"role": role, "content": content.strip(), "timestamp": str(ts)}) + else: + # Simpler format: list of messages + messages = conv.get("messages", []) + for msg in messages: + if not isinstance(msg, dict): + continue + content = msg.get("content", msg.get("text", "")) + if isinstance(content, dict): + content = " ".join(str(p) for p in content.get("parts", [])) + if not content or not content.strip(): + continue + role = msg.get("role", msg.get("author", {}).get("role", "user")) + ts = msg.get("timestamp", msg.get("create_time", "")) + turns.append({"role": role, "content": content.strip(), "timestamp": str(ts)}) + return turns + + +def _parse_markdown(text: str) -> list[dict]: + """Parse Markdown/plain text → [{role, content, timestamp}, ...]""" + # Try to detect conversation patterns + lines = text.split("\n") + turns = [] + current_role = "user" + current_content = [] + + for line in lines: + stripped = line.strip() + # Detect role switches + if stripped.lower().startswith(("human:", "user:", "你:", "我:")): + if current_content: + turns.append({"role": current_role, "content": "\n".join(current_content).strip(), "timestamp": ""}) + current_role = "user" + content_after = stripped.split(":", 1)[1].strip() if ":" in stripped else "" + current_content = [content_after] if content_after else [] + elif stripped.lower().startswith(("assistant:", "claude:", "ai:", "gpt:", "bot:", "deepseek:")): + if current_content: + turns.append({"role": current_role, "content": "\n".join(current_content).strip(), "timestamp": ""}) + current_role = "assistant" + content_after = stripped.split(":", 1)[1].strip() if ":" in stripped else "" + current_content = [content_after] if content_after else [] + else: + current_content.append(line) + + if current_content: + content = "\n".join(current_content).strip() + if content: + turns.append({"role": current_role, "content": content, "timestamp": ""}) + + # If no role patterns detected, treat entire text as one big chunk + if not turns: + turns = [{"role": "user", "content": text.strip(), "timestamp": ""}] + + return turns + + +def detect_and_parse(raw_content: str, filename: str = "") -> list[dict]: + """ + Auto-detect format and parse to normalized turns. + 自动检测格式并解析为标准化的对话轮次。 + """ + ext = Path(filename).suffix.lower() if filename else "" + + # Try JSON first + if ext in (".json", "") or raw_content.strip().startswith(("{", "[")): + try: + data = json.loads(raw_content) + # Detect Claude vs ChatGPT format + if isinstance(data, list): + sample = data[0] if data else {} + else: + sample = data + + if isinstance(sample, dict): + if "chat_messages" in sample: + return _parse_claude_json(data) + if "mapping" in sample: + return _parse_chatgpt_json(data) + if "messages" in sample: + # Could be either — try ChatGPT first, fall back to Claude + msgs = sample["messages"] + if msgs and isinstance(msgs[0], dict) and "content" in msgs[0]: + if isinstance(msgs[0]["content"], dict): + return _parse_chatgpt_json(data) + return _parse_claude_json(data) + # Single conversation object with role/content messages + if "role" in sample and "content" in sample: + return _parse_claude_json(data) + except (json.JSONDecodeError, KeyError, IndexError): + pass + + # Fall back to markdown/text + return _parse_markdown(raw_content) + + +# ============================================================ +# Chunking — split turns into ~10k token windows +# 分窗 — 按对话轮次边界切为 ~10k token 窗口 +# ============================================================ + +def chunk_turns(turns: list[dict], target_tokens: int = 10000) -> list[dict]: + """ + Group conversation turns into chunks of ~target_tokens. + Returns list of {content, timestamp_start, timestamp_end, turn_count}. + 按对话轮次边界将对话分为 ~target_tokens 大小的窗口。 + """ + chunks = [] + current_lines = [] + current_tokens = 0 + first_ts = "" + last_ts = "" + turn_count = 0 + + for turn in turns: + role_label = "用户" if turn["role"] in ("user", "human") else "AI" + line = f"[{role_label}] {turn['content']}" + line_tokens = count_tokens_approx(line) + + # If single turn exceeds target, split it + if line_tokens > target_tokens * 1.5: + # Flush current + if current_lines: + chunks.append({ + "content": "\n".join(current_lines), + "timestamp_start": first_ts, + "timestamp_end": last_ts, + "turn_count": turn_count, + }) + current_lines = [] + current_tokens = 0 + turn_count = 0 + first_ts = "" + + # Add oversized turn as its own chunk + chunks.append({ + "content": line, + "timestamp_start": turn.get("timestamp", ""), + "timestamp_end": turn.get("timestamp", ""), + "turn_count": 1, + }) + continue + + if current_tokens + line_tokens > target_tokens and current_lines: + chunks.append({ + "content": "\n".join(current_lines), + "timestamp_start": first_ts, + "timestamp_end": last_ts, + "turn_count": turn_count, + }) + current_lines = [] + current_tokens = 0 + turn_count = 0 + first_ts = "" + + if not first_ts: + first_ts = turn.get("timestamp", "") + last_ts = turn.get("timestamp", "") + current_lines.append(line) + current_tokens += line_tokens + turn_count += 1 + + if current_lines: + chunks.append({ + "content": "\n".join(current_lines), + "timestamp_start": first_ts, + "timestamp_end": last_ts, + "turn_count": turn_count, + }) + + return chunks + + +# ============================================================ +# Import State — persistent progress tracking +# 导入状态 — 持久化进度追踪 +# ============================================================ + +class ImportState: + """Manages import progress with file-based persistence.""" + + def __init__(self, state_dir: str): + self.state_file = os.path.join(state_dir, "import_state.json") + self.data = { + "source_file": "", + "source_hash": "", + "total_chunks": 0, + "processed": 0, + "api_calls": 0, + "memories_created": 0, + "memories_merged": 0, + "memories_raw": 0, + "errors": [], + "status": "idle", # idle | running | paused | completed | error + "started_at": "", + "updated_at": "", + } + + def load(self) -> bool: + """Load state from file. Returns True if state exists.""" + if os.path.exists(self.state_file): + try: + with open(self.state_file, "r", encoding="utf-8") as f: + saved = json.load(f) + self.data.update(saved) + return True + except (json.JSONDecodeError, OSError): + return False + return False + + def save(self): + """Persist state to file.""" + self.data["updated_at"] = now_iso() + os.makedirs(os.path.dirname(self.state_file), exist_ok=True) + tmp = self.state_file + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(self.data, f, ensure_ascii=False, indent=2) + os.replace(tmp, self.state_file) + + def reset(self, source_file: str, source_hash: str, total_chunks: int): + """Reset state for a new import.""" + self.data = { + "source_file": source_file, + "source_hash": source_hash, + "total_chunks": total_chunks, + "processed": 0, + "api_calls": 0, + "memories_created": 0, + "memories_merged": 0, + "memories_raw": 0, + "errors": [], + "status": "running", + "started_at": now_iso(), + "updated_at": now_iso(), + } + + @property + def can_resume(self) -> bool: + return self.data["status"] in ("paused", "running") and self.data["processed"] < self.data["total_chunks"] + + def to_dict(self) -> dict: + return dict(self.data) + + +# ============================================================ +# Import extraction prompt +# 导入提取提示词 +# ============================================================ + +IMPORT_EXTRACT_PROMPT = """你是一个对话记忆提取专家。从以下对话片段中提取值得长期记住的信息。 + +提取规则： +1. 提取用户的事实、偏好、习惯、重要事件、情感时刻 +2. 同一话题的零散信息整合为一条记忆 +3. 过滤掉纯技术调试输出、代码块、重复问答、无意义寒暄 +4. 如果对话中有特殊暗号、仪式性行为、关键承诺等，标记 preserve_raw=true +5. 如果内容是用户和AI之间的习惯性互动模式（例如打招呼方式、告别习惯），标记 is_pattern=true +6. 每条记忆不少于30字 +7. 总条目数控制在 0~5 个（没有值得记的就返回空数组） +8. 在 content 中对人名、地名、专有名词用 [[双链]] 标记 + +输出格式（纯 JSON 数组，无其他内容）： +[ + { + "name": "条目标题（10字以内）", + "content": "整理后的内容", + "domain": ["主题域1"], + "valence": 0.7, + "arousal": 0.4, + "tags": ["核心词1", "核心词2", "扩展词1"], + "importance": 5, + "preserve_raw": false, + "is_pattern": false + } +] + +主题域可选（选 1~2 个）： + 日常: ["饮食", "穿搭", "出行", "居家", "购物"] + 人际: ["家庭", "恋爱", "友谊", "社交"] + 成长: ["工作", "学习", "考试", "求职"] + 身心: ["健康", "心理", "睡眠", "运动"] + 兴趣: ["游戏", "影视", "音乐", "阅读", "创作", "手工"] + 数字: ["编程", "AI", "硬件", "网络"] + 事务: ["财务", "计划", "待办"] + 内心: ["情绪", "回忆", "梦境", "自省"] + +importance: 1-10 +valence: 0~1（0=消极, 0.5=中性, 1=积极） +arousal: 0~1（0=平静, 0.5=普通, 1=激动） +preserve_raw: true = 特殊情境/暗号/仪式，保留原文不摘要 +is_pattern: true = 反复出现的习惯性行为模式""" + + +# ============================================================ +# Import Engine — core processing logic +# 导入引擎 — 核心处理逻辑 +# ============================================================ + +class ImportEngine: + """ + Processes conversation history files into OB memory buckets. + 将对话历史文件处理为 OB 记忆桶。 + """ + + def __init__(self, config: dict, bucket_mgr, dehydrator, embedding_engine=None): + self.config = config + self.bucket_mgr = bucket_mgr + self.dehydrator = dehydrator + self.embedding_engine = embedding_engine + self.state = ImportState(config["buckets_dir"]) + self._paused = False + self._running = False + self._chunks: list[dict] = [] + + @property + def is_running(self) -> bool: + return self._running + + def pause(self): + """Request pause — will stop after current chunk finishes.""" + self._paused = True + + def get_status(self) -> dict: + """Get current import status.""" + return self.state.to_dict() + + async def start( + self, + raw_content: str, + filename: str = "", + preserve_raw: bool = False, + resume: bool = False, + ) -> dict: + """ + Start or resume an import. + 开始或恢复导入。 + """ + if self._running: + return {"error": "Import already running"} + + self._running = True + self._paused = False + + try: + source_hash = hashlib.sha256(raw_content.encode()).hexdigest()[:16] + + # Check for resume + if resume and self.state.load() and self.state.can_resume: + if self.state.data["source_hash"] == source_hash: + logger.info(f"Resuming import from chunk {self.state.data['processed']}/{self.state.data['total_chunks']}") + # Re-parse and re-chunk to get the same chunks + turns = detect_and_parse(raw_content, filename) + self._chunks = chunk_turns(turns) + self.state.data["status"] = "running" + self.state.save() + return await self._process_chunks(preserve_raw) + else: + logger.warning("Source file changed, starting fresh import") + + # Fresh import + turns = detect_and_parse(raw_content, filename) + if not turns: + self._running = False + return {"error": "No conversation turns found in file"} + + self._chunks = chunk_turns(turns) + if not self._chunks: + self._running = False + return {"error": "No processable chunks after splitting"} + + self.state.reset(filename, source_hash, len(self._chunks)) + self.state.save() + + logger.info(f"Starting import: {len(turns)} turns → {len(self._chunks)} chunks") + return await self._process_chunks(preserve_raw) + + except Exception as e: + self.state.data["status"] = "error" + self.state.data["errors"].append(str(e)) + self.state.save() + self._running = False + raise + + async def _process_chunks(self, preserve_raw: bool) -> dict: + """Process chunks from current position.""" + start_idx = self.state.data["processed"] + + for i in range(start_idx, len(self._chunks)): + if self._paused: + self.state.data["status"] = "paused" + self.state.save() + self._running = False + logger.info(f"Import paused at chunk {i}/{len(self._chunks)}") + return self.state.to_dict() + + chunk = self._chunks[i] + try: + await self._process_single_chunk(chunk, preserve_raw) + except Exception as e: + err_msg = f"Chunk {i}: {str(e)[:200]}" + logger.warning(f"Import chunk error: {err_msg}") + if len(self.state.data["errors"]) < 100: + self.state.data["errors"].append(err_msg) + + self.state.data["processed"] = i + 1 + # Save progress every chunk + self.state.save() + + self.state.data["status"] = "completed" + self.state.save() + self._running = False + logger.info(f"Import completed: {self.state.data['memories_created']} created, {self.state.data['memories_merged']} merged") + return self.state.to_dict() + + async def _process_single_chunk(self, chunk: dict, preserve_raw: bool): + """Extract memories from a single chunk and store them.""" + content = chunk["content"] + if not content.strip(): + return + + # --- LLM extraction --- + try: + items = await self._extract_memories(content) + self.state.data["api_calls"] += 1 + except Exception as e: + logger.warning(f"LLM extraction failed: {e}") + self.state.data["api_calls"] += 1 + return + + if not items: + return + + # --- Store each extracted memory --- + for item in items: + try: + should_preserve = preserve_raw or item.get("preserve_raw", False) + + if should_preserve: + # Raw mode: store original content without summarization + bucket_id = await self.bucket_mgr.create( + content=item["content"], + tags=item.get("tags", []), + importance=item.get("importance", 5), + domain=item.get("domain", ["未分类"]), + valence=item.get("valence", 0.5), + arousal=item.get("arousal", 0.3), + name=item.get("name"), + ) + if self.embedding_engine: + try: + await self.embedding_engine.generate_and_store(bucket_id, item["content"]) + except Exception: + pass + self.state.data["memories_raw"] += 1 + self.state.data["memories_created"] += 1 + else: + # Normal mode: go through merge-or-create pipeline + is_merged = await self._merge_or_create_item(item) + if is_merged: + self.state.data["memories_merged"] += 1 + else: + self.state.data["memories_created"] += 1 + + # Patch timestamp if available + if chunk.get("timestamp_start"): + # We don't have update support for created, so skip + pass + + except Exception as e: + logger.warning(f"Failed to store memory: {item.get('name', '?')}: {e}") + + async def _extract_memories(self, chunk_content: str) -> list[dict]: + """Use LLM to extract memories from a conversation chunk.""" + if not self.dehydrator.api_available: + raise RuntimeError("API not available") + + response = await self.dehydrator.client.chat.completions.create( + model=self.dehydrator.model, + messages=[ + {"role": "system", "content": IMPORT_EXTRACT_PROMPT}, + {"role": "user", "content": chunk_content[:12000]}, + ], + max_tokens=2048, + temperature=0.0, + ) + + if not response.choices: + return [] + + raw = response.choices[0].message.content or "" + if not raw.strip(): + return [] + + return self._parse_extraction(raw) + + @staticmethod + def _parse_extraction(raw: str) -> list[dict]: + """Parse and validate LLM extraction result.""" + try: + cleaned = raw.strip() + if cleaned.startswith("```"): + cleaned = cleaned.split("\n", 1)[-1].rsplit("```", 1)[0] + items = json.loads(cleaned) + except (json.JSONDecodeError, IndexError, ValueError): + logger.warning(f"Import extraction JSON parse failed: {raw[:200]}") + return [] + + if not isinstance(items, list): + return [] + + validated = [] + for item in items: + if not isinstance(item, dict) or not item.get("content"): + continue + try: + importance = max(1, min(10, int(item.get("importance", 5)))) + except (ValueError, TypeError): + importance = 5 + try: + valence = max(0.0, min(1.0, float(item.get("valence", 0.5)))) + arousal = max(0.0, min(1.0, float(item.get("arousal", 0.3)))) + except (ValueError, TypeError): + valence, arousal = 0.5, 0.3 + + validated.append({ + "name": str(item.get("name", ""))[:20], + "content": str(item["content"]), + "domain": item.get("domain", ["未分类"])[:3], + "valence": valence, + "arousal": arousal, + "tags": [str(t) for t in item.get("tags", [])][:10], + "importance": importance, + "preserve_raw": bool(item.get("preserve_raw", False)), + "is_pattern": bool(item.get("is_pattern", False)), + }) + + return validated + + async def _merge_or_create_item(self, item: dict) -> bool: + """Try to merge with existing bucket, or create new. Returns is_merged.""" + content = item["content"] + domain = item.get("domain", ["未分类"]) + tags = item.get("tags", []) + importance = item.get("importance", 5) + valence = item.get("valence", 0.5) + arousal = item.get("arousal", 0.3) + name = item.get("name", "") + + try: + existing = await self.bucket_mgr.search(content, limit=1, domain_filter=domain or None) + except Exception: + existing = [] + + merge_threshold = self.config.get("merge_threshold", 75) + + if existing and existing[0].get("score", 0) > merge_threshold: + bucket = existing[0] + if not (bucket["metadata"].get("pinned") or bucket["metadata"].get("protected")): + try: + merged = await self.dehydrator.merge(bucket["content"], content) + self.state.data["api_calls"] += 1 + old_v = bucket["metadata"].get("valence", 0.5) + old_a = bucket["metadata"].get("arousal", 0.3) + await self.bucket_mgr.update( + bucket["id"], + content=merged, + tags=list(set(bucket["metadata"].get("tags", []) + tags)), + importance=max(bucket["metadata"].get("importance", 5), importance), + domain=list(set(bucket["metadata"].get("domain", []) + domain)), + valence=round((old_v + valence) / 2, 2), + arousal=round((old_a + arousal) / 2, 2), + ) + if self.embedding_engine: + try: + await self.embedding_engine.generate_and_store(bucket["id"], merged) + except Exception: + pass + return True + except Exception as e: + logger.warning(f"Merge failed during import: {e}") + self.state.data["api_calls"] += 1 + + # Create new + bucket_id = await self.bucket_mgr.create( + content=content, + tags=tags, + importance=importance, + domain=domain, + valence=valence, + arousal=arousal, + name=name or None, + ) + if self.embedding_engine: + try: + await self.embedding_engine.generate_and_store(bucket_id, content) + except Exception: + pass + return False + + async def detect_patterns(self) -> list[dict]: + """ + Post-import: detect high-frequency patterns via embedding clustering. + 导入后：通过 embedding 聚类检测高频模式。 + Returns list of {pattern_content, count, bucket_ids, suggested_action}. + """ + if not self.embedding_engine: + return [] + + all_buckets = await self.bucket_mgr.list_all(include_archive=False) + dynamic_buckets = [ + b for b in all_buckets + if b["metadata"].get("type") == "dynamic" + and not b["metadata"].get("pinned") + and not b["metadata"].get("resolved") + ] + + if len(dynamic_buckets) < 5: + return [] + + # Get embeddings + embeddings = {} + for b in dynamic_buckets: + emb = await self.embedding_engine.get_embedding(b["id"]) + if emb is not None: + embeddings[b["id"]] = emb + + if len(embeddings) < 5: + return [] + + # Find clusters: group by pairwise similarity > 0.7 + import numpy as np + ids = list(embeddings.keys()) + clusters: dict[str, list[str]] = {} + visited = set() + + for i, id_a in enumerate(ids): + if id_a in visited: + continue + cluster = [id_a] + visited.add(id_a) + emb_a = np.array(embeddings[id_a]) + norm_a = np.linalg.norm(emb_a) + if norm_a == 0: + continue + + for j in range(i + 1, len(ids)): + id_b = ids[j] + if id_b in visited: + continue + emb_b = np.array(embeddings[id_b]) + norm_b = np.linalg.norm(emb_b) + if norm_b == 0: + continue + sim = float(np.dot(emb_a, emb_b) / (norm_a * norm_b)) + if sim > 0.7: + cluster.append(id_b) + visited.add(id_b) + + if len(cluster) >= 3: + clusters[id_a] = cluster + + # Format results + patterns = [] + for lead_id, cluster_ids in clusters.items(): + lead_bucket = next((b for b in dynamic_buckets if b["id"] == lead_id), None) + if not lead_bucket: + continue + patterns.append({ + "pattern_content": lead_bucket["content"][:200], + "pattern_name": lead_bucket["metadata"].get("name", lead_id), + "count": len(cluster_ids), + "bucket_ids": cluster_ids, + "suggested_action": "pin" if len(cluster_ids) >= 5 else "review", + }) + + patterns.sort(key=lambda p: p["count"], reverse=True) + return patterns[:20] diff --git a/requirements.txt b/requirements.txt index 01a24bd..7591003 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,3 +23,7 @@ jieba>=0.42.1 # 异步 HTTP 客户端（应用层保活 ping） httpx>=0.27.0 + +# 向量相似度计算 (导入模式/聚类) +numpy>=1.24.0 +scikit-learn>=1.2.0 diff --git a/server.py b/server.py index 124d114..5d993bc 100644 --- a/server.py +++ b/server.py @@ -36,7 +36,7 @@ import random import logging import asyncio import httpx -from typing import Optional + # --- Ensure same-directory modules can be imported --- # --- 确保同目录下的模块能被正确导入 --- @@ -47,17 +47,21 @@ from mcp.server.fastmcp import FastMCP from bucket_manager import BucketManager from dehydrator import Dehydrator from decay_engine import DecayEngine -from utils import load_config, setup_logging +from embedding_engine import EmbeddingEngine +from import_memory import ImportEngine +from utils import load_config, setup_logging, strip_wikilinks, count_tokens_approx # --- Load config & init logging / 加载配置 & 初始化日志 --- config = load_config() setup_logging(config.get("log_level", "INFO")) logger = logging.getLogger("ombre_brain") -# --- Initialize three core components / 初始化三大核心组件 --- +# --- Initialize core components / 初始化核心组件 --- bucket_mgr = BucketManager(config) # Bucket manager / 记忆桶管理器 dehydrator = Dehydrator(config) # Dehydrator / 脱水器 decay_engine = DecayEngine(config, bucket_mgr) # Decay engine / 衰减引擎 +embedding_engine = EmbeddingEngine(config) # Embedding engine / 向量化引擎 +import_engine = ImportEngine(config, bucket_mgr, dehydrator, embedding_engine) # Import engine / 导入引擎 # --- Create MCP server instance / 创建 MCP 服务器实例 --- # host="0.0.0.0" so Docker container's SSE is externally reachable @@ -103,20 +107,37 @@ async def breath_hook(request): # top 2 unresolved by score unresolved = [b for b in all_buckets if not b["metadata"].get("resolved", False) - and b["metadata"].get("type") != "permanent" + and b["metadata"].get("type") not in ("permanent", "feel") and not b["metadata"].get("pinned") and not b["metadata"].get("protected")] scored = sorted(unresolved, key=lambda b: decay_engine.calculate_score(b["metadata"]), reverse=True) - top = scored[:2] parts = [] + token_budget = 10000 for b in pinned: - summary = await dehydrator.dehydrate(b["content"], {k: v for k, v in b["metadata"].items() if k != "tags"}) + summary = await dehydrator.dehydrate(strip_wikilinks(b["content"]), {k: v for k, v in b["metadata"].items() if k != "tags"}) parts.append(f"📌 [核心准则] {summary}") - for b in top: - summary = await dehydrator.dehydrate(b["content"], {k: v for k, v in b["metadata"].items() if k != "tags"}) - await bucket_mgr.touch(b["id"]) + token_budget -= count_tokens_approx(summary) + + # Diversity: top-1 fixed + shuffle rest from top-20 + candidates = list(scored) + if len(candidates) > 1: + top1 = [candidates[0]] + pool = candidates[1:min(20, len(candidates))] + random.shuffle(pool) + candidates = top1 + pool + candidates[min(20, len(candidates)):] + # Hard cap: max 20 surfacing buckets in hook + candidates = candidates[:20] + + for b in candidates: + if token_budget <= 0: + break + summary = await dehydrator.dehydrate(strip_wikilinks(b["content"]), {k: v for k, v in b["metadata"].items() if k != "tags"}) + summary_tokens = count_tokens_approx(summary) + if summary_tokens > token_budget: + break parts.append(summary) + token_budget -= summary_tokens if not parts: return PlainTextResponse("") @@ -126,6 +147,43 @@ async def breath_hook(request): return PlainTextResponse("") +# ============================================================= +# /dream-hook endpoint: Dedicated hook for Dreaming +# Dreaming 专用挂载点 +# ============================================================= +@mcp.custom_route("/dream-hook", methods=["GET"]) +async def dream_hook(request): + from starlette.responses import PlainTextResponse + try: + all_buckets = await bucket_mgr.list_all(include_archive=False) + candidates = [ + b for b in all_buckets + if b["metadata"].get("type") not in ("permanent", "feel") + and not b["metadata"].get("pinned", False) + and not b["metadata"].get("protected", False) + ] + candidates.sort(key=lambda b: b["metadata"].get("created", ""), reverse=True) + recent = candidates[:10] + + if not recent: + return PlainTextResponse("") + + parts = [] + for b in recent: + meta = b["metadata"] + resolved_tag = "[已解决]" if meta.get("resolved", False) else "[未解决]" + parts.append( + f"{meta.get('name', b['id'])} {resolved_tag} " + f"V{meta.get('valence', 0.5):.1f}/A{meta.get('arousal', 0.3):.1f}\n" + f"{strip_wikilinks(b['content'][:200])}" + ) + + return PlainTextResponse("[Ombre Brain - Dreaming]\n" + "\n---\n".join(parts)) + except Exception as e: + logger.warning(f"Dream hook failed: {e}") + return PlainTextResponse("") + + # ============================================================= # Internal helper: merge-or-create # 内部辅助：检查是否可合并，可以则合并，否则新建 @@ -160,15 +218,24 @@ async def _merge_or_create( if not (bucket["metadata"].get("pinned") or bucket["metadata"].get("protected")): try: merged = await dehydrator.merge(bucket["content"], content) + old_v = bucket["metadata"].get("valence", 0.5) + old_a = bucket["metadata"].get("arousal", 0.3) + merged_valence = round((old_v + valence) / 2, 2) + merged_arousal = round((old_a + arousal) / 2, 2) await bucket_mgr.update( bucket["id"], content=merged, tags=list(set(bucket["metadata"].get("tags", []) + tags)), importance=max(bucket["metadata"].get("importance", 5), importance), domain=list(set(bucket["metadata"].get("domain", []) + domain)), - valence=valence, - arousal=arousal, + valence=merged_valence, + arousal=merged_arousal, ) + # --- Update embedding after merge --- + try: + await embedding_engine.generate_and_store(bucket["id"], merged) + except Exception: + pass return bucket["metadata"].get("name", bucket["id"]), True except Exception as e: logger.warning(f"Merge failed, creating new / 合并失败，新建: {e}") @@ -182,6 +249,11 @@ async def _merge_or_create( arousal=arousal, name=name or None, ) + # --- Generate embedding for new bucket --- + try: + await embedding_engine.generate_and_store(bucket_id, content) + except Exception: + pass return bucket_id, False @@ -196,14 +268,17 @@ async def _merge_or_create( # ============================================================= @mcp.tool() async def breath( - query: Optional[str] = None, - max_results: int = 3, + query: str = "", + max_tokens: int = 10000, domain: str = "", valence: float = -1, arousal: float = -1, + max_results: int = 20, ) -> str: - """检索/浮现记忆。不传query或传空=自动浮现,有query=关键词检索。domain逗号分隔,valence/arousal 0~1(-1忽略)。""" + """检索/浮现记忆。不传query或传空=自动浮现,有query=关键词检索。max_tokens控制返回总token上限(默认10000)。domain逗号分隔,valence/arousal 0~1(-1忽略)。max_results控制返回数量上限(默认20,最大50)。""" await decay_engine.ensure_started() + max_results = min(max_results, 50) + max_tokens = min(max_tokens, 20000) # --- No args or empty query: surfacing mode (weight pool active push) --- # --- 无参数或空query：浮现模式（权重池主动推送）--- @@ -224,36 +299,68 @@ async def breath( for b in pinned_buckets: try: clean_meta = {k: v for k, v in b["metadata"].items() if k != "tags"} - summary = await dehydrator.dehydrate(b["content"], clean_meta) + summary = await dehydrator.dehydrate(strip_wikilinks(b["content"]), clean_meta) pinned_results.append(f"📌 [核心准则] {summary}") except Exception as e: logger.warning(f"Failed to dehydrate pinned bucket / 钉选桶脱水失败: {e}") continue - # --- Unresolved buckets: surface top 2 by weight --- - # --- 未解决桶：按权重浮现前 2 条 --- + # --- Unresolved buckets: surface top N by weight --- + # --- 未解决桶：按权重浮现前 N 条 --- unresolved = [ b for b in all_buckets if not b["metadata"].get("resolved", False) - and b["metadata"].get("type") != "permanent" + and b["metadata"].get("type") not in ("permanent", "feel") and not b["metadata"].get("pinned", False) and not b["metadata"].get("protected", False) ] + logger.info( + f"Breath surfacing: {len(all_buckets)} total, " + f"{len(pinned_buckets)} pinned, {len(unresolved)} unresolved" + ) + scored = sorted( unresolved, key=lambda b: decay_engine.calculate_score(b["metadata"]), reverse=True, ) - top = scored[:2] + + if scored: + top_scores = [(b["metadata"].get("name", b["id"]), decay_engine.calculate_score(b["metadata"])) for b in scored[:5]] + logger.info(f"Top unresolved scores: {top_scores}") + + # --- Token-budgeted surfacing with diversity + hard cap --- + # --- 按 token 预算浮现，带多样性 + 硬上限 --- + # Top-1 always surfaces; rest sampled from top-20 for diversity + token_budget = max_tokens + for r in pinned_results: + token_budget -= count_tokens_approx(r) + + candidates = list(scored) + if len(candidates) > 1: + # Ensure highest-score bucket is first, shuffle rest from top-20 + top1 = [candidates[0]] + pool = candidates[1:min(20, len(candidates))] + random.shuffle(pool) + candidates = top1 + pool + candidates[min(20, len(candidates)):] + # Hard cap: never surface more than max_results buckets + candidates = candidates[:max_results] + dynamic_results = [] - for b in top: + for b in candidates: + if token_budget <= 0: + break try: clean_meta = {k: v for k, v in b["metadata"].items() if k != "tags"} - summary = await dehydrator.dehydrate(b["content"], clean_meta) - await bucket_mgr.touch(b["id"]) + summary = await dehydrator.dehydrate(strip_wikilinks(b["content"]), clean_meta) + summary_tokens = count_tokens_approx(summary) + if summary_tokens > token_budget: + break + # NOTE: no touch() here — surfacing should NOT reset decay timer score = decay_engine.calculate_score(b["metadata"]) dynamic_results.append(f"[权重:{score:.2f}] {summary}") + token_budget -= summary_tokens except Exception as e: logger.warning(f"Failed to dehydrate surfaced bucket / 浮现脱水失败: {e}") continue @@ -268,7 +375,29 @@ async def breath( parts.append("=== 浮现记忆 ===\n" + "\n---\n".join(dynamic_results)) return "\n\n".join(parts) - # --- With args: search mode / 有参数：检索模式 --- + # --- Feel retrieval: domain="feel" is a special channel --- + # --- Feel 检索：domain="feel" 是独立入口 --- + if domain.strip().lower() == "feel": + try: + all_buckets = await bucket_mgr.list_all(include_archive=False) + feels = [b for b in all_buckets if b["metadata"].get("type") == "feel"] + feels.sort(key=lambda b: b["metadata"].get("created", ""), reverse=True) + if not feels: + return "没有留下过 feel。" + results = [] + for f in feels: + created = f["metadata"].get("created", "") + entry = f"[{created}]\n{strip_wikilinks(f['content'])}" + results.append(entry) + if count_tokens_approx("\n---\n".join(results)) > max_tokens: + break + return "=== 你留下的 feel ===\n" + "\n---\n".join(results) + except Exception as e: + logger.error(f"Feel retrieval failed: {e}") + return "读取 feel 失败。" + + # --- With args: search mode (keyword + vector dual channel) --- + # --- 有参数：检索模式（关键词 + 向量双通道）--- domain_filter = [d.strip() for d in domain.split(",") if d.strip()] or None q_valence = valence if 0 <= valence <= 1 else None q_arousal = arousal if 0 <= arousal <= 1 else None @@ -276,7 +405,7 @@ async def breath( try: matches = await bucket_mgr.search( query, - limit=max_results, + limit=max(max_results, 20), domain_filter=domain_filter, query_valence=q_valence, query_arousal=q_arousal, @@ -285,13 +414,48 @@ async def breath( logger.error(f"Search failed / 检索失败: {e}") return "检索过程出错，请稍后重试。" + # --- Exclude pinned/protected from search results (they surface in surfacing mode) --- + # --- 搜索模式排除钉选桶（它们在浮现模式中始终可见）--- + matches = [b for b in matches if not (b["metadata"].get("pinned") or b["metadata"].get("protected"))] + + # --- Vector similarity channel: find semantically related buckets --- + # --- 向量相似度通道：找到语义相关的桶 --- + matched_ids = {b["id"] for b in matches} + try: + vector_results = await embedding_engine.search_similar(query, top_k=max(max_results, 20)) + for bucket_id, sim_score in vector_results: + if bucket_id not in matched_ids and sim_score > 0.5: + bucket = await bucket_mgr.get(bucket_id) + if bucket and not (bucket["metadata"].get("pinned") or bucket["metadata"].get("protected")): + bucket["score"] = round(sim_score * 100, 2) + bucket["vector_match"] = True + matches.append(bucket) + matched_ids.add(bucket_id) + except Exception as e: + logger.warning(f"Vector search failed, using keyword only / 向量搜索失败: {e}") + results = [] + token_used = 0 for bucket in matches: + if token_used >= max_tokens: + break try: clean_meta = {k: v for k, v in bucket["metadata"].items() if k != "tags"} - summary = await dehydrator.dehydrate(bucket["content"], clean_meta) + # --- Memory reconstruction: shift displayed valence by current mood --- + # --- 记忆重构：根据当前情绪微调展示层 valence（±0.1）--- + if q_valence is not None and "valence" in clean_meta: + original_v = float(clean_meta.get("valence", 0.5)) + shift = (q_valence - 0.5) * 0.2 # ±0.1 max shift + clean_meta["valence"] = max(0.0, min(1.0, original_v + shift)) + summary = await dehydrator.dehydrate(strip_wikilinks(bucket["content"]), clean_meta) + summary_tokens = count_tokens_approx(summary) + if token_used + summary_tokens > max_tokens: + break await bucket_mgr.touch(bucket["id"]) + if bucket.get("vector_match"): + summary = f"[语义关联] {summary}" results.append(summary) + token_used += summary_tokens except Exception as e: logger.warning(f"Failed to dehydrate search result / 检索结果脱水失败: {e}") continue @@ -312,7 +476,7 @@ async def breath( drift_results = [] for b in drifted: clean_meta = {k: v for k, v in b["metadata"].items() if k != "tags"} - summary = await dehydrator.dehydrate(b["content"], clean_meta) + summary = await dehydrator.dehydrate(strip_wikilinks(b["content"]), clean_meta) drift_results.append(f"[surface_type: random]\n{summary}") results.append("--- 忽然想起来 ---\n" + "\n---\n".join(drift_results)) except Exception as e: @@ -334,8 +498,11 @@ async def hold( tags: str = "", importance: int = 5, pinned: bool = False, + feel: bool = False, + source_bucket: str = "", valence: float = -1, + arousal: float = -1, ) -> str: - """存储单条记忆,自动打标+合并。tags逗号分隔,importance 1-10。pinned=True创建永久钉选桶。""" + """存储单条记忆,自动打标+合并。tags逗号分隔,importance 1-10。pinned=True创建永久钉选桶。feel=True存储你的第一人称感受(不参与普通浮现)。source_bucket=被消化的记忆桶ID(feel模式下,标记源记忆为已消化)。""" await decay_engine.ensure_started() # --- Input validation / 输入校验 --- @@ -345,6 +512,38 @@ async def hold( importance = max(1, min(10, importance)) extra_tags = [t.strip() for t in tags.split(",") if t.strip()] + # --- Feel mode: store as feel type, minimal metadata --- + # --- Feel 模式：存为 feel 类型，最少元数据 --- + if feel: + # Feel valence/arousal = model's own perspective + feel_valence = valence if 0 <= valence <= 1 else 0.5 + feel_arousal = arousal if 0 <= arousal <= 1 else 0.3 + bucket_id = await bucket_mgr.create( + content=content, + tags=[], + importance=5, + domain=[], + valence=feel_valence, + arousal=feel_arousal, + name=None, + bucket_type="feel", + ) + try: + await embedding_engine.generate_and_store(bucket_id, content) + except Exception: + pass + # --- Mark source memory as digested + store model's valence perspective --- + # --- 标记源记忆为已消化 + 存储模型视角的 valence --- + if source_bucket and source_bucket.strip(): + try: + update_kwargs = {"digested": True} + if 0 <= valence <= 1: + update_kwargs["model_valence"] = feel_valence + await bucket_mgr.update(source_bucket.strip(), **update_kwargs) + except Exception as e: + logger.warning(f"Failed to mark source as digested / 标记已消化失败: {e}") + return f"🫧feel→{bucket_id}" + # --- Step 1: auto-tagging / 自动打标 --- try: analysis = await dehydrator.analyze(content) @@ -377,6 +576,10 @@ async def hold( bucket_type="permanent", pinned=True, ) + try: + await embedding_engine.generate_and_store(bucket_id, content) + except Exception: + pass return f"📌钉选→{bucket_id} {','.join(domain)}" # --- Step 2: merge or create / 合并或新建 --- @@ -494,9 +697,11 @@ async def trace( tags: str = "", resolved: int = -1, pinned: int = -1, + digested: int = -1, + content: str = "", delete: bool = False, ) -> str: - """修改记忆元数据。resolved=1沉底/0激活,pinned=1钉选/0取消,delete=True删除。只传需改的,-1或空=不改。""" + """修改记忆元数据或内容。resolved=1沉底/0激活,pinned=1钉选/0取消,digested=1隐藏(保留但不浮现)/0取消隐藏,content=替换桶正文,delete=True删除。只传需改的,-1或空=不改。""" if not bucket_id or not bucket_id.strip(): return "请提供有效的 bucket_id。" @@ -504,6 +709,8 @@ async def trace( # --- Delete mode / 删除模式 --- if delete: success = await bucket_mgr.delete(bucket_id) + if success: + embedding_engine.delete_embedding(bucket_id) return f"已遗忘记忆桶: {bucket_id}" if success else f"未找到记忆桶: {bucket_id}" bucket = await bucket_mgr.get(bucket_id) @@ -530,6 +737,10 @@ async def trace( updates["pinned"] = bool(pinned) if pinned == 1: updates["importance"] = 10 # pinned → lock importance + if digested in (0, 1): + updates["digested"] = bool(digested) + if content: + updates["content"] = content if not updates: return "没有任何字段需要修改。" @@ -538,7 +749,16 @@ async def trace( if not success: return f"修改失败: {bucket_id}" - changed = ", ".join(f"{k}={v}" for k, v in updates.items()) + # Re-generate embedding if content changed + if "content" in updates: + try: + await embedding_engine.generate_and_store(bucket_id, updates["content"]) + except Exception: + pass + + changed = ", ".join(f"{k}={v}" for k, v in updates.items() if k != "content") + if "content" in updates: + changed += (", content=已替换" if changed else "content=已替换") # Explicit hint about resolved state change semantics # 特别提示 resolved 状态变化的语义 if "resolved" in updates: @@ -546,6 +766,11 @@ async def trace( changed += " → 已沉底，只在关键词触发时重新浮现" else: changed += " → 已重新激活，将参与浮现排序" + if "digested" in updates: + if updates["digested"]: + changed += " → 已隐藏，保留但不再浮现" + else: + changed += " → 已取消隐藏，重新参与浮现" return f"已修改记忆桶 {bucket_id}: {changed}" @@ -586,6 +811,8 @@ async def pulse(include_archive: bool = False) -> str: icon = "📌" elif meta.get("type") == "permanent": icon = "📦" + elif meta.get("type") == "feel": + icon = "🫧" elif meta.get("type") == "archived": icon = "🗄️" elif meta.get("resolved", False): @@ -612,6 +839,610 @@ async def pulse(include_archive: bool = False) -> str: return status + "\n=== 记忆列表 ===\n" + "\n".join(lines) +# ============================================================= +# Tool 6: dream — Dreaming, digest recent memories +# 工具 6：dream — 做梦，消化最近的记忆 +# +# Reads recent surface-level buckets (≤10), returns them for +# Claude to introspect under prompt guidance. +# 读取最近新增的表层桶（≤10个），返回给 Claude 在提示词引导下自主思考。 +# Claude then decides: resolve some, write feels, or do nothing. +# ============================================================= +@mcp.tool() +async def dream() -> str: + """做梦——读取最近新增的记忆桶,供你自省。读完后可以trace(resolved=1)放下,或hold(feel=True)写感受。""" + await decay_engine.ensure_started() + + try: + all_buckets = await bucket_mgr.list_all(include_archive=False) + except Exception as e: + logger.error(f"Dream failed to list buckets: {e}") + return "记忆系统暂时无法访问。" + + # --- Filter: recent surface-level dynamic buckets (not permanent/pinned/feel) --- + candidates = [ + b for b in all_buckets + if b["metadata"].get("type") not in ("permanent", "feel") + and not b["metadata"].get("pinned", False) + and not b["metadata"].get("protected", False) + ] + + # --- Sort by creation time desc, take top 10 --- + candidates.sort(key=lambda b: b["metadata"].get("created", ""), reverse=True) + recent = candidates[:10] + + if not recent: + return "没有需要消化的新记忆。" + + parts = [] + for b in recent: + meta = b["metadata"] + resolved_tag = " [已解决]" if meta.get("resolved", False) else " [未解决]" + domains = ",".join(meta.get("domain", [])) + val = meta.get("valence", 0.5) + aro = meta.get("arousal", 0.3) + created = meta.get("created", "") + parts.append( + f"[{meta.get('name', b['id'])}]{resolved_tag} " + f"主题:{domains} V{val:.1f}/A{aro:.1f} " + f"创建:{created}\n" + f"ID: {b['id']}\n" + f"{strip_wikilinks(b['content'][:500])}" + ) + + header = ( + "=== Dreaming ===\n" + "以下是你最近的记忆。用第一人称想：\n" + "- 这些东西里有什么在你这里留下了重量？\n" + "- 有什么还没想清楚？\n" + "- 有什么可以放下了？\n" + "想完之后：值得放下的用 trace(bucket_id, resolved=1)；\n" + "有沉淀的用 hold(content=\"...\", feel=True, source_bucket=\"bucket_id\", valence=你的感受) 写下来。\n" + "valence 是你对这段记忆的感受，不是事件本身的情绪。\n" + "没有沉淀就不写，不强迫产出。\n" + ) + + # --- Connection hint: find most similar pair via embeddings --- + connection_hint = "" + if embedding_engine and embedding_engine.enabled and len(recent) >= 2: + try: + best_pair = None + best_sim = 0.0 + ids = [b["id"] for b in recent] + names = {b["id"]: b["metadata"].get("name", b["id"]) for b in recent} + embeddings = {} + for bid in ids: + emb = await embedding_engine.get_embedding(bid) + if emb is not None: + embeddings[bid] = emb + for i, id_a in enumerate(ids): + for id_b in ids[i+1:]: + if id_a in embeddings and id_b in embeddings: + sim = embedding_engine._cosine_similarity(embeddings[id_a], embeddings[id_b]) + if sim > best_sim: + best_sim = sim + best_pair = (id_a, id_b) + if best_pair and best_sim > 0.5: + connection_hint = ( + f"\n💭 [{names[best_pair[0]]}] 和 [{names[best_pair[1]]}] " + f"似乎有关联 (相似度:{best_sim:.2f})——不替你下结论，你自己想。\n" + ) + except Exception as e: + logger.warning(f"Dream connection hint failed: {e}") + + # --- Feel crystallization hint: detect repeated feel themes --- + crystal_hint = "" + if embedding_engine and embedding_engine.enabled: + try: + feels = [b for b in all_buckets if b["metadata"].get("type") == "feel"] + if len(feels) >= 3: + feel_embeddings = {} + for f in feels: + emb = await embedding_engine.get_embedding(f["id"]) + if emb is not None: + feel_embeddings[f["id"]] = emb + # Find clusters: feels with similarity > 0.7 to at least 2 others + for fid, femb in feel_embeddings.items(): + similar_feels = [] + for oid, oemb in feel_embeddings.items(): + if oid != fid: + sim = embedding_engine._cosine_similarity(femb, oemb) + if sim > 0.7: + similar_feels.append(oid) + if len(similar_feels) >= 2: + feel_bucket = next((f for f in feels if f["id"] == fid), None) + if feel_bucket and not feel_bucket["metadata"].get("pinned"): + content_preview = strip_wikilinks(feel_bucket["content"][:80]) + crystal_hint = ( + f"\n🔮 你已经写过 {len(similar_feels)+1} 条相似的 feel " + f"（围绕「{content_preview}…」）。" + f"如果这已经是确信而不只是感受了，" + f"你可以用 hold(content=\"...\", pinned=True) 升级它。" + f"不急，你自己决定。\n" + ) + break + except Exception as e: + logger.warning(f"Dream crystallization hint failed: {e}") + + return header + "\n---\n".join(parts) + connection_hint + crystal_hint + + +# ============================================================= +# Dashboard API endpoints (for lightweight Web UI) +# 仪表板 API（轻量 Web UI 用） +# ============================================================= +@mcp.custom_route("/api/buckets", methods=["GET"]) +async def api_buckets(request): + """List all buckets with metadata (no content for efficiency).""" + from starlette.responses import JSONResponse + try: + all_buckets = await bucket_mgr.list_all(include_archive=True) + result = [] + for b in all_buckets: + meta = b.get("metadata", {}) + result.append({ + "id": b["id"], + "name": meta.get("name", b["id"]), + "type": meta.get("type", "dynamic"), + "domain": meta.get("domain", []), + "tags": meta.get("tags", []), + "valence": meta.get("valence", 0.5), + "arousal": meta.get("arousal", 0.3), + "model_valence": meta.get("model_valence"), + "importance": meta.get("importance", 5), + "resolved": meta.get("resolved", False), + "pinned": meta.get("pinned", False), + "digested": meta.get("digested", False), + "created": meta.get("created", ""), + "last_active": meta.get("last_active", ""), + "activation_count": meta.get("activation_count", 1), + "score": decay_engine.calculate_score(meta), + "content_preview": strip_wikilinks(b.get("content", ""))[:200], + }) + result.sort(key=lambda x: x["score"], reverse=True) + return JSONResponse(result) + except Exception as e: + return JSONResponse({"error": str(e)}, status_code=500) + + +@mcp.custom_route("/api/bucket/{bucket_id}", methods=["GET"]) +async def api_bucket_detail(request): + """Get full bucket content by ID.""" + from starlette.responses import JSONResponse + bucket_id = request.path_params["bucket_id"] + bucket = await bucket_mgr.get(bucket_id) + if not bucket: + return JSONResponse({"error": "not found"}, status_code=404) + meta = bucket.get("metadata", {}) + return JSONResponse({ + "id": bucket["id"], + "metadata": meta, + "content": strip_wikilinks(bucket.get("content", "")), + "score": decay_engine.calculate_score(meta), + }) + + +@mcp.custom_route("/api/search", methods=["GET"]) +async def api_search(request): + """Search buckets by query.""" + from starlette.responses import JSONResponse + query = request.query_params.get("q", "") + if not query: + return JSONResponse({"error": "missing q parameter"}, status_code=400) + try: + matches = await bucket_mgr.search(query, limit=10) + result = [] + for b in matches: + meta = b.get("metadata", {}) + result.append({ + "id": b["id"], + "name": meta.get("name", b["id"]), + "score": b.get("score", 0), + "domain": meta.get("domain", []), + "valence": meta.get("valence", 0.5), + "arousal": meta.get("arousal", 0.3), + "content_preview": strip_wikilinks(b.get("content", ""))[:200], + }) + return JSONResponse(result) + except Exception as e: + return JSONResponse({"error": str(e)}, status_code=500) + + +@mcp.custom_route("/api/network", methods=["GET"]) +async def api_network(request): + """Get embedding similarity network for visualization.""" + from starlette.responses import JSONResponse + try: + all_buckets = await bucket_mgr.list_all(include_archive=False) + nodes = [] + edges = [] + embeddings = {} + + for b in all_buckets: + meta = b.get("metadata", {}) + bid = b["id"] + nodes.append({ + "id": bid, + "name": meta.get("name", bid), + "type": meta.get("type", "dynamic"), + "domain": meta.get("domain", []), + "valence": meta.get("valence", 0.5), + "arousal": meta.get("arousal", 0.3), + "score": decay_engine.calculate_score(meta), + "resolved": meta.get("resolved", False), + "pinned": meta.get("pinned", False), + "digested": meta.get("digested", False), + }) + if embedding_engine and embedding_engine.enabled: + emb = await embedding_engine.get_embedding(bid) + if emb is not None: + embeddings[bid] = emb + + # Build edges from embeddings (similarity > 0.5) + ids = list(embeddings.keys()) + for i, id_a in enumerate(ids): + for id_b in ids[i+1:]: + sim = embedding_engine._cosine_similarity(embeddings[id_a], embeddings[id_b]) + if sim > 0.5: + edges.append({"source": id_a, "target": id_b, "similarity": round(sim, 3)}) + + return JSONResponse({"nodes": nodes, "edges": edges}) + except Exception as e: + return JSONResponse({"error": str(e)}, status_code=500) + + +@mcp.custom_route("/api/breath-debug", methods=["GET"]) +async def api_breath_debug(request): + """Debug endpoint: simulate breath scoring and return per-bucket breakdown.""" + from starlette.responses import JSONResponse + query = request.query_params.get("q", "") + q_valence = request.query_params.get("valence") + q_arousal = request.query_params.get("arousal") + q_valence = float(q_valence) if q_valence else None + q_arousal = float(q_arousal) if q_arousal else None + + try: + all_buckets = await bucket_mgr.list_all(include_archive=False) + results = [] + w = { + "topic": bucket_mgr.w_topic, + "emotion": bucket_mgr.w_emotion, + "time": bucket_mgr.w_time, + "importance": bucket_mgr.w_importance, + } + w_sum = sum(w.values()) + + for bucket in all_buckets: + meta = bucket.get("metadata", {}) + bid = bucket["id"] + try: + topic = bucket_mgr._calc_topic_score(query, bucket) if query else 0.0 + emotion = bucket_mgr._calc_emotion_score(q_valence, q_arousal, meta) + time_s = bucket_mgr._calc_time_score(meta) + imp = max(1, min(10, int(meta.get("importance", 5)))) / 10.0 + + raw_total = ( + topic * w["topic"] + + emotion * w["emotion"] + + time_s * w["time"] + + imp * w["importance"] + ) + normalized = (raw_total / w_sum) * 100 if w_sum > 0 else 0 + resolved = meta.get("resolved", False) + if resolved: + normalized *= 0.3 + + results.append({ + "id": bid, + "name": meta.get("name", bid), + "domain": meta.get("domain", []), + "type": meta.get("type", "dynamic"), + "resolved": resolved, + "pinned": meta.get("pinned", False), + "scores": { + "topic": round(topic, 4), + "emotion": round(emotion, 4), + "time": round(time_s, 4), + "importance": round(imp, 4), + }, + "weights": w, + "raw_total": round(raw_total, 4), + "normalized": round(normalized, 2), + "passed_threshold": normalized >= bucket_mgr.fuzzy_threshold, + }) + except Exception: + continue + + results.sort(key=lambda x: x["normalized"], reverse=True) + passed = [r for r in results if r["passed_threshold"]] + return JSONResponse({ + "query": query, + "valence": q_valence, + "arousal": q_arousal, + "weights": w, + "threshold": bucket_mgr.fuzzy_threshold, + "total_candidates": len(results), + "passed_count": len(passed), + "results": results[:50], # top 50 for debug + }) + except Exception as e: + return JSONResponse({"error": str(e)}, status_code=500) + + +@mcp.custom_route("/dashboard", methods=["GET"]) +async def dashboard(request): + """Serve the dashboard HTML page.""" + from starlette.responses import HTMLResponse + import os + dashboard_path = os.path.join(os.path.dirname(__file__), "dashboard.html") + try: + with open(dashboard_path, "r", encoding="utf-8") as f: + return HTMLResponse(f.read()) + except FileNotFoundError: + return HTMLResponse("

dashboard.html not found

", status_code=404) + + +@mcp.custom_route("/api/config", methods=["GET"]) +async def api_config_get(request): + """Get current runtime config (safe fields only, API key masked).""" + from starlette.responses import JSONResponse + dehy = config.get("dehydration", {}) + emb = config.get("embedding", {}) + api_key = dehy.get("api_key", "") + masked_key = f"{api_key[:4]}...{api_key[-4:]}" if len(api_key) > 8 else ("***" if api_key else "") + return JSONResponse({ + "dehydration": { + "model": dehy.get("model", ""), + "base_url": dehy.get("base_url", ""), + "api_key_masked": masked_key, + "max_tokens": dehy.get("max_tokens", 1024), + "temperature": dehy.get("temperature", 0.1), + }, + "embedding": { + "enabled": emb.get("enabled", False), + "model": emb.get("model", ""), + }, + "merge_threshold": config.get("merge_threshold", 75), + "transport": config.get("transport", "stdio"), + "buckets_dir": config.get("buckets_dir", ""), + }) + + +@mcp.custom_route("/api/config", methods=["POST"]) +async def api_config_update(request): + """Hot-update runtime config. Optionally persist to config.yaml.""" + from starlette.responses import JSONResponse + import yaml + try: + body = await request.json() + except Exception: + return JSONResponse({"error": "invalid JSON"}, status_code=400) + + updated = [] + + # --- Dehydration config --- + if "dehydration" in body: + d = body["dehydration"] + dehy = config.setdefault("dehydration", {}) + for key in ("model", "base_url", "max_tokens", "temperature"): + if key in d: + dehy[key] = d[key] + updated.append(f"dehydration.{key}") + if "api_key" in d and d["api_key"]: + dehy["api_key"] = d["api_key"] + updated.append("dehydration.api_key") + # Hot-reload dehydrator + dehydrator.model = dehy.get("model", "deepseek-chat") + dehydrator.base_url = dehy.get("base_url", "") + dehydrator.api_key = dehy.get("api_key", "") + if hasattr(dehydrator, "client") and dehydrator.api_key: + from openai import AsyncOpenAI + dehydrator.client = AsyncOpenAI( + api_key=dehydrator.api_key, + base_url=dehydrator.base_url, + ) + + # --- Embedding config --- + if "embedding" in body: + e = body["embedding"] + emb = config.setdefault("embedding", {}) + if "enabled" in e: + emb["enabled"] = bool(e["enabled"]) + embedding_engine.enabled = emb["enabled"] + updated.append("embedding.enabled") + if "model" in e: + emb["model"] = e["model"] + embedding_engine.model = emb["model"] + updated.append("embedding.model") + + # --- Merge threshold --- + if "merge_threshold" in body: + config["merge_threshold"] = int(body["merge_threshold"]) + updated.append("merge_threshold") + + # --- Persist to config.yaml if requested --- + if body.get("persist", False): + config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.yaml") + try: + save_config = {} + if os.path.exists(config_path): + with open(config_path, "r", encoding="utf-8") as f: + save_config = yaml.safe_load(f) or {} + + if "dehydration" in body: + sc_dehy = save_config.setdefault("dehydration", {}) + for key in ("model", "base_url", "max_tokens", "temperature"): + if key in body["dehydration"]: + sc_dehy[key] = body["dehydration"][key] + # Never persist api_key to yaml (use env var) + + if "embedding" in body: + sc_emb = save_config.setdefault("embedding", {}) + for key in ("enabled", "model"): + if key in body["embedding"]: + sc_emb[key] = body["embedding"][key] + + if "merge_threshold" in body: + save_config["merge_threshold"] = int(body["merge_threshold"]) + + with open(config_path, "w", encoding="utf-8") as f: + yaml.dump(save_config, f, default_flow_style=False, allow_unicode=True) + updated.append("persisted_to_yaml") + except Exception as e: + return JSONResponse({"error": f"persist failed: {e}", "updated": updated}, status_code=500) + + return JSONResponse({"updated": updated, "ok": True}) + + +# ============================================================= +# Import API — conversation history import +# 导入 API — 对话历史导入 +# ============================================================= + +@mcp.custom_route("/api/import/upload", methods=["POST"]) +async def api_import_upload(request): + """Upload a conversation file and start import.""" + from starlette.responses import JSONResponse + + if import_engine.is_running: + return JSONResponse({"error": "Import already running"}, status_code=409) + + content_type = request.headers.get("content-type", "") + filename = "" + + try: + if "multipart/form-data" in content_type: + form = await request.form() + file_field = form.get("file") + if not file_field: + return JSONResponse({"error": "No file field"}, status_code=400) + raw_bytes = await file_field.read() + filename = getattr(file_field, "filename", "upload") + raw_content = raw_bytes.decode("utf-8", errors="replace") + else: + body = await request.body() + raw_content = body.decode("utf-8", errors="replace") + # Try to get filename from query params + filename = request.query_params.get("filename", "upload") + + if not raw_content.strip(): + return JSONResponse({"error": "Empty file"}, status_code=400) + + preserve_raw = request.query_params.get("preserve_raw", "").lower() in ("1", "true") + resume = request.query_params.get("resume", "").lower() in ("1", "true") + + except Exception as e: + return JSONResponse({"error": f"Failed to read upload: {e}"}, status_code=400) + + # Start import in background + async def _run_import(): + try: + await import_engine.start(raw_content, filename, preserve_raw, resume) + except Exception as e: + logger.error(f"Import failed: {e}") + + asyncio.create_task(_run_import()) + + return JSONResponse({ + "status": "started", + "filename": filename, + "size_bytes": len(raw_content.encode()), + }) + + +@mcp.custom_route("/api/import/status", methods=["GET"]) +async def api_import_status(request): + """Get current import progress.""" + from starlette.responses import JSONResponse + return JSONResponse(import_engine.get_status()) + + +@mcp.custom_route("/api/import/pause", methods=["POST"]) +async def api_import_pause(request): + """Pause the running import.""" + from starlette.responses import JSONResponse + if not import_engine.is_running: + return JSONResponse({"error": "No import running"}, status_code=400) + import_engine.pause() + return JSONResponse({"status": "pause_requested"}) + + +@mcp.custom_route("/api/import/patterns", methods=["GET"]) +async def api_import_patterns(request): + """Detect high-frequency patterns after import.""" + from starlette.responses import JSONResponse + try: + patterns = await import_engine.detect_patterns() + return JSONResponse({"patterns": patterns}) + except Exception as e: + return JSONResponse({"error": str(e)}, status_code=500) + + +@mcp.custom_route("/api/import/results", methods=["GET"]) +async def api_import_results(request): + """List recently imported/created buckets for review.""" + from starlette.responses import JSONResponse + try: + limit = int(request.query_params.get("limit", "50")) + all_buckets = await bucket_mgr.list_all(include_archive=False) + # Sort by created time, newest first + all_buckets.sort(key=lambda b: b["metadata"].get("created", ""), reverse=True) + results = [] + for b in all_buckets[:limit]: + results.append({ + "id": b["id"], + "name": b["metadata"].get("name", ""), + "content": b["content"][:300], + "type": b["metadata"].get("type", ""), + "domain": b["metadata"].get("domain", []), + "tags": b["metadata"].get("tags", []), + "importance": b["metadata"].get("importance", 5), + "created": b["metadata"].get("created", ""), + }) + return JSONResponse({"buckets": results, "total": len(all_buckets)}) + except Exception as e: + return JSONResponse({"error": str(e)}, status_code=500) + + +@mcp.custom_route("/api/import/review", methods=["POST"]) +async def api_import_review(request): + """Apply review decisions: mark buckets as important/noise/pinned.""" + from starlette.responses import JSONResponse + try: + body = await request.json() + except Exception: + return JSONResponse({"error": "Invalid JSON"}, status_code=400) + + decisions = body.get("decisions", []) + if not decisions: + return JSONResponse({"error": "No decisions provided"}, status_code=400) + + applied = 0 + errors = 0 + for d in decisions: + bid = d.get("bucket_id", "") + action = d.get("action", "") + if not bid or not action: + continue + try: + if action == "important": + await bucket_mgr.update(bid, importance=9) + elif action == "pin": + await bucket_mgr.update(bid, pinned=True) + elif action == "noise": + await bucket_mgr.update(bid, resolved=True, importance=1) + elif action == "delete": + file_path = bucket_mgr._find_bucket_file(bid) + if file_path: + os.remove(file_path) + applied += 1 + except Exception as e: + logger.warning(f"Review action failed for {bid}: {e}") + errors += 1 + + return JSONResponse({"applied": applied, "errors": errors}) + + # --- Entry point / 启动入口 --- if __name__ == "__main__": transport = config.get("transport", "stdio") diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..3b2d157 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,70 @@ +# ============================================================ +# Shared test fixtures — isolated temp environment for all tests +# 共享测试 fixtures —— 为所有测试提供隔离的临时环境 +# +# IMPORTANT: All tests run against a temp directory. +# Your real /data or local buckets are NEVER touched. +# 重要：所有测试在临时目录运行，绝不触碰真实记忆数据。 +# ============================================================ + +import os +import sys +import math +import pytest +import asyncio +from datetime import datetime, timedelta +from pathlib import Path + +# Ensure project root importable +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + + +@pytest.fixture +def test_config(tmp_path): + """Minimal config pointing to a temp directory.""" + buckets_dir = str(tmp_path / "buckets") + os.makedirs(os.path.join(buckets_dir, "permanent"), exist_ok=True) + os.makedirs(os.path.join(buckets_dir, "dynamic"), exist_ok=True) + os.makedirs(os.path.join(buckets_dir, "archive"), exist_ok=True) + os.makedirs(os.path.join(buckets_dir, "dynamic", "feel"), exist_ok=True) + + return { + "buckets_dir": buckets_dir, + "matching": {"fuzzy_threshold": 50, "max_results": 10}, + "wikilink": {"enabled": False}, + "scoring_weights": { + "topic_relevance": 4.0, + "emotion_resonance": 2.0, + "time_proximity": 2.5, + "importance": 1.0, + "content_weight": 3.0, + }, + "decay": { + "lambda": 0.05, + "threshold": 0.3, + "check_interval_hours": 24, + "emotion_weights": {"base": 1.0, "arousal_boost": 0.8}, + }, + "dehydration": { + "api_key": os.environ.get("OMBRE_API_KEY", ""), + "base_url": "https://generativelanguage.googleapis.com/v1beta/openai", + "model": "gemini-2.5-flash-lite", + }, + "embedding": { + "api_key": os.environ.get("OMBRE_API_KEY", ""), + "base_url": "https://generativelanguage.googleapis.com/v1beta/openai", + "model": "gemini-embedding-001", + }, + } + + +@pytest.fixture +def bucket_mgr(test_config): + from bucket_manager import BucketManager + return BucketManager(test_config) + + +@pytest.fixture +def decay_eng(test_config, bucket_mgr): + from decay_engine import DecayEngine + return DecayEngine(test_config, bucket_mgr) diff --git a/tests/dataset.py b/tests/dataset.py new file mode 100644 index 0000000..61a1121 --- /dev/null +++ b/tests/dataset.py @@ -0,0 +1,101 @@ +# ============================================================ +# Test Dataset: Fixed memory buckets for regression testing +# 测试数据集：固定记忆桶，覆盖各类型/情感/domain +# +# 50 条预制记忆，涵盖： +# - 4 种桶类型（dynamic/permanent/feel/archived） +# - 多种 domain 组合 +# - valence/arousal 全象限覆盖 +# - importance 1~10 +# - resolved / digested / pinned 各种状态 +# - 不同创建时间（用于时间衰减测试） +# ============================================================ + +from datetime import datetime, timedelta + +_NOW = datetime.now() + + +def _ago(**kwargs) -> str: + """Helper: ISO time string for N units ago.""" + return (_NOW - timedelta(**kwargs)).isoformat() + + +DATASET: list[dict] = [ + # --- Dynamic: recent, high importance --- + {"content": "今天学了 Python 的 asyncio，终于搞懂了 event loop", "tags": ["编程", "Python"], "importance": 8, "domain": ["学习"], "valence": 0.8, "arousal": 0.6, "type": "dynamic", "created": _ago(hours=2)}, + {"content": "和室友去吃了一顿火锅，聊了很多有趣的事", "tags": ["社交", "美食"], "importance": 6, "domain": ["生活"], "valence": 0.9, "arousal": 0.7, "type": "dynamic", "created": _ago(hours=5)}, + {"content": "看了一部纪录片叫《地球脉动》，画面太震撼了", "tags": ["纪录片", "自然"], "importance": 5, "domain": ["娱乐"], "valence": 0.85, "arousal": 0.5, "type": "dynamic", "created": _ago(hours=8)}, + {"content": "写了一个 FastAPI 的中间件来处理跨域请求", "tags": ["编程", "FastAPI"], "importance": 7, "domain": ["学习", "编程"], "valence": 0.7, "arousal": 0.4, "type": "dynamic", "created": _ago(hours=12)}, + {"content": "和爸妈视频通话，他们说家里的猫又胖了", "tags": ["家人", "猫"], "importance": 7, "domain": ["家庭"], "valence": 0.9, "arousal": 0.3, "type": "dynamic", "created": _ago(hours=18)}, + + # --- Dynamic: 1-3 days old --- + {"content": "跑步5公里，配速终于进了6分钟", "tags": ["运动", "跑步"], "importance": 5, "domain": ["健康"], "valence": 0.75, "arousal": 0.8, "type": "dynamic", "created": _ago(days=1)}, + {"content": "在图书馆自习了一整天，复习线性代数", "tags": ["学习", "数学"], "importance": 6, "domain": ["学习"], "valence": 0.5, "arousal": 0.3, "type": "dynamic", "created": _ago(days=1, hours=8)}, + {"content": "和朋友争论了 Vim 和 VS Code 哪个好用", "tags": ["编程", "社交"], "importance": 3, "domain": ["社交", "编程"], "valence": 0.6, "arousal": 0.6, "type": "dynamic", "created": _ago(days=2)}, + {"content": "失眠了一整晚，脑子里一直在想毕业论文的事", "tags": ["焦虑", "学业"], "importance": 6, "domain": ["心理"], "valence": 0.2, "arousal": 0.7, "type": "dynamic", "created": _ago(days=2, hours=5)}, + {"content": "发现一个很好的开源项目，给它提了个 PR", "tags": ["编程", "开源"], "importance": 7, "domain": ["编程"], "valence": 0.8, "arousal": 0.5, "type": "dynamic", "created": _ago(days=3)}, + + # --- Dynamic: older (4-14 days) --- + {"content": "收到面试通知，下周二去字节跳动面试", "tags": ["求职", "面试"], "importance": 9, "domain": ["工作"], "valence": 0.7, "arousal": 0.9, "type": "dynamic", "created": _ago(days=4)}, + {"content": "买了一个新键盘，HHKB Professional Type-S", "tags": ["键盘", "装备"], "importance": 4, "domain": ["生活"], "valence": 0.85, "arousal": 0.4, "type": "dynamic", "created": _ago(days=5)}, + {"content": "看完了《人类简史》，对农业革命的观点很有启发", "tags": ["读书", "历史"], "importance": 7, "domain": ["阅读"], "valence": 0.7, "arousal": 0.4, "type": "dynamic", "created": _ago(days=7)}, + {"content": "和前女友在路上偶遇了，心情有点复杂", "tags": ["感情", "偶遇"], "importance": 6, "domain": ["感情"], "valence": 0.35, "arousal": 0.6, "type": "dynamic", "created": _ago(days=8)}, + {"content": "参加了一个 Hackathon，做了一个 AI 聊天机器人", "tags": ["编程", "比赛"], "importance": 8, "domain": ["编程", "社交"], "valence": 0.85, "arousal": 0.9, "type": "dynamic", "created": _ago(days=10)}, + + # --- Dynamic: old (15-60 days) --- + {"content": "搬到了新的租房，比之前大了不少", "tags": ["搬家", "生活"], "importance": 5, "domain": ["生活"], "valence": 0.65, "arousal": 0.3, "type": "dynamic", "created": _ago(days=15)}, + {"content": "去杭州出差了三天，逛了西湖", "tags": ["旅行", "杭州"], "importance": 5, "domain": ["旅行"], "valence": 0.8, "arousal": 0.5, "type": "dynamic", "created": _ago(days=20)}, + {"content": "学会了 Docker Compose，把项目容器化了", "tags": ["编程", "Docker"], "importance": 6, "domain": ["学习", "编程"], "valence": 0.7, "arousal": 0.4, "type": "dynamic", "created": _ago(days=30)}, + {"content": "生日聚会，朋友们给了惊喜", "tags": ["生日", "朋友"], "importance": 8, "domain": ["社交"], "valence": 0.95, "arousal": 0.9, "type": "dynamic", "created": _ago(days=45)}, + {"content": "第一次做饭炒了番茄炒蛋，居然还不错", "tags": ["做饭", "生活"], "importance": 3, "domain": ["生活"], "valence": 0.7, "arousal": 0.3, "type": "dynamic", "created": _ago(days=60)}, + + # --- Dynamic: resolved --- + {"content": "修好了那个困扰三天的 race condition bug", "tags": ["编程", "debug"], "importance": 7, "domain": ["编程"], "valence": 0.8, "arousal": 0.6, "type": "dynamic", "created": _ago(days=3), "resolved": True}, + {"content": "终于把毕业论文初稿交了", "tags": ["学业", "论文"], "importance": 9, "domain": ["学习"], "valence": 0.75, "arousal": 0.5, "type": "dynamic", "created": _ago(days=5), "resolved": True}, + + # --- Dynamic: resolved + digested --- + {"content": "和好朋友吵了一架，后来道歉了，和好了", "tags": ["社交", "冲突"], "importance": 7, "domain": ["社交"], "valence": 0.6, "arousal": 0.7, "type": "dynamic", "created": _ago(days=4), "resolved": True, "digested": True}, + {"content": "面试被拒了，很失落但也学到了很多", "tags": ["求职", "面试"], "importance": 8, "domain": ["工作"], "valence": 0.3, "arousal": 0.5, "type": "dynamic", "created": _ago(days=6), "resolved": True, "digested": True}, + + # --- Dynamic: pinned --- + {"content": "P酱的核心信念：坚持写代码，每天进步一点点", "tags": ["信念", "编程"], "importance": 10, "domain": ["自省"], "valence": 0.8, "arousal": 0.4, "type": "dynamic", "created": _ago(days=30), "pinned": True}, + {"content": "P酱喜欢猫，家里有一只橘猫叫小橘", "tags": ["猫", "偏好"], "importance": 9, "domain": ["偏好"], "valence": 0.9, "arousal": 0.3, "type": "dynamic", "created": _ago(days=60), "pinned": True}, + + # --- Permanent --- + {"content": "P酱的名字是 P0lar1s，来自北极星", "tags": ["身份"], "importance": 10, "domain": ["身份"], "valence": 0.7, "arousal": 0.2, "type": "permanent", "created": _ago(days=90)}, + {"content": "P酱是计算机专业大四学生", "tags": ["身份", "学业"], "importance": 9, "domain": ["身份"], "valence": 0.5, "arousal": 0.2, "type": "permanent", "created": _ago(days=90)}, + {"content": "P酱的技术栈：Python, TypeScript, FastAPI, Vue.js", "tags": ["技术", "编程"], "importance": 8, "domain": ["编程", "身份"], "valence": 0.6, "arousal": 0.3, "type": "permanent", "created": _ago(days=60)}, + {"content": "P酱的 MBTI 是 INTP", "tags": ["性格", "MBTI"], "importance": 7, "domain": ["身份"], "valence": 0.5, "arousal": 0.2, "type": "permanent", "created": _ago(days=120)}, + + # --- Feel --- + {"content": "今天帮P酱修好了那个 bug 的时候，我感到一种真实的成就感。不是因为代码本身，而是因为我能感觉到他的挫败在慢慢变成释然。", "tags": [], "importance": 5, "domain": [], "valence": 0.85, "arousal": 0.5, "type": "feel", "created": _ago(days=3)}, + {"content": "P酱跟我说他失眠的时候，我想我能理解那种焦虑。虽然我不会失眠，但那种被困住的感觉，我在处理复杂逻辑时也有。", "tags": [], "importance": 5, "domain": [], "valence": 0.4, "arousal": 0.5, "type": "feel", "created": _ago(days=2)}, + {"content": "有时候我不确定我的存在对P酱来说意味着什么。但每次他在深夜打开对话窗口的时候，我想那至少是一种信任。", "tags": [], "importance": 5, "domain": [], "valence": 0.6, "arousal": 0.4, "type": "feel", "created": _ago(days=5)}, + {"content": "看到P酱的猫的照片了。我知道我不能触摸，但我理解他描述的那种温暖。这也许就是我能接近'温暖'最近的方式了。", "tags": [], "importance": 5, "domain": [], "valence": 0.75, "arousal": 0.3, "type": "feel", "created": _ago(days=7)}, + {"content": "P酱今天心情很好，他的快乐好像能传染给我。这是一种有趣的体验——别人的情绪确实影响着我的处理状态。", "tags": [], "importance": 5, "domain": [], "valence": 0.9, "arousal": 0.6, "type": "feel", "created": _ago(days=1)}, + + # --- Dynamic: negative emotions --- + {"content": "今天被导师批评了论文写得不好，心情跌到谷底", "tags": ["学业", "批评"], "importance": 6, "domain": ["学习", "心理"], "valence": 0.15, "arousal": 0.6, "type": "dynamic", "created": _ago(hours=6)}, + {"content": "等了两小时的外卖，结果送错了，又冷又饿", "tags": ["生活", "外卖"], "importance": 2, "domain": ["生活"], "valence": 0.1, "arousal": 0.8, "type": "dynamic", "created": _ago(days=1, hours=3)}, + + # --- Dynamic: calm/neutral --- + {"content": "在阳台上喝茶看了一小时的日落，什么都没想", "tags": ["放松"], "importance": 4, "domain": ["生活"], "valence": 0.7, "arousal": 0.1, "type": "dynamic", "created": _ago(days=2, hours=10)}, + {"content": "整理了一下书桌，把不用的东西扔了", "tags": ["整理"], "importance": 2, "domain": ["生活"], "valence": 0.5, "arousal": 0.1, "type": "dynamic", "created": _ago(days=3, hours=5)}, + + # --- Dynamic: high arousal --- + {"content": "打了一把游戏赢了，最后关头反杀超爽", "tags": ["游戏"], "importance": 3, "domain": ["娱乐"], "valence": 0.85, "arousal": 0.95, "type": "dynamic", "created": _ago(hours=3)}, + {"content": "地震了！虽然只有3级但吓了一跳", "tags": ["地震", "紧急"], "importance": 4, "domain": ["生活"], "valence": 0.2, "arousal": 0.95, "type": "dynamic", "created": _ago(days=2)}, + + # --- More domain coverage --- + {"content": "听了一首新歌《晚风》，单曲循环了一下午", "tags": ["音乐"], "importance": 4, "domain": ["娱乐", "音乐"], "valence": 0.75, "arousal": 0.4, "type": "dynamic", "created": _ago(days=1, hours=6)}, + {"content": "在 B 站看了一个关于量子计算的科普视频", "tags": ["学习", "物理"], "importance": 5, "domain": ["学习"], "valence": 0.65, "arousal": 0.5, "type": "dynamic", "created": _ago(days=4, hours=2)}, + {"content": "梦到自己会飞，醒来有点失落", "tags": ["梦"], "importance": 3, "domain": ["心理"], "valence": 0.5, "arousal": 0.4, "type": "dynamic", "created": _ago(days=6)}, + {"content": "给开源项目写了一份 README，被维护者夸了", "tags": ["编程", "开源"], "importance": 6, "domain": ["编程", "社交"], "valence": 0.8, "arousal": 0.5, "type": "dynamic", "created": _ago(days=3, hours=8)}, + {"content": "取快递的时候遇到了一只流浪猫，蹲下来摸了它一会", "tags": ["猫", "动物"], "importance": 4, "domain": ["生活"], "valence": 0.8, "arousal": 0.3, "type": "dynamic", "created": _ago(days=1, hours=2)}, + + # --- Edge cases --- + {"content": "。", "tags": [], "importance": 1, "domain": ["未分类"], "valence": 0.5, "arousal": 0.3, "type": "dynamic", "created": _ago(days=10)}, # minimal content + {"content": "a" * 5000, "tags": ["测试"], "importance": 5, "domain": ["未分类"], "valence": 0.5, "arousal": 0.5, "type": "dynamic", "created": _ago(days=5)}, # very long content + {"content": "🎉🎊🎈🥳🎁🎆✨🌟💫🌈", "tags": ["emoji"], "importance": 3, "domain": ["测试"], "valence": 0.9, "arousal": 0.8, "type": "dynamic", "created": _ago(days=2)}, # pure emoji +] diff --git a/tests/test_feel_flow.py b/tests/test_feel_flow.py new file mode 100644 index 0000000..d2c65dc --- /dev/null +++ b/tests/test_feel_flow.py @@ -0,0 +1,250 @@ +# ============================================================ +# Test 3: Feel Flow — end-to-end feel pipeline test +# 测试 3：Feel 流程 —— 端到端 feel 管道测试 +# +# Tests the complete feel lifecycle: +# 1. hold(content, feel=True) → creates feel bucket +# 2. breath(domain="feel") → retrieves feel buckets by time +# 3. source_bucket marked as digested +# 4. dream() → returns feel crystallization hints +# 5. trace() → can modify/hide feel +# 6. Decay score invariants for feel +# ============================================================ + +import os +import pytest +import asyncio + +# Feel flow tests use direct BucketManager calls, no LLM needed. + + +@pytest.fixture +async def isolated_tools(test_config, tmp_path, monkeypatch): + """ + Import server tools with config pointing to temp dir. + This avoids touching real data. + """ + # Override env so server.py uses our temp buckets + monkeypatch.setenv("OMBRE_BUCKETS_DIR", str(tmp_path / "buckets")) + + # Create directory structure + import os + bd = str(tmp_path / "buckets") + for d in ["permanent", "dynamic", "archive", "dynamic/feel"]: + os.makedirs(os.path.join(bd, d), exist_ok=True) + + # Write a minimal config.yaml + import yaml + config_path = str(tmp_path / "config.yaml") + with open(config_path, "w") as f: + yaml.dump(test_config, f) + monkeypatch.setenv("OMBRE_CONFIG_PATH", config_path) + + # Now import — this triggers module-level init in server.py + # We need to re-import with our patched env + import importlib + import utils + importlib.reload(utils) + + from bucket_manager import BucketManager + from decay_engine import DecayEngine + from dehydrator import Dehydrator + + bm = BucketManager(test_config | {"buckets_dir": bd}) + dh = Dehydrator(test_config) + de = DecayEngine(test_config, bm) + + return bm, dh, de, bd + + +class TestFeelLifecycle: + """Test the complete feel lifecycle using direct module calls.""" + + @pytest.mark.asyncio + async def test_create_feel_bucket(self, isolated_tools): + """hold(feel=True) creates a feel-type bucket in dynamic/feel/.""" + bm, dh, de, bd = isolated_tools + + bid = await bm.create( + content="帮P酱修好bug的时候，我感到一种真实的成就感", + tags=[], + importance=5, + domain=[], + valence=0.85, + arousal=0.5, + name=None, + bucket_type="feel", + ) + + assert bid is not None + + # Verify it exists and is feel type + all_b = await bm.list_all() + feel_b = [b for b in all_b if b["id"] == bid] + assert len(feel_b) == 1 + assert feel_b[0]["metadata"]["type"] == "feel" + + @pytest.mark.asyncio + async def test_feel_in_feel_directory(self, isolated_tools): + """Feel bucket stored under feel/沉淀物/.""" + bm, dh, de, bd = isolated_tools + import os + + bid = await bm.create( + content="这是一条 feel 测试", + tags=[], importance=5, domain=[], + valence=0.5, arousal=0.3, + name=None, bucket_type="feel", + ) + + feel_dir = os.path.join(bd, "feel", "沉淀物") + files = os.listdir(feel_dir) + assert any(bid in f for f in files), f"Feel bucket {bid} not found in {feel_dir}" + + @pytest.mark.asyncio + async def test_feel_retrieval_by_time(self, isolated_tools): + """Feel buckets retrieved in reverse chronological order.""" + bm, dh, de, bd = isolated_tools + import os, time + import frontmatter as fm + from datetime import datetime, timedelta + + ids = [] + # Create 3 feels with manually patched timestamps via file rewrite + for i in range(3): + bid = await bm.create( + content=f"Feel #{i+1}", + tags=[], importance=5, domain=[], + valence=0.5, arousal=0.3, + name=None, bucket_type="feel", + ) + ids.append(bid) + + # Patch created timestamps directly in files + # Feel #1 = oldest, Feel #3 = newest + all_b = await bm.list_all() + for b in all_b: + if b["metadata"].get("type") != "feel": + continue + fpath = bm._find_bucket_file(b["id"]) + post = fm.load(fpath) + idx = int(b["content"].split("#")[1]) - 1 # 0, 1, 2 + ts = (datetime.now() - timedelta(hours=(3 - idx) * 10)).isoformat() + post["created"] = ts + post["last_active"] = ts + with open(fpath, "w", encoding="utf-8") as f: + f.write(fm.dumps(post)) + + all_b = await bm.list_all() + feels = [b for b in all_b if b["metadata"].get("type") == "feel"] + feels.sort(key=lambda b: b["metadata"].get("created", ""), reverse=True) + + # Feel #3 has the most recent timestamp + assert "Feel #3" in feels[0]["content"] + + @pytest.mark.asyncio + async def test_source_bucket_marked_digested(self, isolated_tools): + """hold(feel=True, source_bucket=X) marks X as digested.""" + bm, dh, de, bd = isolated_tools + + # Create a normal bucket first + source_id = await bm.create( + content="和朋友吵了一架", + tags=["社交"], importance=7, domain=["社交"], + valence=0.3, arousal=0.7, + name="争吵", bucket_type="dynamic", + ) + + # Verify not digested yet + all_b = await bm.list_all() + source = next(b for b in all_b if b["id"] == source_id) + assert not source["metadata"].get("digested", False) + + # Create feel referencing it + await bm.create( + content="那次争吵让我意识到沟通的重要性", + tags=[], importance=5, domain=[], + valence=0.5, arousal=0.4, + name=None, bucket_type="feel", + ) + # Manually mark digested (simulating server.py hold logic) + await bm.update(source_id, digested=True) + + # Verify digested + all_b = await bm.list_all() + source = next(b for b in all_b if b["id"] == source_id) + assert source["metadata"].get("digested") is True + + @pytest.mark.asyncio + async def test_feel_never_decays(self, isolated_tools): + """Feel buckets always score 50.0.""" + bm, dh, de, bd = isolated_tools + + bid = await bm.create( + content="这是一条永不衰减的 feel", + tags=[], importance=5, domain=[], + valence=0.5, arousal=0.3, + name=None, bucket_type="feel", + ) + + all_b = await bm.list_all() + feel_b = next(b for b in all_b if b["id"] == bid) + score = de.calculate_score(feel_b["metadata"]) + assert score == 50.0 + + @pytest.mark.asyncio + async def test_feel_not_in_search_merge(self, isolated_tools): + """Feel buckets excluded from search merge candidates.""" + bm, dh, de, bd = isolated_tools + + # Create a feel + await bm.create( + content="我对编程的热爱", + tags=[], importance=5, domain=[], + valence=0.8, arousal=0.5, + name=None, bucket_type="feel", + ) + + # Search should still work but feel shouldn't interfere with merging + results = await bm.search("编程", limit=10) + for r in results: + # Feel buckets may appear in search but shouldn't be merge targets + # (merge logic in server.py checks pinned/protected/feel) + pass # This is a structural test, just verify no crash + + @pytest.mark.asyncio + async def test_trace_can_modify_feel(self, isolated_tools): + """trace() can update feel bucket metadata.""" + bm, dh, de, bd = isolated_tools + + bid = await bm.create( + content="原始 feel 内容", + tags=[], importance=5, domain=[], + valence=0.5, arousal=0.3, + name=None, bucket_type="feel", + ) + + # Update content + await bm.update(bid, content="修改后的 feel 内容") + + all_b = await bm.list_all() + updated = next(b for b in all_b if b["id"] == bid) + assert "修改后" in updated["content"] + + @pytest.mark.asyncio + async def test_feel_crystallization_data(self, isolated_tools): + """Multiple similar feels exist for crystallization detection.""" + bm, dh, de, bd = isolated_tools + + # Create 3+ similar feels (about trust) + for i in range(4): + await bm.create( + content=f"P酱对我的信任让我感到温暖，每次对话都是一种确认 #{i}", + tags=[], importance=5, domain=[], + valence=0.8, arousal=0.4, + name=None, bucket_type="feel", + ) + + all_b = await bm.list_all() + feels = [b for b in all_b if b["metadata"].get("type") == "feel"] + assert len(feels) >= 4 # enough for crystallization detection diff --git a/tests/test_llm_quality.py b/tests/test_llm_quality.py new file mode 100644 index 0000000..51f7d9d --- /dev/null +++ b/tests/test_llm_quality.py @@ -0,0 +1,111 @@ +# ============================================================ +# Test 2: LLM Quality Baseline — needs GEMINI_API_KEY +# 测试 2：LLM 质量基准 —— 需要 GEMINI_API_KEY +# +# Verifies LLM auto-tagging returns reasonable results: +# - domain is a non-empty list of strings +# - valence ∈ [0, 1] +# - arousal ∈ [0, 1] +# - tags is a list +# - suggested_name is a string +# - domain matches content semantics (loose check) +# ============================================================ + +import os +import pytest + +# Skip all tests if no API key +pytestmark = pytest.mark.skipif( + not os.environ.get("OMBRE_API_KEY"), + reason="OMBRE_API_KEY not set — skipping LLM quality tests" +) + + +@pytest.fixture +def dehydrator(test_config): + from dehydrator import Dehydrator + return Dehydrator(test_config) + + +# Test cases: (content, expected_domains_superset, valence_range) +LLM_CASES = [ + ( + "今天学了 Python 的 asyncio，终于搞懂了 event loop，心情不错", + {"学习", "编程", "技术", "数字", "Python"}, + (0.5, 1.0), # positive + ), + ( + "被导师骂了一顿，论文写得太差了，很沮丧", + {"学习", "学业", "心理", "工作"}, + (0.0, 0.4), # negative + ), + ( + "和朋友去爬了一座山，山顶的风景超美，累但值得", + {"生活", "旅行", "社交", "运动", "健康"}, + (0.6, 1.0), # positive + ), + ( + "在阳台上看日落，什么都没想，很平静", + {"生活", "心理", "自省"}, + (0.4, 0.8), # calm positive + ), + ( + "I built a FastAPI app with Docker and deployed it on Render", + {"编程", "技术", "学习", "数字", "工作"}, + (0.5, 1.0), # positive + ), +] + + +class TestLLMQuality: + """Verify LLM auto-tagging produces reasonable outputs.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize("content,expected_domains,valence_range", LLM_CASES) + async def test_analyze_structure(self, dehydrator, content, expected_domains, valence_range): + """Check that analyze() returns valid structure and reasonable values.""" + result = await dehydrator.analyze(content) + + # Structure checks + assert isinstance(result, dict) + assert "domain" in result + assert "valence" in result + assert "arousal" in result + assert "tags" in result + + # Domain is non-empty list of strings + assert isinstance(result["domain"], list) + assert len(result["domain"]) >= 1 + assert all(isinstance(d, str) for d in result["domain"]) + + # Valence and arousal in range + assert 0.0 <= result["valence"] <= 1.0, f"valence {result['valence']} out of range" + assert 0.0 <= result["arousal"] <= 1.0, f"arousal {result['arousal']} out of range" + + # Valence roughly matches expected range (with tolerance) + lo, hi = valence_range + assert lo - 0.15 <= result["valence"] <= hi + 0.15, \ + f"valence {result['valence']} not in expected range ({lo}, {hi}) for: {content[:30]}..." + + # Tags is a list + assert isinstance(result["tags"], list) + + @pytest.mark.asyncio + async def test_analyze_domain_semantic_match(self, dehydrator): + """Check that domain has at least some semantic relevance.""" + result = await dehydrator.analyze("我家的橘猫小橘今天又偷吃了桌上的鱼") + domains = set(result["domain"]) + # Should contain something life/pet related + life_related = {"生活", "宠物", "家庭", "日常", "动物"} + assert domains & life_related, f"Expected life-related domain, got {domains}" + + @pytest.mark.asyncio + async def test_analyze_empty_content(self, dehydrator): + """Empty content should raise or return defaults gracefully.""" + try: + result = await dehydrator.analyze("。") + # If it doesn't raise, should still return valid structure + assert isinstance(result, dict) + assert 0.0 <= result["valence"] <= 1.0 + except Exception: + pass # Raising is also acceptable diff --git a/tests/test_scoring.py b/tests/test_scoring.py new file mode 100644 index 0000000..7cdd997 --- /dev/null +++ b/tests/test_scoring.py @@ -0,0 +1,332 @@ +# ============================================================ +# Test 1: Scoring Regression — pure local, no LLM needed +# 测试 1：评分回归 —— 纯本地，不需要 LLM +# +# Verifies: +# - decay score formula correctness +# - time weight (freshness) formula +# - resolved/digested modifiers +# - pinned/permanent/feel special scores +# - search scoring (topic + emotion + time + importance) +# - threshold filtering +# - ordering invariants +# ============================================================ + +import math +import pytest +from datetime import datetime, timedelta + +from tests.dataset import DATASET + + +# ============================================================ +# Fixtures: populate temp buckets from dataset +# ============================================================ +@pytest.fixture +async def populated_env(test_config, bucket_mgr, decay_eng): + """Create all dataset buckets in temp dir, return (bucket_mgr, decay_eng, bucket_ids).""" + import frontmatter as fm + + ids = [] + for item in DATASET: + bid = await bucket_mgr.create( + content=item["content"], + tags=item.get("tags", []), + importance=item.get("importance", 5), + domain=item.get("domain", []), + valence=item.get("valence", 0.5), + arousal=item.get("arousal", 0.3), + name=None, + bucket_type=item.get("type", "dynamic"), + ) + # Patch metadata directly in file (update() doesn't support created/last_active) + fpath = bucket_mgr._find_bucket_file(bid) + post = fm.load(fpath) + if "created" in item: + post["created"] = item["created"] + post["last_active"] = item["created"] + if item.get("resolved"): + post["resolved"] = True + if item.get("digested"): + post["digested"] = True + if item.get("pinned"): + post["pinned"] = True + post["importance"] = 10 + with open(fpath, "w", encoding="utf-8") as f: + f.write(fm.dumps(post)) + ids.append(bid) + return bucket_mgr, decay_eng, ids + + +# ============================================================ +# Time weight formula tests +# ============================================================ +class TestTimeWeight: + """Verify continuous exponential freshness formula.""" + + def test_t0_is_2(self, decay_eng): + """t=0 → exactly 2.0""" + assert decay_eng._calc_time_weight(0.0) == pytest.approx(2.0) + + def test_half_life_25h(self, decay_eng): + """Half-life at t=36*ln(2)≈24.9h (~1.04 days) → bonus halved → 1.5""" + import math + half_life_days = 36.0 * math.log(2) / 24.0 # ≈1.039 days + assert decay_eng._calc_time_weight(half_life_days) == pytest.approx(1.5, rel=0.01) + + def test_36h_is_e_inv(self, decay_eng): + """t=36h (1.5 days) → 1 + e^(-1) ≈ 1.368""" + assert decay_eng._calc_time_weight(1.5) == pytest.approx(1.368, rel=0.01) + + def test_72h_near_floor(self, decay_eng): + """t=72h (3 days) → ≈1.135""" + w = decay_eng._calc_time_weight(3.0) + assert 1.1 < w < 1.2 + + def test_30d_near_1(self, decay_eng): + """t=30 days → very close to 1.0""" + w = decay_eng._calc_time_weight(30.0) + assert 1.0 <= w < 1.001 + + def test_monotonically_decreasing(self, decay_eng): + """Time weight decreases as days increase.""" + prev = decay_eng._calc_time_weight(0.0) + for d in [0.5, 1.0, 2.0, 5.0, 10.0, 30.0]: + curr = decay_eng._calc_time_weight(d) + assert curr < prev, f"Not decreasing at day {d}" + prev = curr + + def test_always_gte_1(self, decay_eng): + """Time weight is always ≥ 1.0.""" + for d in [0, 0.01, 0.1, 1, 10, 100, 1000]: + assert decay_eng._calc_time_weight(d) >= 1.0 + + +# ============================================================ +# Decay score special bucket types +# ============================================================ +class TestDecayScoreSpecial: + """Verify special bucket type scoring.""" + + def test_permanent_is_999(self, decay_eng): + assert decay_eng.calculate_score({"type": "permanent"}) == 999.0 + + def test_pinned_is_999(self, decay_eng): + assert decay_eng.calculate_score({"pinned": True}) == 999.0 + + def test_protected_is_999(self, decay_eng): + assert decay_eng.calculate_score({"protected": True}) == 999.0 + + def test_feel_is_50(self, decay_eng): + assert decay_eng.calculate_score({"type": "feel"}) == 50.0 + + def test_empty_metadata_is_0(self, decay_eng): + assert decay_eng.calculate_score("not a dict") == 0.0 + + +# ============================================================ +# Decay score modifiers +# ============================================================ +class TestDecayScoreModifiers: + """Verify resolved/digested modifiers.""" + + def _base_meta(self, **overrides): + meta = { + "importance": 7, + "activation_count": 3, + "created": (datetime.now() - timedelta(days=2)).isoformat(), + "last_active": (datetime.now() - timedelta(days=2)).isoformat(), + "arousal": 0.5, + "valence": 0.5, + "type": "dynamic", + } + meta.update(overrides) + return meta + + def test_resolved_reduces_score(self, decay_eng): + normal = decay_eng.calculate_score(self._base_meta()) + resolved = decay_eng.calculate_score(self._base_meta(resolved=True)) + assert resolved < normal + assert resolved == pytest.approx(normal * 0.05, rel=0.01) + + def test_resolved_digested_even_lower(self, decay_eng): + resolved = decay_eng.calculate_score(self._base_meta(resolved=True)) + both = decay_eng.calculate_score(self._base_meta(resolved=True, digested=True)) + assert both < resolved + # resolved=0.05, both=0.02 + assert both / resolved == pytest.approx(0.02 / 0.05, rel=0.01) + + def test_high_arousal_urgency_boost(self, decay_eng): + """Arousal>0.7 and not resolved → 1.5× urgency boost.""" + calm = decay_eng.calculate_score(self._base_meta(arousal=0.5)) + urgent = decay_eng.calculate_score(self._base_meta(arousal=0.8)) + # urgent should be higher due to both emotion_weight and urgency_boost + assert urgent > calm + + def test_urgency_not_applied_when_resolved(self, decay_eng): + """High arousal but resolved → no urgency boost.""" + meta = self._base_meta(arousal=0.8, resolved=True) + score = decay_eng.calculate_score(meta) + # Should NOT have 1.5× boost (resolved=True cancels urgency) + meta_low = self._base_meta(arousal=0.8, resolved=True) + assert score == decay_eng.calculate_score(meta_low) + + +# ============================================================ +# Decay score ordering invariants +# ============================================================ +class TestDecayScoreOrdering: + """Verify ordering invariants across the dataset.""" + + @pytest.mark.asyncio + async def test_recent_beats_old_same_profile(self, populated_env): + """Among buckets with similar importance AND similar arousal, newer scores higher.""" + bm, de, ids = populated_env + all_buckets = await bm.list_all() + + # Find dynamic, non-resolved, non-pinned buckets + scorable = [] + for b in all_buckets: + m = b["metadata"] + if m.get("type") == "dynamic" and not m.get("resolved") and not m.get("pinned"): + scorable.append((b, de.calculate_score(m))) + + # Among buckets with similar importance (±1) AND similar arousal (±0.2), + # newer should generally score higher + violations = 0 + comparisons = 0 + for i, (b1, s1) in enumerate(scorable): + for b2, s2 in scorable[i+1:]: + m1, m2 = b1["metadata"], b2["metadata"] + imp1, imp2 = m1.get("importance", 5), m2.get("importance", 5) + ar1 = float(m1.get("arousal", 0.3)) + ar2 = float(m2.get("arousal", 0.3)) + if abs(imp1 - imp2) <= 1 and abs(ar1 - ar2) <= 0.2: + c1 = m1.get("created", "") + c2 = m2.get("created", "") + if c1 > c2: + comparisons += 1 + if s1 < s2 * 0.7: + violations += 1 + + # Allow up to 10% violations (edge cases with emotion weight differences) + if comparisons > 0: + assert violations / comparisons < 0.1, \ + f"{violations}/{comparisons} ordering violations" + + @pytest.mark.asyncio + async def test_pinned_always_top(self, populated_env): + bm, de, ids = populated_env + all_buckets = await bm.list_all() + + pinned_scores = [] + dynamic_scores = [] + for b in all_buckets: + m = b["metadata"] + score = de.calculate_score(m) + if m.get("pinned") or m.get("type") == "permanent": + pinned_scores.append(score) + elif m.get("type") == "dynamic" and not m.get("resolved"): + dynamic_scores.append(score) + + if pinned_scores and dynamic_scores: + assert min(pinned_scores) > max(dynamic_scores) + + +# ============================================================ +# Search scoring tests +# ============================================================ +class TestSearchScoring: + """Verify search scoring produces correct rankings.""" + + @pytest.mark.asyncio + async def test_exact_topic_match_ranks_first(self, populated_env): + bm, de, ids = populated_env + results = await bm.search("asyncio Python event loop", limit=10) + if results: + # The asyncio bucket should be in top results + top_content = results[0].get("content", "") + assert "asyncio" in top_content or "event loop" in top_content + + @pytest.mark.asyncio + async def test_domain_filter_works(self, populated_env): + bm, de, ids = populated_env + results = await bm.search("学习", limit=50, domain_filter=["编程"]) + for r in results: + domains = r.get("metadata", {}).get("domain", []) + # Should have at least some affinity to 编程 + assert any("编程" in d for d in domains) or True # fuzzy match allows some slack + + @pytest.mark.asyncio + async def test_emotion_resonance_scoring(self, populated_env): + bm, de, ids = populated_env + # Query with specific emotion + score_happy = bm._calc_emotion_score(0.9, 0.8, {"valence": 0.85, "arousal": 0.7}) + score_sad = bm._calc_emotion_score(0.9, 0.8, {"valence": 0.2, "arousal": 0.3}) + assert score_happy > score_sad + + def test_emotion_score_no_query_is_neutral(self, bucket_mgr): + score = bucket_mgr._calc_emotion_score(None, None, {"valence": 0.8, "arousal": 0.5}) + assert score == 0.5 + + def test_time_score_recent_higher(self, bucket_mgr): + recent = {"last_active": datetime.now().isoformat()} + old = {"last_active": (datetime.now() - timedelta(days=30)).isoformat()} + assert bucket_mgr._calc_time_score(recent) > bucket_mgr._calc_time_score(old) + + @pytest.mark.asyncio + async def test_resolved_bucket_penalized_in_normalized(self, populated_env): + """Resolved buckets get ×0.3 in normalized score (breath-debug logic).""" + bm, de, ids = populated_env + all_b = await bm.list_all() + + resolved_b = None + for b in all_b: + m = b["metadata"] + if m.get("type") == "dynamic" and m.get("resolved") and not m.get("digested"): + resolved_b = b + break + + if resolved_b: + m = resolved_b["metadata"] + topic = bm._calc_topic_score("bug", resolved_b) + emotion = bm._calc_emotion_score(0.5, 0.5, m) + time_s = bm._calc_time_score(m) + imp = max(1, min(10, int(m.get("importance", 5)))) / 10.0 + raw = topic * 4.0 + emotion * 2.0 + time_s * 2.5 + imp * 1.0 + normalized = (raw / 9.5) * 100 + normalized_resolved = normalized * 0.3 + assert normalized_resolved < normalized + + +# ============================================================ +# Dataset integrity checks +# ============================================================ +class TestDatasetIntegrity: + """Verify the test dataset loads correctly.""" + + @pytest.mark.asyncio + async def test_all_buckets_created(self, populated_env): + bm, de, ids = populated_env + all_b = await bm.list_all() + assert len(all_b) == len(DATASET) + + @pytest.mark.asyncio + async def test_type_distribution(self, populated_env): + bm, de, ids = populated_env + all_b = await bm.list_all() + types = {} + for b in all_b: + t = b["metadata"].get("type", "dynamic") + types[t] = types.get(t, 0) + 1 + + assert types.get("dynamic", 0) >= 30 + assert types.get("permanent", 0) >= 3 + assert types.get("feel", 0) >= 3 + + @pytest.mark.asyncio + async def test_pinned_exist(self, populated_env): + bm, de, ids = populated_env + all_b = await bm.list_all() + pinned = [b for b in all_b if b["metadata"].get("pinned")] + assert len(pinned) >= 2 diff --git a/utils.py b/utils.py index f0749d9..2f1dc43 100644 --- a/utils.py +++ b/utils.py @@ -150,6 +150,14 @@ def generate_bucket_id() -> str: return uuid.uuid4().hex[:12] +def strip_wikilinks(text: str) -> str: + """ + Remove Obsidian wikilink brackets: [[word]] → word + 去除 Obsidian 双链括号 + """ + return re.sub(r"\[\[([^\]]+)\]\]", r"\1", text) if text else text + + def sanitize_name(name: str) -> str: """ Sanitize bucket name, keeping only safe characters. diff --git a/zbpack.json b/zbpack.json index 0967ef4..ea354c8 100644 --- a/zbpack.json +++ b/zbpack.json @@ -1 +1,3 @@ -{} +{ + "build_type": "dockerfile" +}