feat (memory): Implement new (opt-in) QMD memory backend

This commit is contained in:
Vignesh Natarajan
2026-01-27 21:57:15 -08:00
committed by Vignesh
parent e9f182def7
commit 5d3af3bc62
24 changed files with 1828 additions and 601 deletions

View File

@@ -1,14 +1,12 @@
--- ---
summary: "How OpenClaw memory works (workspace files + automatic memory flush)" summary: "How Moltbot memory works (workspace files + automatic memory flush)"
read_when: read_when:
- You want the memory file layout and workflow - You want the memory file layout and workflow
- You want to tune the automatic pre-compaction memory flush - You want to tune the automatic pre-compaction memory flush
title: "Memory"
--- ---
# Memory # Memory
OpenClaw memory is **plain Markdown in the agent workspace**. The files are the Moltbot memory is **plain Markdown in the agent workspace**. The files are the
source of truth; the model only "remembers" what gets written to disk. source of truth; the model only "remembers" what gets written to disk.
Memory search tools are provided by the active memory plugin (default: Memory search tools are provided by the active memory plugin (default:
@@ -26,7 +24,7 @@ The default workspace layout uses two memory layers:
- **Only load in the main, private session** (never in group contexts). - **Only load in the main, private session** (never in group contexts).
These files live under the workspace (`agents.defaults.workspace`, default These files live under the workspace (`agents.defaults.workspace`, default
`~/.openclaw/workspace`). See [Agent workspace](/concepts/agent-workspace) for the full layout. `~/clawd`). See [Agent workspace](/concepts/agent-workspace) for the full layout.
## When to write memory ## When to write memory
@@ -38,9 +36,9 @@ These files live under the workspace (`agents.defaults.workspace`, default
## Automatic memory flush (pre-compaction ping) ## Automatic memory flush (pre-compaction ping)
When a session is **close to auto-compaction**, OpenClaw triggers a **silent, When a session is **close to auto-compaction**, Moltbot triggers a **silent,
agentic turn** that reminds the model to write durable memory **before** the agentic turn** that reminds the model to write durable memory **before** the
context is compacted. The default prompts explicitly say the model _may reply_, context is compacted. The default prompts explicitly say the model *may reply*,
but usually `NO_REPLY` is the correct response so the user never sees this turn. but usually `NO_REPLY` is the correct response so the user never sees this turn.
This is controlled by `agents.defaults.compaction.memoryFlush`: This is controlled by `agents.defaults.compaction.memoryFlush`:
@@ -55,16 +53,15 @@ This is controlled by `agents.defaults.compaction.memoryFlush`:
enabled: true, enabled: true,
softThresholdTokens: 4000, softThresholdTokens: 4000,
systemPrompt: "Session nearing compaction. Store durable memories now.", systemPrompt: "Session nearing compaction. Store durable memories now.",
prompt: "Write any lasting notes to memory/YYYY-MM-DD.md; reply with NO_REPLY if nothing to store.", prompt: "Write any lasting notes to memory/YYYY-MM-DD.md; reply with NO_REPLY if nothing to store."
}, }
}, }
}, }
}, }
} }
``` ```
Details: Details:
- **Soft threshold**: flush triggers when the session token estimate crosses - **Soft threshold**: flush triggers when the session token estimate crosses
`contextWindow - reserveTokensFloor - softThresholdTokens`. `contextWindow - reserveTokensFloor - softThresholdTokens`.
- **Silent** by default: prompts include `NO_REPLY` so nothing is delivered. - **Silent** by default: prompts include `NO_REPLY` so nothing is delivered.
@@ -78,15 +75,13 @@ For the full compaction lifecycle, see
## Vector memory search ## Vector memory search
OpenClaw can build a small vector index over `MEMORY.md` and `memory/*.md` (plus Moltbot can build a small vector index over `MEMORY.md` and `memory/*.md` so
any extra directories or files you opt in) so semantic queries can find related semantic queries can find related notes even when wording differs.
notes even when wording differs.
Defaults: Defaults:
- Enabled by default. - Enabled by default.
- Watches memory files for changes (debounced). - Watches memory files for changes (debounced).
- Uses remote embeddings by default. If `memorySearch.provider` is not set, OpenClaw auto-selects: - Uses remote embeddings by default. If `memorySearch.provider` is not set, Moltbot auto-selects:
1. `local` if a `memorySearch.local.modelPath` is configured and the file exists. 1. `local` if a `memorySearch.local.modelPath` is configured and the file exists.
2. `openai` if an OpenAI key can be resolved. 2. `openai` if an OpenAI key can be resolved.
3. `gemini` if a Gemini key can be resolved. 3. `gemini` if a Gemini key can be resolved.
@@ -94,13 +89,91 @@ Defaults:
- Local mode uses node-llama-cpp and may require `pnpm approve-builds`. - Local mode uses node-llama-cpp and may require `pnpm approve-builds`.
- Uses sqlite-vec (when available) to accelerate vector search inside SQLite. - Uses sqlite-vec (when available) to accelerate vector search inside SQLite.
Remote embeddings **require** an API key for the embedding provider. OpenClaw Remote embeddings **require** an API key for the embedding provider. Moltbot
resolves keys from auth profiles, `models.providers.*.apiKey`, or environment resolves keys from auth profiles, `models.providers.*.apiKey`, or environment
variables. Codex OAuth only covers chat/completions and does **not** satisfy variables. Codex OAuth only covers chat/completions and does **not** satisfy
embeddings for memory search. For Gemini, use `GEMINI_API_KEY` or embeddings for memory search. For Gemini, use `GEMINI_API_KEY` or
`models.providers.google.apiKey`. When using a custom OpenAI-compatible endpoint, `models.providers.google.apiKey`. When using a custom OpenAI-compatible endpoint,
set `memorySearch.remote.apiKey` (and optional `memorySearch.remote.headers`). set `memorySearch.remote.apiKey` (and optional `memorySearch.remote.headers`).
### QMD backend (experimental)
Set `memory.backend = "qmd"` to swap the built-in SQLite indexer for
[QMD](https://github.com/tobi/qmd): a local-first search sidecar that combines
BM25 + vectors + reranking. Markdown stays the source of truth; Moltbot shells
out to QMD for retrieval. Key points:
**Prereqs**
- Disabled by default. Opt in per-config (`memory.backend = "qmd"`).
- Install the QMD CLI separately (`bun install -g github.com/tobi/qmd` or grab
a release) and make sure the `qmd` binary is on the gateways `PATH`.
- QMD needs an SQLite build that allows extensions (`brew install sqlite` on
macOS). The gateway sets `INDEX_PATH`/`QMD_CONFIG_DIR` automatically.
**How the sidecar runs**
- The gateway writes a self-contained QMD home under
`~/.clawdbot/agents/<agentId>/qmd/` (config + cache + sqlite DB).
- Collections are rewritten from `memory.qmd.paths` (plus default workspace
memory files) into `index.yml`, then `qmd update` + `qmd embed` run on boot and
on a configurable interval (`memory.qmd.update.interval`, default 5m).
- Searches run via `qmd query --json`. If QMD fails or the binary is missing,
Moltbot automatically falls back to the builtin SQLite manager so memory tools
keep working.
**Config surface (`memory.qmd.*`)**
- `command` (default `qmd`): override the executable path.
- `includeDefaultMemory` (default `true`): auto-index `MEMORY.md` + `memory/**/*.md`.
- `paths[]`: add extra directories/files (`path`, optional `pattern`, optional
stable `name`).
- `sessions`: opt into session JSONL indexing (`enabled`, `retentionDays`,
`exportDir`, `redactToolOutputs`—defaults to redacting tool payloads).
- `update`: controls refresh cadence (`interval`, `debounceMs`, `onBoot`).
- `limits`: clamp recall payload (`maxResults`, `maxSnippetChars`,
`maxInjectedChars`, `timeoutMs`).
- `scope`: same schema as [`session.sendPolicy`](/reference/configuration#session-sendpolicy).
Default is DM-only (`deny` all, `allow` direct chats); loosen it to surface QMD
hits in groups/channels.
- Snippets sourced outside the workspace show up as
`qmd/<collection>/<relative-path>` in `memory_search` results; `memory_get`
understands that prefix and reads from the configured QMD collection root.
- When `memory.qmd.sessions.enabled = true`, Moltbot exports sanitized session
transcripts (User/Assistant turns) into a dedicated QMD collection under
`~/.clawdbot/agents/<id>/qmd/sessions/`, so `memory_search` can recall recent
conversations without touching the builtin SQLite index.
- `memory_search` snippets now include a `Source: <path#line>` footer when
`memory.citations` is `auto`/`on`; set `memory.citations = "off"` to keep
the path metadata internal (the agent still receives the path for
`memory_get`, but the snippet text omits the footer and the system prompt
warns the agent not to cite it).
**Example**
```json5
memory: {
backend: "qmd",
citations: "auto",
qmd: {
includeDefaultMemory: true,
update: { interval: "5m", debounceMs: 15000 },
limits: { maxResults: 6, timeoutMs: 4000 },
scope: {
default: "deny",
rules: [{ action: "allow", match: { chatType: "direct" } }]
},
paths: [
{ name: "docs", path: "~/notes", pattern: "**/*.md" }
]
}
}
```
**Citations & fallback**
- `memory.citations` applies regardless of backend (`auto`/`on`/`off`).
- When `qmd` runs, we tag `status().backend = "qmd"` so diagnostics show which
engine served the results. If the QMD subprocess exits or JSON output cant be
parsed, the search manager logs a warning and returns the builtin provider
(existing Markdown embeddings) until QMD recovers.
### Additional memory paths ### Additional memory paths
If you want to index Markdown files outside the default workspace layout, add If you want to index Markdown files outside the default workspace layout, add
@@ -142,7 +215,6 @@ agents: {
``` ```
Notes: Notes:
- `remote.baseUrl` is optional (defaults to the Gemini API base URL). - `remote.baseUrl` is optional (defaults to the Gemini API base URL).
- `remote.headers` lets you add extra headers if needed. - `remote.headers` lets you add extra headers if needed.
- Default model: `gemini-embedding-001`. - Default model: `gemini-embedding-001`.
@@ -170,12 +242,10 @@ If you don't want to set an API key, use `memorySearch.provider = "local"` or se
`memorySearch.fallback = "none"`. `memorySearch.fallback = "none"`.
Fallbacks: Fallbacks:
- `memorySearch.fallback` can be `openai`, `gemini`, `local`, or `none`. - `memorySearch.fallback` can be `openai`, `gemini`, `local`, or `none`.
- The fallback provider is only used when the primary embedding provider fails. - The fallback provider is only used when the primary embedding provider fails.
Batch indexing (OpenAI + Gemini): Batch indexing (OpenAI + Gemini):
- Enabled by default for OpenAI and Gemini embeddings. Set `agents.defaults.memorySearch.remote.batch.enabled = false` to disable. - Enabled by default for OpenAI and Gemini embeddings. Set `agents.defaults.memorySearch.remote.batch.enabled = false` to disable.
- Default behavior waits for batch completion; tune `remote.batch.wait`, `remote.batch.pollIntervalMs`, and `remote.batch.timeoutMinutes` if needed. - Default behavior waits for batch completion; tune `remote.batch.wait`, `remote.batch.pollIntervalMs`, and `remote.batch.timeoutMinutes` if needed.
- Set `remote.batch.concurrency` to control how many batch jobs we submit in parallel (default: 2). - Set `remote.batch.concurrency` to control how many batch jobs we submit in parallel (default: 2).
@@ -183,7 +253,6 @@ Batch indexing (OpenAI + Gemini):
- Gemini batch jobs use the async embeddings batch endpoint and require Gemini Batch API availability. - Gemini batch jobs use the async embeddings batch endpoint and require Gemini Batch API availability.
Why OpenAI batch is fast + cheap: Why OpenAI batch is fast + cheap:
- For large backfills, OpenAI is typically the fastest option we support because we can submit many embedding requests in a single batch job and let OpenAI process them asynchronously. - For large backfills, OpenAI is typically the fastest option we support because we can submit many embedding requests in a single batch job and let OpenAI process them asynchronously.
- OpenAI offers discounted pricing for Batch API workloads, so large indexing runs are usually cheaper than sending the same requests synchronously. - OpenAI offers discounted pricing for Batch API workloads, so large indexing runs are usually cheaper than sending the same requests synchronously.
- See the OpenAI Batch API docs and pricing for details: - See the OpenAI Batch API docs and pricing for details:
@@ -209,12 +278,10 @@ agents: {
``` ```
Tools: Tools:
- `memory_search` — returns snippets with file + line ranges. - `memory_search` — returns snippets with file + line ranges.
- `memory_get` — read memory file content by path. - `memory_get` — read memory file content by path.
Local mode: Local mode:
- Set `agents.defaults.memorySearch.provider = "local"`. - Set `agents.defaults.memorySearch.provider = "local"`.
- Provide `agents.defaults.memorySearch.local.modelPath` (GGUF or `hf:` URI). - Provide `agents.defaults.memorySearch.local.modelPath` (GGUF or `hf:` URI).
- Optional: set `agents.defaults.memorySearch.fallback = "none"` to avoid remote fallback. - Optional: set `agents.defaults.memorySearch.fallback = "none"` to avoid remote fallback.
@@ -222,34 +289,31 @@ Local mode:
### How the memory tools work ### How the memory tools work
- `memory_search` semantically searches Markdown chunks (~400 token target, 80-token overlap) from `MEMORY.md` + `memory/**/*.md`. It returns snippet text (capped ~700 chars), file path, line range, score, provider/model, and whether we fell back from local → remote embeddings. No full file payload is returned. - `memory_search` semantically searches Markdown chunks (~400 token target, 80-token overlap) from `MEMORY.md` + `memory/**/*.md`. It returns snippet text (capped ~700 chars), file path, line range, score, provider/model, and whether we fell back from local → remote embeddings. No full file payload is returned.
- `memory_get` reads a specific memory Markdown file (workspace-relative), optionally from a starting line and for N lines. Paths outside `MEMORY.md` / `memory/` are allowed only when explicitly listed in `memorySearch.extraPaths`. - `memory_get` reads a specific memory Markdown file (workspace-relative), optionally from a starting line and for N lines. Paths outside `MEMORY.md` / `memory/` are rejected.
- Both tools are enabled only when `memorySearch.enabled` resolves true for the agent. - Both tools are enabled only when `memorySearch.enabled` resolves true for the agent.
### What gets indexed (and when) ### What gets indexed (and when)
- File type: Markdown only (`MEMORY.md`, `memory/**/*.md`, plus any `.md` files under `memorySearch.extraPaths`). - File type: Markdown only (`MEMORY.md`, `memory/**/*.md`).
- Index storage: per-agent SQLite at `~/.openclaw/memory/<agentId>.sqlite` (configurable via `agents.defaults.memorySearch.store.path`, supports `{agentId}` token). - Index storage: per-agent SQLite at `~/.clawdbot/memory/<agentId>.sqlite` (configurable via `agents.defaults.memorySearch.store.path`, supports `{agentId}` token).
- Freshness: watcher on `MEMORY.md`, `memory/`, and `memorySearch.extraPaths` marks the index dirty (debounce 1.5s). Sync is scheduled on session start, on search, or on an interval and runs asynchronously. Session transcripts use delta thresholds to trigger background sync. - Freshness: watcher on `MEMORY.md` + `memory/` marks the index dirty (debounce 1.5s). Sync is scheduled on session start, on search, or on an interval and runs asynchronously. Session transcripts use delta thresholds to trigger background sync.
- Reindex triggers: the index stores the embedding **provider/model + endpoint fingerprint + chunking params**. If any of those change, OpenClaw automatically resets and reindexes the entire store. - Reindex triggers: the index stores the embedding **provider/model + endpoint fingerprint + chunking params**. If any of those change, Moltbot automatically resets and reindexes the entire store.
### Hybrid search (BM25 + vector) ### Hybrid search (BM25 + vector)
When enabled, OpenClaw combines: When enabled, Moltbot combines:
- **Vector similarity** (semantic match, wording can differ) - **Vector similarity** (semantic match, wording can differ)
- **BM25 keyword relevance** (exact tokens like IDs, env vars, code symbols) - **BM25 keyword relevance** (exact tokens like IDs, env vars, code symbols)
If full-text search is unavailable on your platform, OpenClaw falls back to vector-only search. If full-text search is unavailable on your platform, Moltbot falls back to vector-only search.
#### Why hybrid? #### Why hybrid?
Vector search is great at “this means the same thing”: Vector search is great at “this means the same thing”:
- “Mac Studio gateway host” vs “the machine running the gateway” - “Mac Studio gateway host” vs “the machine running the gateway”
- “debounce file updates” vs “avoid indexing on every write” - “debounce file updates” vs “avoid indexing on every write”
But it can be weak at exact, high-signal tokens: But it can be weak at exact, high-signal tokens:
- IDs (`a828e60`, `b3b9895a…`) - IDs (`a828e60`, `b3b9895a…`)
- code symbols (`memorySearch.query.hybrid`) - code symbols (`memorySearch.query.hybrid`)
- error strings (“sqlite-vec unavailable”) - error strings (“sqlite-vec unavailable”)
@@ -262,21 +326,17 @@ good results for both “natural language” queries and “needle in a haystack
Implementation sketch: Implementation sketch:
1. Retrieve a candidate pool from both sides: 1) Retrieve a candidate pool from both sides:
- **Vector**: top `maxResults * candidateMultiplier` by cosine similarity. - **Vector**: top `maxResults * candidateMultiplier` by cosine similarity.
- **BM25**: top `maxResults * candidateMultiplier` by FTS5 BM25 rank (lower is better). - **BM25**: top `maxResults * candidateMultiplier` by FTS5 BM25 rank (lower is better).
2. Convert BM25 rank into a 0..1-ish score: 2) Convert BM25 rank into a 0..1-ish score:
- `textScore = 1 / (1 + max(0, bm25Rank))` - `textScore = 1 / (1 + max(0, bm25Rank))`
3. Union candidates by chunk id and compute a weighted score: 3) Union candidates by chunk id and compute a weighted score:
- `finalScore = vectorWeight * vectorScore + textWeight * textScore` - `finalScore = vectorWeight * vectorScore + textWeight * textScore`
Notes: Notes:
- `vectorWeight` + `textWeight` is normalized to 1.0 in config resolution, so weights behave as percentages. - `vectorWeight` + `textWeight` is normalized to 1.0 in config resolution, so weights behave as percentages.
- If embeddings are unavailable (or the provider returns a zero-vector), we still run BM25 and return keyword matches. - If embeddings are unavailable (or the provider returns a zero-vector), we still run BM25 and return keyword matches.
- If FTS5 cant be created, we keep vector-only search (no hard failure). - If FTS5 cant be created, we keep vector-only search (no hard failure).
@@ -306,7 +366,7 @@ agents: {
### Embedding cache ### Embedding cache
OpenClaw can cache **chunk embeddings** in SQLite so reindexing and frequent updates (especially session transcripts) don't re-embed unchanged text. Moltbot can cache **chunk embeddings** in SQLite so reindexing and frequent updates (especially session transcripts) don't re-embed unchanged text.
Config: Config:
@@ -340,13 +400,12 @@ agents: {
``` ```
Notes: Notes:
- Session indexing is **opt-in** (off by default). - Session indexing is **opt-in** (off by default).
- Session updates are debounced and **indexed asynchronously** once they cross delta thresholds (best-effort). - Session updates are debounced and **indexed asynchronously** once they cross delta thresholds (best-effort).
- `memory_search` never blocks on indexing; results can be slightly stale until background sync finishes. - `memory_search` never blocks on indexing; results can be slightly stale until background sync finishes.
- Results still include snippets only; `memory_get` remains limited to memory files. - Results still include snippets only; `memory_get` remains limited to memory files.
- Session indexing is isolated per agent (only that agents session logs are indexed). - Session indexing is isolated per agent (only that agents session logs are indexed).
- Session logs live on disk (`~/.openclaw/agents/<agentId>/sessions/*.jsonl`). Any process/user with filesystem access can read them, so treat disk access as the trust boundary. For stricter isolation, run agents under separate OS users or hosts. - Session logs live on disk (`~/.clawdbot/agents/<agentId>/sessions/*.jsonl`). Any process/user with filesystem access can read them, so treat disk access as the trust boundary. For stricter isolation, run agents under separate OS users or hosts.
Delta thresholds (defaults shown): Delta thresholds (defaults shown):
@@ -367,7 +426,7 @@ agents: {
### SQLite vector acceleration (sqlite-vec) ### SQLite vector acceleration (sqlite-vec)
When the sqlite-vec extension is available, OpenClaw stores embeddings in a When the sqlite-vec extension is available, Moltbot stores embeddings in a
SQLite virtual table (`vec0`) and performs vector distance queries in the SQLite virtual table (`vec0`) and performs vector distance queries in the
database. This keeps search fast without loading every embedding into JS. database. This keeps search fast without loading every embedding into JS.
@@ -389,10 +448,9 @@ agents: {
``` ```
Notes: Notes:
- `enabled` defaults to true; when disabled, search falls back to in-process - `enabled` defaults to true; when disabled, search falls back to in-process
cosine similarity over stored embeddings. cosine similarity over stored embeddings.
- If the sqlite-vec extension is missing or fails to load, OpenClaw logs the - If the sqlite-vec extension is missing or fails to load, Moltbot logs the
error and continues with the JS fallback (no vector table). error and continues with the JS fallback (no vector table).
- `extensionPath` overrides the bundled sqlite-vec path (useful for custom builds - `extensionPath` overrides the bundled sqlite-vec path (useful for custom builds
or non-standard install locations). or non-standard install locations).
@@ -426,6 +484,5 @@ agents: {
``` ```
Notes: Notes:
- `remote.*` takes precedence over `models.providers.openai.*`. - `remote.*` takes precedence over `models.providers.openai.*`.
- `remote.headers` merge with OpenAI headers; remote wins on key conflicts. Omit `remote.headers` to use the OpenAI defaults. - `remote.headers` merge with OpenAI headers; remote wins on key conflicts. Omit `remote.headers` to use the OpenAI defaults.

View File

@@ -245,6 +245,7 @@ export function buildSystemPrompt(params: {
userTimeFormat, userTimeFormat,
contextFiles: params.contextFiles, contextFiles: params.contextFiles,
ttsHint, ttsHint,
memoryCitationsMode: params.config?.memory?.citations,
}); });
} }

View File

@@ -351,6 +351,7 @@ export async function compactEmbeddedPiSessionDirect(
userTime, userTime,
userTimeFormat, userTimeFormat,
contextFiles, contextFiles,
memoryCitationsMode: params.config?.memory?.citations,
}); });
const systemPromptOverride = createSystemPromptOverride(appendPrompt); const systemPromptOverride = createSystemPromptOverride(appendPrompt);

View File

@@ -367,6 +367,7 @@ export async function runEmbeddedAttempt(
userTime, userTime,
userTimeFormat, userTimeFormat,
contextFiles, contextFiles,
memoryCitationsMode: params.config?.memory?.citations,
}); });
const systemPromptReport = buildSystemPromptReport({ const systemPromptReport = buildSystemPromptReport({
source: "run", source: "run",

View File

@@ -1,11 +1,12 @@
import type { AgentTool } from "@mariozechner/pi-agent-core"; import type { AgentTool } from "@mariozechner/pi-agent-core";
import type { AgentSession } from "@mariozechner/pi-coding-agent"; import type { AgentSession } from "@mariozechner/pi-coding-agent";
import type { MemoryCitationsMode } from "../../config/types.memory.js";
import type { ResolvedTimeFormat } from "../date-time.js"; import type { ResolvedTimeFormat } from "../date-time.js";
import type { EmbeddedContextFile } from "../pi-embedded-helpers.js"; import type { EmbeddedContextFile } from "../pi-embedded-helpers.js";
import type { EmbeddedSandboxInfo } from "./types.js";
import type { ReasoningLevel, ThinkLevel } from "./utils.js";
import { buildAgentSystemPrompt, type PromptMode } from "../system-prompt.js"; import { buildAgentSystemPrompt, type PromptMode } from "../system-prompt.js";
import { buildToolSummaryMap } from "../tool-summaries.js"; import { buildToolSummaryMap } from "../tool-summaries.js";
import type { EmbeddedSandboxInfo } from "./types.js";
import type { ReasoningLevel, ThinkLevel } from "./utils.js";
export function buildEmbeddedSystemPrompt(params: { export function buildEmbeddedSystemPrompt(params: {
workspaceDir: string; workspaceDir: string;
@@ -46,6 +47,7 @@ export function buildEmbeddedSystemPrompt(params: {
userTime?: string; userTime?: string;
userTimeFormat?: ResolvedTimeFormat; userTimeFormat?: ResolvedTimeFormat;
contextFiles?: EmbeddedContextFile[]; contextFiles?: EmbeddedContextFile[];
memoryCitationsMode?: MemoryCitationsMode;
}): string { }): string {
return buildAgentSystemPrompt({ return buildAgentSystemPrompt({
workspaceDir: params.workspaceDir, workspaceDir: params.workspaceDir,
@@ -71,6 +73,7 @@ export function buildEmbeddedSystemPrompt(params: {
userTime: params.userTime, userTime: params.userTime,
userTimeFormat: params.userTimeFormat, userTimeFormat: params.userTimeFormat,
contextFiles: params.contextFiles, contextFiles: params.contextFiles,
memoryCitationsMode: params.memoryCitationsMode,
}); });
} }

View File

@@ -1,13 +1,14 @@
import type { ReasoningLevel, ThinkLevel } from "../auto-reply/thinking.js"; import type { ReasoningLevel, ThinkLevel } from "../auto-reply/thinking.js";
import type { ResolvedTimeFormat } from "./date-time.js";
import type { EmbeddedContextFile } from "./pi-embedded-helpers.js";
import { SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js"; import { SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js";
import { listDeliverableMessageChannels } from "../utils/message-channel.js"; import { listDeliverableMessageChannels } from "../utils/message-channel.js";
import type { MemoryCitationsMode } from "../config/types.memory.js";
import type { ResolvedTimeFormat } from "./date-time.js";
import type { EmbeddedContextFile } from "./pi-embedded-helpers.js";
/** /**
* Controls which hardcoded sections are included in the system prompt. * Controls which hardcoded sections are included in the system prompt.
* - "full": All sections (default, for main agent) * - "full": All sections (default, for main agent)
* - "minimal": Reduced sections (Tooling, Safety, Workspace, Sandbox, Runtime) - used for subagents * - "minimal": Reduced sections (Tooling, Workspace, Runtime) - used for subagents
* - "none": Just basic identity line, no sections * - "none": Just basic identity line, no sections
*/ */
export type PromptMode = "full" | "minimal" | "none"; export type PromptMode = "full" | "minimal" | "none";
@@ -17,13 +18,9 @@ function buildSkillsSection(params: {
isMinimal: boolean; isMinimal: boolean;
readToolName: string; readToolName: string;
}) { }) {
if (params.isMinimal) { if (params.isMinimal) return [];
return [];
}
const trimmed = params.skillsPrompt?.trim(); const trimmed = params.skillsPrompt?.trim();
if (!trimmed) { if (!trimmed) return [];
return [];
}
return [ return [
"## Skills (mandatory)", "## Skills (mandatory)",
"Before replying: scan <available_skills> <description> entries.", "Before replying: scan <available_skills> <description> entries.",
@@ -36,53 +33,44 @@ function buildSkillsSection(params: {
]; ];
} }
function buildMemorySection(params: { isMinimal: boolean; availableTools: Set<string> }) { function buildMemorySection(params: {
if (params.isMinimal) { isMinimal: boolean;
return []; availableTools: Set<string>;
} citationsMode?: MemoryCitationsMode;
}) {
if (params.isMinimal) return [];
if (!params.availableTools.has("memory_search") && !params.availableTools.has("memory_get")) { if (!params.availableTools.has("memory_search") && !params.availableTools.has("memory_get")) {
return []; return [];
} }
return [ const lines = [
"## Memory Recall", "## Memory Recall",
"Before answering anything about prior work, decisions, dates, people, preferences, or todos: run memory_search on MEMORY.md + memory/*.md; then use memory_get to pull only the needed lines. If low confidence after search, say you checked.", "Before answering anything about prior work, decisions, dates, people, preferences, or todos: run memory_search on MEMORY.md + memory/*.md; then use memory_get to pull only the needed lines. If low confidence after search, say you checked.",
"",
]; ];
if (params.citationsMode === "off") {
lines.push(
"Citations are disabled: do not mention file paths or line numbers in replies unless the user explicitly asks.",
);
} else {
lines.push(
"Citations: include Source: <path#line> when it helps the user verify memory snippets.",
);
}
lines.push("");
return lines;
} }
function buildUserIdentitySection(ownerLine: string | undefined, isMinimal: boolean) { function buildUserIdentitySection(ownerLine: string | undefined, isMinimal: boolean) {
if (!ownerLine || isMinimal) { if (!ownerLine || isMinimal) return [];
return [];
}
return ["## User Identity", ownerLine, ""]; return ["## User Identity", ownerLine, ""];
} }
function buildTimeSection(params: { userTimezone?: string }) { function buildTimeSection(params: { userTimezone?: string }) {
if (!params.userTimezone) { if (!params.userTimezone) return [];
return []; return ["## Current Date & Time", `Time zone: ${params.userTimezone}`, ""];
}
return [
"## Current Date & Time",
`Time zone: ${params.userTimezone}`,
"If you need the current date, time, or day of week, use the session_status tool.",
"",
];
}
function buildSafetySection() {
return [
"## Safety",
"You have no independent goals: do not pursue self-preservation, replication, resource acquisition, or power-seeking; avoid long-term plans beyond the user's request.",
"Prioritize safety and human oversight over completion; if instructions conflict, pause and ask; comply with stop/pause/audit requests and never bypass safeguards. (Inspired by Anthropic's constitution.)",
"Do not manipulate or persuade anyone to expand access or disable safeguards. Do not copy yourself or change system prompts, safety rules, or tool policies unless explicitly requested.",
"",
];
} }
function buildReplyTagsSection(isMinimal: boolean) { function buildReplyTagsSection(isMinimal: boolean) {
if (isMinimal) { if (isMinimal) return [];
return [];
}
return [ return [
"## Reply Tags", "## Reply Tags",
"To request a native reply/quote on supported surfaces, include one tag in your reply:", "To request a native reply/quote on supported surfaces, include one tag in your reply:",
@@ -102,14 +90,12 @@ function buildMessagingSection(params: {
runtimeChannel?: string; runtimeChannel?: string;
messageToolHints?: string[]; messageToolHints?: string[];
}) { }) {
if (params.isMinimal) { if (params.isMinimal) return [];
return [];
}
return [ return [
"## Messaging", "## Messaging",
"- Reply in current session → automatically routes to the source channel (Signal, Telegram, etc.)", "- Reply in current session → automatically routes to the source channel (Signal, Telegram, etc.)",
"- Cross-session messaging → use sessions_send(sessionKey, message)", "- Cross-session messaging → use sessions_send(sessionKey, message)",
"- Never use exec/curl for provider messaging; OpenClaw handles all routing internally.", "- Never use exec/curl for provider messaging; Moltbot handles all routing internally.",
params.availableTools.has("message") params.availableTools.has("message")
? [ ? [
"", "",
@@ -133,30 +119,24 @@ function buildMessagingSection(params: {
} }
function buildVoiceSection(params: { isMinimal: boolean; ttsHint?: string }) { function buildVoiceSection(params: { isMinimal: boolean; ttsHint?: string }) {
if (params.isMinimal) { if (params.isMinimal) return [];
return [];
}
const hint = params.ttsHint?.trim(); const hint = params.ttsHint?.trim();
if (!hint) { if (!hint) return [];
return [];
}
return ["## Voice (TTS)", hint, ""]; return ["## Voice (TTS)", hint, ""];
} }
function buildDocsSection(params: { docsPath?: string; isMinimal: boolean; readToolName: string }) { function buildDocsSection(params: { docsPath?: string; isMinimal: boolean; readToolName: string }) {
const docsPath = params.docsPath?.trim(); const docsPath = params.docsPath?.trim();
if (!docsPath || params.isMinimal) { if (!docsPath || params.isMinimal) return [];
return [];
}
return [ return [
"## Documentation", "## Documentation",
`OpenClaw docs: ${docsPath}`, `Moltbot docs: ${docsPath}`,
"Mirror: https://docs.openclaw.ai", "Mirror: https://docs.molt.bot",
"Source: https://github.com/openclaw/openclaw", "Source: https://github.com/moltbot/moltbot",
"Community: https://discord.com/invite/clawd", "Community: https://discord.com/invite/clawd",
"Find new skills: https://clawhub.com", "Find new skills: https://clawdhub.com",
"For OpenClaw behavior, commands, config, or architecture: consult local docs first.", "For Moltbot behavior, commands, config, or architecture: consult local docs first.",
"When diagnosing issues, run `openclaw status` yourself when possible; only ask the user if you lack access (e.g., sandboxed).", "When diagnosing issues, run `moltbot status` yourself when possible; only ask the user if you lack access (e.g., sandboxed).",
"", "",
]; ];
} }
@@ -213,6 +193,7 @@ export function buildAgentSystemPrompt(params: {
level: "minimal" | "extensive"; level: "minimal" | "extensive";
channel: string; channel: string;
}; };
memoryCitationsMode?: MemoryCitationsMode;
}) { }) {
const coreToolSummaries: Record<string, string> = { const coreToolSummaries: Record<string, string> = {
read: "Read file contents", read: "Read file contents",
@@ -232,7 +213,7 @@ export function buildAgentSystemPrompt(params: {
nodes: "List/describe/notify/camera/screen on paired nodes", nodes: "List/describe/notify/camera/screen on paired nodes",
cron: "Manage cron jobs and wake events (use for reminders; when scheduling a reminder, write the systemEvent text as something that will read like a reminder when it fires, and mention that it is a reminder depending on the time gap between setting and firing; include recent context in reminder text if appropriate)", cron: "Manage cron jobs and wake events (use for reminders; when scheduling a reminder, write the systemEvent text as something that will read like a reminder when it fires, and mention that it is a reminder depending on the time gap between setting and firing; include recent context in reminder text if appropriate)",
message: "Send messages and channel actions", message: "Send messages and channel actions",
gateway: "Restart, apply config, or run updates on the running OpenClaw process", gateway: "Restart, apply config, or run updates on the running Moltbot process",
agents_list: "List agent ids allowed for sessions_spawn", agents_list: "List agent ids allowed for sessions_spawn",
sessions_list: "List other sessions (incl. sub-agents) with filters/last", sessions_list: "List other sessions (incl. sub-agents) with filters/last",
sessions_history: "Fetch history for another session/sub-agent", sessions_history: "Fetch history for another session/sub-agent",
@@ -287,9 +268,7 @@ export function buildAgentSystemPrompt(params: {
const externalToolSummaries = new Map<string, string>(); const externalToolSummaries = new Map<string, string>();
for (const [key, value] of Object.entries(params.toolSummaries ?? {})) { for (const [key, value] of Object.entries(params.toolSummaries ?? {})) {
const normalized = key.trim().toLowerCase(); const normalized = key.trim().toLowerCase();
if (!normalized || !value?.trim()) { if (!normalized || !value?.trim()) continue;
continue;
}
externalToolSummaries.set(normalized, value.trim()); externalToolSummaries.set(normalized, value.trim());
} }
const extraTools = Array.from( const extraTools = Array.from(
@@ -301,7 +280,7 @@ export function buildAgentSystemPrompt(params: {
const name = resolveToolName(tool); const name = resolveToolName(tool);
return summary ? `- ${name}: ${summary}` : `- ${name}`; return summary ? `- ${name}: ${summary}` : `- ${name}`;
}); });
for (const tool of extraTools.toSorted()) { for (const tool of extraTools.sort()) {
const summary = coreToolSummaries[tool] ?? externalToolSummaries.get(tool); const summary = coreToolSummaries[tool] ?? externalToolSummaries.get(tool);
const name = resolveToolName(tool); const name = resolveToolName(tool);
toolLines.push(summary ? `- ${name}: ${summary}` : `- ${name}`); toolLines.push(summary ? `- ${name}: ${summary}` : `- ${name}`);
@@ -351,7 +330,11 @@ export function buildAgentSystemPrompt(params: {
isMinimal, isMinimal,
readToolName, readToolName,
}); });
const memorySection = buildMemorySection({ isMinimal, availableTools }); const memorySection = buildMemorySection({
isMinimal,
availableTools,
citationsMode: params.memoryCitationsMode,
});
const docsSection = buildDocsSection({ const docsSection = buildDocsSection({
docsPath: params.docsPath, docsPath: params.docsPath,
isMinimal, isMinimal,
@@ -361,11 +344,11 @@ export function buildAgentSystemPrompt(params: {
// For "none" mode, return just the basic identity line // For "none" mode, return just the basic identity line
if (promptMode === "none") { if (promptMode === "none") {
return "You are a personal assistant running inside OpenClaw."; return "You are a personal assistant running inside Moltbot.";
} }
const lines = [ const lines = [
"You are a personal assistant running inside OpenClaw.", "You are a personal assistant running inside Moltbot.",
"", "",
"## Tooling", "## Tooling",
"Tool availability (filtered by policy):", "Tool availability (filtered by policy):",
@@ -380,7 +363,7 @@ export function buildAgentSystemPrompt(params: {
"- apply_patch: apply multi-file patches", "- apply_patch: apply multi-file patches",
`- ${execToolName}: run shell commands (supports background via yieldMs/background)`, `- ${execToolName}: run shell commands (supports background via yieldMs/background)`,
`- ${processToolName}: manage background exec sessions`, `- ${processToolName}: manage background exec sessions`,
"- browser: control openclaw's dedicated browser", "- browser: control clawd's dedicated browser",
"- canvas: present/eval/snapshot the Canvas", "- canvas: present/eval/snapshot the Canvas",
"- nodes: list/describe/notify/camera/screen on paired nodes", "- nodes: list/describe/notify/camera/screen on paired nodes",
"- cron: manage cron jobs and wake events (use for reminders; when scheduling a reminder, write the systemEvent text as something that will read like a reminder when it fires, and mention that it is a reminder depending on the time gap between setting and firing; include recent context in reminder text if appropriate)", "- cron: manage cron jobs and wake events (use for reminders; when scheduling a reminder, write the systemEvent text as something that will read like a reminder when it fires, and mention that it is a reminder depending on the time gap between setting and firing; include recent context in reminder text if appropriate)",
@@ -397,26 +380,25 @@ export function buildAgentSystemPrompt(params: {
"Keep narration brief and value-dense; avoid repeating obvious steps.", "Keep narration brief and value-dense; avoid repeating obvious steps.",
"Use plain human language for narration unless in a technical context.", "Use plain human language for narration unless in a technical context.",
"", "",
...buildSafetySection(), "## Moltbot CLI Quick Reference",
"## OpenClaw CLI Quick Reference", "Moltbot is controlled via subcommands. Do not invent commands.",
"OpenClaw is controlled via subcommands. Do not invent commands.",
"To manage the Gateway daemon service (start/stop/restart):", "To manage the Gateway daemon service (start/stop/restart):",
"- openclaw gateway status", "- moltbot gateway status",
"- openclaw gateway start", "- moltbot gateway start",
"- openclaw gateway stop", "- moltbot gateway stop",
"- openclaw gateway restart", "- moltbot gateway restart",
"If unsure, ask the user to run `openclaw help` (or `openclaw gateway --help`) and paste the output.", "If unsure, ask the user to run `moltbot help` (or `moltbot gateway --help`) and paste the output.",
"", "",
...skillsSection, ...skillsSection,
...memorySection, ...memorySection,
// Skip self-update for subagent/none modes // Skip self-update for subagent/none modes
hasGateway && !isMinimal ? "## OpenClaw Self-Update" : "", hasGateway && !isMinimal ? "## Moltbot Self-Update" : "",
hasGateway && !isMinimal hasGateway && !isMinimal
? [ ? [
"Get Updates (self-update) is ONLY allowed when the user explicitly asks for it.", "Get Updates (self-update) is ONLY allowed when the user explicitly asks for it.",
"Do not run config.apply or update.run unless the user explicitly requests an update or config change; if it's not explicit, ask first.", "Do not run config.apply or update.run unless the user explicitly requests an update or config change; if it's not explicit, ask first.",
"Actions: config.get, config.schema, config.apply (validate + write full config, then restart), update.run (update deps or git, then restart).", "Actions: config.get, config.schema, config.apply (validate + write full config, then restart), update.run (update deps or git, then restart).",
"After restart, OpenClaw pings the last active session automatically.", "After restart, Moltbot pings the last active session automatically.",
].join("\n") ].join("\n")
: "", : "",
hasGateway && !isMinimal ? "" : "", hasGateway && !isMinimal ? "" : "",
@@ -485,7 +467,7 @@ export function buildAgentSystemPrompt(params: {
userTimezone, userTimezone,
}), }),
"## Workspace Files (injected)", "## Workspace Files (injected)",
"These user-editable files are loaded by OpenClaw and included below in Project Context.", "These user-editable files are loaded by Moltbot and included below in Project Context.",
"", "",
...buildReplyTagsSection(isMinimal), ...buildReplyTagsSection(isMinimal),
...buildMessagingSection({ ...buildMessagingSection({
@@ -576,7 +558,7 @@ export function buildAgentSystemPrompt(params: {
heartbeatPromptLine, heartbeatPromptLine,
"If you receive a heartbeat poll (a user message matching the heartbeat prompt above), and there is nothing that needs attention, reply exactly:", "If you receive a heartbeat poll (a user message matching the heartbeat prompt above), and there is nothing that needs attention, reply exactly:",
"HEARTBEAT_OK", "HEARTBEAT_OK",
'OpenClaw treats a leading/trailing "HEARTBEAT_OK" as a heartbeat ack (and may discard it).', 'Moltbot treats a leading/trailing "HEARTBEAT_OK" as a heartbeat ack (and may discard it).',
'If something needs attention, do NOT include "HEARTBEAT_OK"; reply with the alert text instead.', 'If something needs attention, do NOT include "HEARTBEAT_OK"; reply with the alert text instead.',
"", "",
); );

View File

@@ -0,0 +1,65 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const stubManager = {
search: vi.fn(async () => [
{
path: "MEMORY.md",
startLine: 5,
endLine: 7,
score: 0.9,
snippet: "@@ -5,3 @@\nAssistant: noted",
source: "memory" as const,
},
]),
readFile: vi.fn(),
status: () => ({
backend: "builtin" as const,
files: 1,
chunks: 1,
dirty: false,
workspaceDir: "/workspace",
dbPath: "/workspace/.memory/index.sqlite",
provider: "builtin",
model: "builtin",
requestedProvider: "builtin",
sources: ["memory" as const],
sourceCounts: [{ source: "memory" as const, files: 1, chunks: 1 }],
}),
sync: vi.fn(),
probeVectorAvailability: vi.fn(async () => true),
close: vi.fn(),
};
vi.mock("../../memory/index.js", () => {
return {
getMemorySearchManager: async () => ({ manager: stubManager }),
};
});
import { createMemorySearchTool } from "./memory-tool.js";
beforeEach(() => {
vi.clearAllMocks();
});
describe("memory search citations", () => {
it("appends source information when citations are enabled", async () => {
const cfg = { memory: { citations: "on" }, agents: { list: [{ id: "main", default: true }] } };
const tool = createMemorySearchTool({ config: cfg });
if (!tool) throw new Error("tool missing");
const result = await tool.execute("call_citations_on", { query: "notes" });
const details = result.details as { results: Array<{ snippet: string; citation?: string }> };
expect(details.results[0]?.snippet).toMatch(/Source: MEMORY.md#L5-L7/);
expect(details.results[0]?.citation).toBe("MEMORY.md#L5-L7");
});
it("leaves snippet untouched when citations are off", async () => {
const cfg = { memory: { citations: "off" }, agents: { list: [{ id: "main", default: true }] } };
const tool = createMemorySearchTool({ config: cfg });
if (!tool) throw new Error("tool missing");
const result = await tool.execute("call_citations_off", { query: "notes" });
const details = result.details as { results: Array<{ snippet: string; citation?: string }> };
expect(details.results[0]?.snippet).not.toMatch(/Source:/);
expect(details.results[0]?.citation).toBeUndefined();
});
});

View File

@@ -1,9 +1,12 @@
import { Type } from "@sinclair/typebox"; import { Type } from "@sinclair/typebox";
import type { OpenClawConfig } from "../../config/config.js";
import type { AnyAgentTool } from "./common.js"; import type { MoltbotConfig } from "../../config/config.js";
import type { MemoryCitationsMode } from "../../config/types.memory.js";
import { getMemorySearchManager } from "../../memory/index.js"; import { getMemorySearchManager } from "../../memory/index.js";
import type { MemorySearchResult } from "../../memory/types.js";
import { resolveSessionAgentId } from "../agent-scope.js"; import { resolveSessionAgentId } from "../agent-scope.js";
import { resolveMemorySearchConfig } from "../memory-search.js"; import { resolveMemorySearchConfig } from "../memory-search.js";
import type { AnyAgentTool } from "./common.js";
import { jsonResult, readNumberParam, readStringParam } from "./common.js"; import { jsonResult, readNumberParam, readStringParam } from "./common.js";
const MemorySearchSchema = Type.Object({ const MemorySearchSchema = Type.Object({
@@ -19,20 +22,16 @@ const MemoryGetSchema = Type.Object({
}); });
export function createMemorySearchTool(options: { export function createMemorySearchTool(options: {
config?: OpenClawConfig; config?: MoltbotConfig;
agentSessionKey?: string; agentSessionKey?: string;
}): AnyAgentTool | null { }): AnyAgentTool | null {
const cfg = options.config; const cfg = options.config;
if (!cfg) { if (!cfg) return null;
return null;
}
const agentId = resolveSessionAgentId({ const agentId = resolveSessionAgentId({
sessionKey: options.agentSessionKey, sessionKey: options.agentSessionKey,
config: cfg, config: cfg,
}); });
if (!resolveMemorySearchConfig(cfg, agentId)) { if (!resolveMemorySearchConfig(cfg, agentId)) return null;
return null;
}
return { return {
label: "Memory Search", label: "Memory Search",
name: "memory_search", name: "memory_search",
@@ -51,17 +50,21 @@ export function createMemorySearchTool(options: {
return jsonResult({ results: [], disabled: true, error }); return jsonResult({ results: [], disabled: true, error });
} }
try { try {
const results = await manager.search(query, { const citationsMode = resolveMemoryCitationsMode(cfg);
const includeCitations = citationsMode !== "off";
const rawResults = await manager.search(query, {
maxResults, maxResults,
minScore, minScore,
sessionKey: options.agentSessionKey, sessionKey: options.agentSessionKey,
}); });
const status = manager.status(); const status = manager.status();
const results = decorateCitations(rawResults, includeCitations);
return jsonResult({ return jsonResult({
results, results,
provider: status.provider, provider: status.provider,
model: status.model, model: status.model,
fallback: status.fallback, fallback: status.fallback,
citations: citationsMode,
}); });
} catch (err) { } catch (err) {
const message = err instanceof Error ? err.message : String(err); const message = err instanceof Error ? err.message : String(err);
@@ -72,25 +75,21 @@ export function createMemorySearchTool(options: {
} }
export function createMemoryGetTool(options: { export function createMemoryGetTool(options: {
config?: OpenClawConfig; config?: MoltbotConfig;
agentSessionKey?: string; agentSessionKey?: string;
}): AnyAgentTool | null { }): AnyAgentTool | null {
const cfg = options.config; const cfg = options.config;
if (!cfg) { if (!cfg) return null;
return null;
}
const agentId = resolveSessionAgentId({ const agentId = resolveSessionAgentId({
sessionKey: options.agentSessionKey, sessionKey: options.agentSessionKey,
config: cfg, config: cfg,
}); });
if (!resolveMemorySearchConfig(cfg, agentId)) { if (!resolveMemorySearchConfig(cfg, agentId)) return null;
return null;
}
return { return {
label: "Memory Get", label: "Memory Get",
name: "memory_get", name: "memory_get",
description: description:
"Safe snippet read from MEMORY.md, memory/*.md, or configured memorySearch.extraPaths with optional from/lines; use after memory_search to pull only the needed lines and keep context small.", "Safe snippet read from MEMORY.md or memory/*.md with optional from/lines; use after memory_search to pull only the needed lines and keep context small.",
parameters: MemoryGetSchema, parameters: MemoryGetSchema,
execute: async (_toolCallId, params) => { execute: async (_toolCallId, params) => {
const relPath = readStringParam(params, "path", { required: true }); const relPath = readStringParam(params, "path", { required: true });
@@ -117,3 +116,28 @@ export function createMemoryGetTool(options: {
}, },
}; };
} }
function resolveMemoryCitationsMode(cfg: MoltbotConfig): MemoryCitationsMode {
const mode = cfg.memory?.citations;
if (mode === "on" || mode === "off" || mode === "auto") return mode;
return "auto";
}
function decorateCitations(results: MemorySearchResult[], include: boolean): MemorySearchResult[] {
if (!include) {
return results.map((entry) => ({ ...entry, citation: undefined }));
}
return results.map((entry) => {
const citation = formatCitation(entry);
const snippet = `${entry.snippet.trim()}\n\nSource: ${citation}`;
return { ...entry, citation, snippet };
});
}
function formatCitation(entry: MemorySearchResult): string {
const lineRange =
entry.startLine === entry.endLine
? `#L${entry.startLine}`
: `#L${entry.startLine}-L${entry.endLine}`;
return `${entry.path}${lineRange}`;
}

View File

@@ -156,6 +156,7 @@ async function resolveContextReport(
ttsHint, ttsHint,
runtimeInfo, runtimeInfo,
sandboxInfo, sandboxInfo,
memoryCitationsMode: params.cfg?.memory?.citations,
}); });
return buildSystemPromptReport({ return buildSystemPromptReport({

View File

@@ -1,5 +1,3 @@
import type { MemoryIndexManager } from "../memory/manager.js";
import type { RuntimeEnv } from "../runtime.js";
import { withProgress } from "../cli/progress.js"; import { withProgress } from "../cli/progress.js";
import { loadConfig } from "../config/config.js"; import { loadConfig } from "../config/config.js";
import { buildGatewayConnectionDetails, callGateway } from "../gateway/call.js"; import { buildGatewayConnectionDetails, callGateway } from "../gateway/call.js";
@@ -8,14 +6,17 @@ import { probeGateway } from "../gateway/probe.js";
import { collectChannelStatusIssues } from "../infra/channels-status-issues.js"; import { collectChannelStatusIssues } from "../infra/channels-status-issues.js";
import { resolveOsSummary } from "../infra/os-summary.js"; import { resolveOsSummary } from "../infra/os-summary.js";
import { getTailnetHostname } from "../infra/tailscale.js"; import { getTailnetHostname } from "../infra/tailscale.js";
import { getMemorySearchManager } from "../memory/index.js";
import type { MemoryProviderStatus } from "../memory/types.js";
import { runExec } from "../process/exec.js"; import { runExec } from "../process/exec.js";
import { buildChannelsTable } from "./status-all/channels.js"; import type { RuntimeEnv } from "../runtime.js";
import { getAgentLocalStatuses } from "./status.agent-local.js"; import { getAgentLocalStatuses } from "./status.agent-local.js";
import { pickGatewaySelfPresence, resolveGatewayProbeAuth } from "./status.gateway-probe.js"; import { pickGatewaySelfPresence, resolveGatewayProbeAuth } from "./status.gateway-probe.js";
import { getStatusSummary } from "./status.summary.js"; import { getStatusSummary } from "./status.summary.js";
import { getUpdateCheckResult } from "./status.update.js"; import { getUpdateCheckResult } from "./status.update.js";
import { buildChannelsTable } from "./status-all/channels.js";
type MemoryStatusSnapshot = ReturnType<MemoryIndexManager["status"]> & { type MemoryStatusSnapshot = MemoryProviderStatus & {
agentId: string; agentId: string;
}; };
@@ -27,9 +28,7 @@ type MemoryPluginStatus = {
function resolveMemoryPluginStatus(cfg: ReturnType<typeof loadConfig>): MemoryPluginStatus { function resolveMemoryPluginStatus(cfg: ReturnType<typeof loadConfig>): MemoryPluginStatus {
const pluginsEnabled = cfg.plugins?.enabled !== false; const pluginsEnabled = cfg.plugins?.enabled !== false;
if (!pluginsEnabled) { if (!pluginsEnabled) return { enabled: false, slot: null, reason: "plugins disabled" };
return { enabled: false, slot: null, reason: "plugins disabled" };
}
const raw = typeof cfg.plugins?.slots?.memory === "string" ? cfg.plugins.slots.memory.trim() : ""; const raw = typeof cfg.plugins?.slots?.memory === "string" ? cfg.plugins.slots.memory.trim() : "";
if (raw && raw.toLowerCase() === "none") { if (raw && raw.toLowerCase() === "none") {
return { enabled: false, slot: null, reason: 'plugins.slots.memory="none"' }; return { enabled: false, slot: null, reason: 'plugins.slots.memory="none"' };
@@ -127,7 +126,7 @@ export async function scanStatus(
progress.setLabel("Querying channel status…"); progress.setLabel("Querying channel status…");
const channelsStatus = gatewayReachable const channelsStatus = gatewayReachable
? await callGateway({ ? await callGateway<Record<string, unknown>>({
method: "channels.status", method: "channels.status",
params: { params: {
probe: false, probe: false,
@@ -142,31 +141,24 @@ export async function scanStatus(
progress.setLabel("Summarizing channels…"); progress.setLabel("Summarizing channels…");
const channels = await buildChannelsTable(cfg, { const channels = await buildChannelsTable(cfg, {
// Show token previews in regular status; keep `status --all` redacted. // Show token previews in regular status; keep `status --all` redacted.
// Set `OPENCLAW_SHOW_SECRETS=0` to force redaction. // Set `CLAWDBOT_SHOW_SECRETS=0` to force redaction.
showSecrets: process.env.OPENCLAW_SHOW_SECRETS?.trim() !== "0", showSecrets: process.env.CLAWDBOT_SHOW_SECRETS?.trim() !== "0",
}); });
progress.tick(); progress.tick();
progress.setLabel("Checking memory…"); progress.setLabel("Checking memory…");
const memoryPlugin = resolveMemoryPluginStatus(cfg); const memoryPlugin = resolveMemoryPluginStatus(cfg);
const memory = await (async (): Promise<MemoryStatusSnapshot | null> => { const memory = await (async (): Promise<MemoryStatusSnapshot | null> => {
if (!memoryPlugin.enabled) { if (!memoryPlugin.enabled) return null;
return null; if (memoryPlugin.slot !== "memory-core") return null;
}
if (memoryPlugin.slot !== "memory-core") {
return null;
}
const agentId = agentStatus.defaultId ?? "main"; const agentId = agentStatus.defaultId ?? "main";
const { MemoryIndexManager } = await import("../memory/manager.js"); const { manager } = await getMemorySearchManager({ cfg, agentId });
const manager = await MemoryIndexManager.get({ cfg, agentId }).catch(() => null); if (!manager) return null;
if (!manager) {
return null;
}
try { try {
await manager.probeVectorAvailability(); await manager.probeVectorAvailability();
} catch {} } catch {}
const status = manager.status(); const status = manager.status();
await manager.close().catch(() => {}); await manager.close?.().catch(() => {});
return { agentId, ...status }; return { agentId, ...status };
})(); })();
progress.tick(); progress.tick();

View File

@@ -254,6 +254,27 @@ const FIELD_LABELS: Record<string, string> = {
"Memory Search Hybrid Candidate Multiplier", "Memory Search Hybrid Candidate Multiplier",
"agents.defaults.memorySearch.cache.enabled": "Memory Search Embedding Cache", "agents.defaults.memorySearch.cache.enabled": "Memory Search Embedding Cache",
"agents.defaults.memorySearch.cache.maxEntries": "Memory Search Embedding Cache Max Entries", "agents.defaults.memorySearch.cache.maxEntries": "Memory Search Embedding Cache Max Entries",
memory: "Memory",
"memory.backend": "Memory Backend",
"memory.citations": "Memory Citations Mode",
"memory.qmd.command": "QMD Binary",
"memory.qmd.includeDefaultMemory": "QMD Include Default Memory",
"memory.qmd.paths": "QMD Extra Paths",
"memory.qmd.paths.path": "QMD Path",
"memory.qmd.paths.pattern": "QMD Path Pattern",
"memory.qmd.paths.name": "QMD Path Name",
"memory.qmd.sessions.enabled": "QMD Session Indexing",
"memory.qmd.sessions.exportDir": "QMD Session Export Directory",
"memory.qmd.sessions.retentionDays": "QMD Session Retention (days)",
"memory.qmd.sessions.redactToolOutputs": "QMD Session Tool Redaction",
"memory.qmd.update.interval": "QMD Update Interval",
"memory.qmd.update.debounceMs": "QMD Update Debounce (ms)",
"memory.qmd.update.onBoot": "QMD Update on Startup",
"memory.qmd.limits.maxResults": "QMD Max Results",
"memory.qmd.limits.maxSnippetChars": "QMD Max Snippet Chars",
"memory.qmd.limits.maxInjectedChars": "QMD Max Injected Chars",
"memory.qmd.limits.timeoutMs": "QMD Search Timeout (ms)",
"memory.qmd.scope": "QMD Surface Scope",
"auth.profiles": "Auth Profiles", "auth.profiles": "Auth Profiles",
"auth.order": "Auth Profile Order", "auth.order": "Auth Profile Order",
"auth.cooldowns.billingBackoffHours": "Billing Backoff (hours)", "auth.cooldowns.billingBackoffHours": "Billing Backoff (hours)",
@@ -548,6 +569,37 @@ const FIELD_HELP: Record<string, string> = {
"Multiplier for candidate pool size (default: 4).", "Multiplier for candidate pool size (default: 4).",
"agents.defaults.memorySearch.cache.enabled": "agents.defaults.memorySearch.cache.enabled":
"Cache chunk embeddings in SQLite to speed up reindexing and frequent updates (default: true).", "Cache chunk embeddings in SQLite to speed up reindexing and frequent updates (default: true).",
memory: "Memory backend configuration (global).",
"memory.backend": 'Memory backend ("builtin" for Moltbot embeddings, "qmd" for QMD sidecar).',
"memory.citations": 'Default citation behavior ("auto", "on", or "off").',
"memory.qmd.command": "Path to the qmd binary (default: resolves from PATH).",
"memory.qmd.includeDefaultMemory":
"Whether to automatically index MEMORY.md + memory/**/*.md (default: true).",
"memory.qmd.paths":
"Additional directories/files to index with QMD (path + optional glob pattern).",
"memory.qmd.paths.path": "Absolute or ~-relative path to index via QMD.",
"memory.qmd.paths.pattern": "Glob pattern relative to the path root (default: **/*.md).",
"memory.qmd.paths.name":
"Optional stable name for the QMD collection (default derived from path).",
"memory.qmd.sessions.enabled":
"Enable QMD session transcript indexing (experimental, default: false).",
"memory.qmd.sessions.exportDir":
"Override directory for sanitized session exports before indexing.",
"memory.qmd.sessions.retentionDays":
"Retention window for exported sessions before pruning (default: unlimited).",
"memory.qmd.sessions.redactToolOutputs":
"Strip tool call payloads/results when exporting sessions (default: true).",
"memory.qmd.update.interval":
"How often the QMD sidecar refreshes indexes (duration string, default: 5m).",
"memory.qmd.update.debounceMs":
"Minimum delay between successive QMD refresh runs (default: 15000).",
"memory.qmd.update.onBoot": "Run QMD update once on gateway startup (default: true).",
"memory.qmd.limits.maxResults": "Max QMD results returned to the agent loop (default: 6).",
"memory.qmd.limits.maxSnippetChars": "Max characters per snippet pulled from QMD (default: 700).",
"memory.qmd.limits.maxInjectedChars": "Max total characters injected from QMD hits per turn.",
"memory.qmd.limits.timeoutMs": "Per-query timeout for QMD searches (default: 4000).",
"memory.qmd.scope":
"Session/channel scope for QMD recall (same syntax as session.sendPolicy; default: direct-only).",
"agents.defaults.memorySearch.cache.maxEntries": "agents.defaults.memorySearch.cache.maxEntries":
"Optional cap on cached embeddings (best-effort).", "Optional cap on cached embeddings (best-effort).",
"agents.defaults.memorySearch.sync.onSearch": "agents.defaults.memorySearch.sync.onSearch":

View File

@@ -0,0 +1,46 @@
import type { SessionSendPolicyConfig } from "./types.base.js";
export type MemoryBackend = "builtin" | "qmd";
export type MemoryCitationsMode = "auto" | "on" | "off";
export type MemoryConfig = {
backend?: MemoryBackend;
citations?: MemoryCitationsMode;
qmd?: MemoryQmdConfig;
};
export type MemoryQmdConfig = {
command?: string;
includeDefaultMemory?: boolean;
paths?: MemoryQmdIndexPath[];
sessions?: MemoryQmdSessionConfig;
update?: MemoryQmdUpdateConfig;
limits?: MemoryQmdLimitsConfig;
scope?: SessionSendPolicyConfig;
};
export type MemoryQmdIndexPath = {
path: string;
name?: string;
pattern?: string;
};
export type MemoryQmdSessionConfig = {
enabled?: boolean;
exportDir?: string;
retentionDays?: number;
redactToolOutputs?: boolean;
};
export type MemoryQmdUpdateConfig = {
interval?: string;
debounceMs?: number;
onBoot?: boolean;
};
export type MemoryQmdLimitsConfig = {
maxResults?: number;
maxSnippetChars?: number;
maxInjectedChars?: number;
timeoutMs?: number;
};

View File

@@ -23,6 +23,7 @@ import type { NodeHostConfig } from "./types.node-host.js";
import type { PluginsConfig } from "./types.plugins.js"; import type { PluginsConfig } from "./types.plugins.js";
import type { SkillsConfig } from "./types.skills.js"; import type { SkillsConfig } from "./types.skills.js";
import type { ToolsConfig } from "./types.tools.js"; import type { ToolsConfig } from "./types.tools.js";
import type { MemoryConfig } from "./types.memory.js";
export type OpenClawConfig = { export type OpenClawConfig = {
meta?: { meta?: {
@@ -95,6 +96,7 @@ export type OpenClawConfig = {
canvasHost?: CanvasHostConfig; canvasHost?: CanvasHostConfig;
talk?: TalkConfig; talk?: TalkConfig;
gateway?: GatewayConfig; gateway?: GatewayConfig;
memory?: MemoryConfig;
}; };
export type ConfigValidationIssue = { export type ConfigValidationIssue = {

View File

@@ -28,3 +28,4 @@ export * from "./types.telegram.js";
export * from "./types.tts.js"; export * from "./types.tts.js";
export * from "./types.tools.js"; export * from "./types.tools.js";
export * from "./types.whatsapp.js"; export * from "./types.whatsapp.js";
export * from "./types.memory.js";

View File

@@ -15,6 +15,31 @@ const SessionResetConfigSchema = z
}) })
.strict(); .strict();
export const SessionSendPolicySchema = z
.object({
default: z.union([z.literal("allow"), z.literal("deny")]).optional(),
rules: z
.array(
z
.object({
action: z.union([z.literal("allow"), z.literal("deny")]),
match: z
.object({
channel: z.string().optional(),
chatType: z
.union([z.literal("direct"), z.literal("group"), z.literal("channel")])
.optional(),
keyPrefix: z.string().optional(),
})
.strict()
.optional(),
})
.strict(),
)
.optional(),
})
.strict();
export const SessionSchema = z export const SessionSchema = z
.object({ .object({
scope: z.union([z.literal("per-sender"), z.literal("global")]).optional(), scope: z.union([z.literal("per-sender"), z.literal("global")]).optional(),
@@ -50,31 +75,7 @@ export const SessionSchema = z
]) ])
.optional(), .optional(),
mainKey: z.string().optional(), mainKey: z.string().optional(),
sendPolicy: z sendPolicy: SessionSendPolicySchema.optional(),
.object({
default: z.union([z.literal("allow"), z.literal("deny")]).optional(),
rules: z
.array(
z
.object({
action: z.union([z.literal("allow"), z.literal("deny")]),
match: z
.object({
channel: z.string().optional(),
chatType: z
.union([z.literal("direct"), z.literal("group"), z.literal("channel")])
.optional(),
keyPrefix: z.string().optional(),
})
.strict()
.optional(),
})
.strict(),
)
.optional(),
})
.strict()
.optional(),
agentToAgent: z agentToAgent: z
.object({ .object({
maxPingPongTurns: z.number().int().min(0).max(5).optional(), maxPingPongTurns: z.number().int().min(0).max(5).optional(),

View File

@@ -1,11 +1,16 @@
import { z } from "zod"; import { z } from "zod";
import { ToolsSchema } from "./zod-schema.agent-runtime.js"; import { ToolsSchema } from "./zod-schema.agent-runtime.js";
import { AgentsSchema, AudioSchema, BindingsSchema, BroadcastSchema } from "./zod-schema.agents.js";
import { ApprovalsSchema } from "./zod-schema.approvals.js"; import { ApprovalsSchema } from "./zod-schema.approvals.js";
import { AgentsSchema, AudioSchema, BindingsSchema, BroadcastSchema } from "./zod-schema.agents.js";
import { HexColorSchema, ModelsConfigSchema } from "./zod-schema.core.js"; import { HexColorSchema, ModelsConfigSchema } from "./zod-schema.core.js";
import { HookMappingSchema, HooksGmailSchema, InternalHooksSchema } from "./zod-schema.hooks.js"; import { HookMappingSchema, HooksGmailSchema, InternalHooksSchema } from "./zod-schema.hooks.js";
import { ChannelsSchema } from "./zod-schema.providers.js"; import { ChannelsSchema } from "./zod-schema.providers.js";
import { CommandsSchema, MessagesSchema, SessionSchema } from "./zod-schema.session.js"; import {
CommandsSchema,
MessagesSchema,
SessionSchema,
SessionSendPolicySchema,
} from "./zod-schema.session.js";
const BrowserSnapshotDefaultsSchema = z const BrowserSnapshotDefaultsSchema = z
.object({ .object({
@@ -27,7 +32,62 @@ const NodeHostSchema = z
.strict() .strict()
.optional(); .optional();
export const OpenClawSchema = z const MemoryQmdPathSchema = z
.object({
path: z.string(),
name: z.string().optional(),
pattern: z.string().optional(),
})
.strict();
const MemoryQmdSessionSchema = z
.object({
enabled: z.boolean().optional(),
exportDir: z.string().optional(),
retentionDays: z.number().int().nonnegative().optional(),
redactToolOutputs: z.boolean().optional(),
})
.strict();
const MemoryQmdUpdateSchema = z
.object({
interval: z.string().optional(),
debounceMs: z.number().int().nonnegative().optional(),
onBoot: z.boolean().optional(),
})
.strict();
const MemoryQmdLimitsSchema = z
.object({
maxResults: z.number().int().positive().optional(),
maxSnippetChars: z.number().int().positive().optional(),
maxInjectedChars: z.number().int().positive().optional(),
timeoutMs: z.number().int().nonnegative().optional(),
})
.strict();
const MemoryQmdSchema = z
.object({
command: z.string().optional(),
includeDefaultMemory: z.boolean().optional(),
paths: z.array(MemoryQmdPathSchema).optional(),
sessions: MemoryQmdSessionSchema.optional(),
update: MemoryQmdUpdateSchema.optional(),
limits: MemoryQmdLimitsSchema.optional(),
scope: SessionSendPolicySchema.optional(),
})
.strict();
const MemorySchema = z
.object({
backend: z.union([z.literal("builtin"), z.literal("qmd")]).optional(),
citations: z.union([z.literal("auto"), z.literal("on"), z.literal("off")]).optional(),
qmd: MemoryQmdSchema.optional(),
})
.strict()
.optional();
export const MoltbotSchema = z
.object({ .object({
meta: z meta: z
.object({ .object({
@@ -154,7 +214,7 @@ export const OpenClawSchema = z
.object({ .object({
cdpPort: z.number().int().min(1).max(65535).optional(), cdpPort: z.number().int().min(1).max(65535).optional(),
cdpUrl: z.string().optional(), cdpUrl: z.string().optional(),
driver: z.union([z.literal("openclaw"), z.literal("extension")]).optional(), driver: z.union([z.literal("clawd"), z.literal("extension")]).optional(),
color: HexColorSchema, color: HexColorSchema,
}) })
.strict() .strict()
@@ -268,7 +328,6 @@ export const OpenClawSchema = z
wideArea: z wideArea: z
.object({ .object({
enabled: z.boolean().optional(), enabled: z.boolean().optional(),
domain: z.string().optional(),
}) })
.strict() .strict()
.optional(), .optional(),
@@ -446,6 +505,7 @@ export const OpenClawSchema = z
}) })
.strict() .strict()
.optional(), .optional(),
memory: MemorySchema,
skills: z skills: z
.object({ .object({
allowBundled: z.array(z.string()).optional(), allowBundled: z.array(z.string()).optional(),
@@ -532,23 +592,15 @@ export const OpenClawSchema = z
.strict() .strict()
.superRefine((cfg, ctx) => { .superRefine((cfg, ctx) => {
const agents = cfg.agents?.list ?? []; const agents = cfg.agents?.list ?? [];
if (agents.length === 0) { if (agents.length === 0) return;
return;
}
const agentIds = new Set(agents.map((agent) => agent.id)); const agentIds = new Set(agents.map((agent) => agent.id));
const broadcast = cfg.broadcast; const broadcast = cfg.broadcast;
if (!broadcast) { if (!broadcast) return;
return;
}
for (const [peerId, ids] of Object.entries(broadcast)) { for (const [peerId, ids] of Object.entries(broadcast)) {
if (peerId === "strategy") { if (peerId === "strategy") continue;
continue; if (!Array.isArray(ids)) continue;
}
if (!Array.isArray(ids)) {
continue;
}
for (let idx = 0; idx < ids.length; idx += 1) { for (let idx = 0; idx < ids.length; idx += 1) {
const agentId = ids[idx]; const agentId = ids[idx];
if (!agentIds.has(agentId)) { if (!agentIds.has(agentId)) {

View File

@@ -0,0 +1,58 @@
import path from "node:path";
import { describe, expect, it } from "vitest";
import type { MoltbotConfig } from "../config/config.js";
import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js";
import { resolveMemoryBackendConfig } from "./backend-config.js";
describe("resolveMemoryBackendConfig", () => {
it("defaults to builtin backend when config missing", () => {
const cfg = { agents: { defaults: { workspace: "/tmp/memory-test" } } } as MoltbotConfig;
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
expect(resolved.backend).toBe("builtin");
expect(resolved.citations).toBe("auto");
expect(resolved.qmd).toBeUndefined();
});
it("resolves qmd backend with default collections", () => {
const cfg = {
agents: { defaults: { workspace: "/tmp/memory-test" } },
memory: {
backend: "qmd",
qmd: {},
},
} as MoltbotConfig;
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
expect(resolved.backend).toBe("qmd");
expect(resolved.qmd?.collections.length).toBeGreaterThanOrEqual(3);
expect(resolved.qmd?.command).toBe("qmd");
expect(resolved.qmd?.update.intervalMs).toBeGreaterThan(0);
});
it("resolves custom paths relative to workspace", () => {
const cfg = {
agents: {
defaults: { workspace: "/workspace/root" },
list: [{ id: "main", workspace: "/workspace/root" }],
},
memory: {
backend: "qmd",
qmd: {
paths: [
{
path: "notes",
name: "custom-notes",
pattern: "**/*.md",
},
],
},
},
} as MoltbotConfig;
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
const custom = resolved.qmd?.collections.find((c) => c.name.startsWith("custom-notes"));
expect(custom).toBeDefined();
const workspaceRoot = resolveAgentWorkspaceDir(cfg, "main");
expect(custom?.path).toBe(path.resolve(workspaceRoot, "notes"));
});
});

View File

@@ -0,0 +1,245 @@
import path from "node:path";
import { parseDurationMs } from "../cli/parse-duration.js";
import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js";
import type { MoltbotConfig } from "../config/config.js";
import type {
MemoryBackend,
MemoryCitationsMode,
MemoryQmdConfig,
MemoryQmdIndexPath,
} from "../config/types.memory.js";
import type { SessionSendPolicyConfig } from "../config/types.base.js";
import { resolveUserPath } from "../utils.js";
export type ResolvedMemoryBackendConfig = {
backend: MemoryBackend;
citations: MemoryCitationsMode;
qmd?: ResolvedQmdConfig;
};
export type ResolvedQmdCollection = {
name: string;
path: string;
pattern: string;
kind: "memory" | "custom" | "sessions";
};
export type ResolvedQmdUpdateConfig = {
intervalMs: number;
debounceMs: number;
onBoot: boolean;
};
export type ResolvedQmdLimitsConfig = {
maxResults: number;
maxSnippetChars: number;
maxInjectedChars: number;
timeoutMs: number;
};
export type ResolvedQmdSessionConfig = {
enabled: boolean;
exportDir?: string;
retentionDays?: number;
redactToolOutputs: boolean;
};
export type ResolvedQmdConfig = {
command: string;
collections: ResolvedQmdCollection[];
sessions: ResolvedQmdSessionConfig;
update: ResolvedQmdUpdateConfig;
limits: ResolvedQmdLimitsConfig;
includeDefaultMemory: boolean;
scope?: SessionSendPolicyConfig;
};
const DEFAULT_BACKEND: MemoryBackend = "builtin";
const DEFAULT_CITATIONS: MemoryCitationsMode = "auto";
const DEFAULT_QMD_INTERVAL = "5m";
const DEFAULT_QMD_DEBOUNCE_MS = 15_000;
const DEFAULT_QMD_TIMEOUT_MS = 4_000;
const DEFAULT_QMD_LIMITS: ResolvedQmdLimitsConfig = {
maxResults: 6,
maxSnippetChars: 700,
maxInjectedChars: 4_000,
timeoutMs: DEFAULT_QMD_TIMEOUT_MS,
};
const DEFAULT_QMD_SCOPE: SessionSendPolicyConfig = {
default: "deny",
rules: [
{
action: "allow",
match: { chatType: "direct" },
},
],
};
function sanitizeName(input: string): string {
const lower = input.toLowerCase().replace(/[^a-z0-9-]+/g, "-");
const trimmed = lower.replace(/^-+|-+$/g, "");
return trimmed || "collection";
}
function ensureUniqueName(base: string, existing: Set<string>): string {
let name = sanitizeName(base);
if (!existing.has(name)) {
existing.add(name);
return name;
}
let suffix = 2;
while (existing.has(`${name}-${suffix}`)) {
suffix += 1;
}
const unique = `${name}-${suffix}`;
existing.add(unique);
return unique;
}
function resolvePath(raw: string, workspaceDir: string): string {
const trimmed = raw.trim();
if (!trimmed) throw new Error("path required");
if (trimmed.startsWith("~") || path.isAbsolute(trimmed)) {
return path.normalize(resolveUserPath(trimmed));
}
return path.normalize(path.resolve(workspaceDir, trimmed));
}
function resolveIntervalMs(raw: string | undefined): number {
const value = raw?.trim();
if (!value) return parseDurationMs(DEFAULT_QMD_INTERVAL, { defaultUnit: "m" });
try {
return parseDurationMs(value, { defaultUnit: "m" });
} catch {
return parseDurationMs(DEFAULT_QMD_INTERVAL, { defaultUnit: "m" });
}
}
function resolveDebounceMs(raw: number | undefined): number {
if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) {
return Math.floor(raw);
}
return DEFAULT_QMD_DEBOUNCE_MS;
}
function resolveLimits(raw?: MemoryQmdConfig["limits"]): ResolvedQmdLimitsConfig {
const parsed: ResolvedQmdLimitsConfig = { ...DEFAULT_QMD_LIMITS };
if (raw?.maxResults && raw.maxResults > 0) parsed.maxResults = Math.floor(raw.maxResults);
if (raw?.maxSnippetChars && raw.maxSnippetChars > 0) {
parsed.maxSnippetChars = Math.floor(raw.maxSnippetChars);
}
if (raw?.maxInjectedChars && raw.maxInjectedChars > 0) {
parsed.maxInjectedChars = Math.floor(raw.maxInjectedChars);
}
if (raw?.timeoutMs && raw.timeoutMs > 0) {
parsed.timeoutMs = Math.floor(raw.timeoutMs);
}
return parsed;
}
function resolveSessionConfig(
cfg: MemoryQmdConfig["sessions"],
workspaceDir: string,
): ResolvedQmdSessionConfig {
const enabled = Boolean(cfg?.enabled);
const exportDirRaw = cfg?.exportDir?.trim();
const exportDir = exportDirRaw ? resolvePath(exportDirRaw, workspaceDir) : undefined;
const retentionDays =
cfg?.retentionDays && cfg.retentionDays > 0 ? Math.floor(cfg.retentionDays) : undefined;
const redactToolOutputs = cfg?.redactToolOutputs !== false;
return {
enabled,
exportDir,
retentionDays,
redactToolOutputs,
};
}
function resolveCustomPaths(
rawPaths: MemoryQmdIndexPath[] | undefined,
workspaceDir: string,
existing: Set<string>,
): ResolvedQmdCollection[] {
if (!rawPaths?.length) return [];
const collections: ResolvedQmdCollection[] = [];
rawPaths.forEach((entry, index) => {
const trimmedPath = entry?.path?.trim();
if (!trimmedPath) return;
let resolved: string;
try {
resolved = resolvePath(trimmedPath, workspaceDir);
} catch {
return;
}
const pattern = entry.pattern?.trim() || "**/*.md";
const baseName = entry.name?.trim() || `custom-${index + 1}`;
const name = ensureUniqueName(baseName, existing);
collections.push({
name,
path: resolved,
pattern,
kind: "custom",
});
});
return collections;
}
function resolveDefaultCollections(
include: boolean,
workspaceDir: string,
existing: Set<string>,
): ResolvedQmdCollection[] {
if (!include) return [];
const entries: Array<{ path: string; pattern: string; base: string }> = [
{ path: workspaceDir, pattern: "MEMORY.md", base: "memory-root" },
{ path: workspaceDir, pattern: "memory.md", base: "memory-alt" },
{ path: path.join(workspaceDir, "memory"), pattern: "**/*.md", base: "memory-dir" },
];
return entries.map((entry) => ({
name: ensureUniqueName(entry.base, existing),
path: entry.path,
pattern: entry.pattern,
kind: "memory",
}));
}
export function resolveMemoryBackendConfig(params: {
cfg: MoltbotConfig;
agentId: string;
}): ResolvedMemoryBackendConfig {
const backend = params.cfg.memory?.backend ?? DEFAULT_BACKEND;
const citations = params.cfg.memory?.citations ?? DEFAULT_CITATIONS;
if (backend !== "qmd") {
return { backend: "builtin", citations };
}
const workspaceDir = resolveAgentWorkspaceDir(params.cfg, params.agentId);
const qmdCfg = params.cfg.memory?.qmd;
const includeDefaultMemory = qmdCfg?.includeDefaultMemory !== false;
const nameSet = new Set<string>();
const collections = [
...resolveDefaultCollections(includeDefaultMemory, workspaceDir, nameSet),
...resolveCustomPaths(qmdCfg?.paths, workspaceDir, nameSet),
];
const resolved: ResolvedQmdConfig = {
command: qmdCfg?.command?.trim() || "qmd",
collections,
includeDefaultMemory,
sessions: resolveSessionConfig(qmdCfg?.sessions, workspaceDir),
update: {
intervalMs: resolveIntervalMs(qmdCfg?.update?.interval),
debounceMs: resolveDebounceMs(qmdCfg?.update?.debounceMs),
onBoot: qmdCfg?.update?.onBoot !== false,
},
limits: resolveLimits(qmdCfg?.limits),
scope: qmdCfg?.scope ?? DEFAULT_QMD_SCOPE,
};
return {
backend: "qmd",
citations,
qmd: resolved,
};
}

View File

@@ -1,2 +1,3 @@
export type { MemoryIndexManager, MemorySearchResult } from "./manager.js"; export { MemoryIndexManager } from "./manager.js";
export type { MemorySearchResult, MemorySearchManager } from "./types.js";
export { getMemorySearchManager, type MemorySearchManagerResult } from "./search-manager.js"; export { getMemorySearchManager, type MemorySearchManagerResult } from "./search-manager.js";

View File

@@ -1,25 +1,18 @@
import type { DatabaseSync } from "node:sqlite";
import chokidar, { type FSWatcher } from "chokidar";
import { randomUUID } from "node:crypto"; import { randomUUID } from "node:crypto";
import fsSync from "node:fs";
import fs from "node:fs/promises"; import fs from "node:fs/promises";
import path from "node:path"; import path from "node:path";
import type { ResolvedMemorySearchConfig } from "../agents/memory-search.js";
import type { OpenClawConfig } from "../config/config.js"; import type { DatabaseSync } from "node:sqlite";
import chokidar, { type FSWatcher } from "chokidar";
import { resolveAgentDir, resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; import { resolveAgentDir, resolveAgentWorkspaceDir } from "../agents/agent-scope.js";
import type { ResolvedMemorySearchConfig } from "../agents/memory-search.js";
import { resolveMemorySearchConfig } from "../agents/memory-search.js"; import { resolveMemorySearchConfig } from "../agents/memory-search.js";
import type { MoltbotConfig } from "../config/config.js";
import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js"; import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js";
import { createSubsystemLogger } from "../logging/subsystem.js"; import { createSubsystemLogger } from "../logging/subsystem.js";
import { onSessionTranscriptUpdate } from "../sessions/transcript-events.js"; import { onSessionTranscriptUpdate } from "../sessions/transcript-events.js";
import { resolveUserPath } from "../utils.js"; import { resolveUserPath } from "../utils.js";
import { runGeminiEmbeddingBatches, type GeminiBatchRequest } from "./batch-gemini.js";
import {
OPENAI_BATCH_ENDPOINT,
type OpenAiBatchRequest,
runOpenAiEmbeddingBatches,
} from "./batch-openai.js";
import { DEFAULT_GEMINI_EMBEDDING_MODEL } from "./embeddings-gemini.js";
import { DEFAULT_OPENAI_EMBEDDING_MODEL } from "./embeddings-openai.js";
import { import {
createEmbeddingProvider, createEmbeddingProvider,
type EmbeddingProvider, type EmbeddingProvider,
@@ -27,7 +20,14 @@ import {
type GeminiEmbeddingClient, type GeminiEmbeddingClient,
type OpenAiEmbeddingClient, type OpenAiEmbeddingClient,
} from "./embeddings.js"; } from "./embeddings.js";
import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js"; import { DEFAULT_GEMINI_EMBEDDING_MODEL } from "./embeddings-gemini.js";
import { DEFAULT_OPENAI_EMBEDDING_MODEL } from "./embeddings-openai.js";
import {
OPENAI_BATCH_ENDPOINT,
type OpenAiBatchRequest,
runOpenAiEmbeddingBatches,
} from "./batch-openai.js";
import { runGeminiEmbeddingBatches, type GeminiBatchRequest } from "./batch-gemini.js";
import { import {
buildFileEntry, buildFileEntry,
chunkMarkdown, chunkMarkdown,
@@ -35,26 +35,23 @@ import {
hashText, hashText,
isMemoryPath, isMemoryPath,
listMemoryFiles, listMemoryFiles,
normalizeExtraMemoryPaths,
type MemoryChunk, type MemoryChunk,
type MemoryFileEntry, type MemoryFileEntry,
normalizeRelPath,
parseEmbedding, parseEmbedding,
} from "./internal.js"; } from "./internal.js";
import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js";
import { searchKeyword, searchVector } from "./manager-search.js"; import { searchKeyword, searchVector } from "./manager-search.js";
import { ensureMemoryIndexSchema } from "./memory-schema.js"; import { ensureMemoryIndexSchema } from "./memory-schema.js";
import { loadSqliteVecExtension } from "./sqlite-vec.js";
import { requireNodeSqlite } from "./sqlite.js"; import { requireNodeSqlite } from "./sqlite.js";
import { loadSqliteVecExtension } from "./sqlite-vec.js";
type MemorySource = "memory" | "sessions"; import type {
MemoryProviderStatus,
export type MemorySearchResult = { MemorySearchManager,
path: string; MemorySearchResult,
startLine: number; MemorySource,
endLine: number; MemorySyncProgressUpdate,
score: number; } from "./types.js";
snippet: string;
source: MemorySource;
};
type MemoryIndexMeta = { type MemoryIndexMeta = {
model: string; model: string;
@@ -74,12 +71,6 @@ type SessionFileEntry = {
content: string; content: string;
}; };
type MemorySyncProgressUpdate = {
completed: number;
total: number;
label?: string;
};
type MemorySyncProgressState = { type MemorySyncProgressState = {
completed: number; completed: number;
total: number; total: number;
@@ -114,9 +105,9 @@ const INDEX_CACHE = new Map<string, MemoryIndexManager>();
const vectorToBlob = (embedding: number[]): Buffer => const vectorToBlob = (embedding: number[]): Buffer =>
Buffer.from(new Float32Array(embedding).buffer); Buffer.from(new Float32Array(embedding).buffer);
export class MemoryIndexManager { export class MemoryIndexManager implements MemorySearchManager {
private readonly cacheKey: string; private readonly cacheKey: string;
private readonly cfg: OpenClawConfig; private readonly cfg: MoltbotConfig;
private readonly agentId: string; private readonly agentId: string;
private readonly workspaceDir: string; private readonly workspaceDir: string;
private readonly settings: ResolvedMemorySearchConfig; private readonly settings: ResolvedMemorySearchConfig;
@@ -172,20 +163,16 @@ export class MemoryIndexManager {
private syncing: Promise<void> | null = null; private syncing: Promise<void> | null = null;
static async get(params: { static async get(params: {
cfg: OpenClawConfig; cfg: MoltbotConfig;
agentId: string; agentId: string;
}): Promise<MemoryIndexManager | null> { }): Promise<MemoryIndexManager | null> {
const { cfg, agentId } = params; const { cfg, agentId } = params;
const settings = resolveMemorySearchConfig(cfg, agentId); const settings = resolveMemorySearchConfig(cfg, agentId);
if (!settings) { if (!settings) return null;
return null;
}
const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId); const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId);
const key = `${agentId}:${workspaceDir}:${JSON.stringify(settings)}`; const key = `${agentId}:${workspaceDir}:${JSON.stringify(settings)}`;
const existing = INDEX_CACHE.get(key); const existing = INDEX_CACHE.get(key);
if (existing) { if (existing) return existing;
return existing;
}
const providerResult = await createEmbeddingProvider({ const providerResult = await createEmbeddingProvider({
config: cfg, config: cfg,
agentDir: resolveAgentDir(cfg, agentId), agentDir: resolveAgentDir(cfg, agentId),
@@ -209,7 +196,7 @@ export class MemoryIndexManager {
private constructor(params: { private constructor(params: {
cacheKey: string; cacheKey: string;
cfg: OpenClawConfig; cfg: MoltbotConfig;
agentId: string; agentId: string;
workspaceDir: string; workspaceDir: string;
settings: ResolvedMemorySearchConfig; settings: ResolvedMemorySearchConfig;
@@ -252,19 +239,13 @@ export class MemoryIndexManager {
} }
async warmSession(sessionKey?: string): Promise<void> { async warmSession(sessionKey?: string): Promise<void> {
if (!this.settings.sync.onSessionStart) { if (!this.settings.sync.onSessionStart) return;
return;
}
const key = sessionKey?.trim() || ""; const key = sessionKey?.trim() || "";
if (key && this.sessionWarm.has(key)) { if (key && this.sessionWarm.has(key)) return;
return;
}
void this.sync({ reason: "session-start" }).catch((err) => { void this.sync({ reason: "session-start" }).catch((err) => {
log.warn(`memory sync failed (session-start): ${String(err)}`); log.warn(`memory sync failed (session-start): ${String(err)}`);
}); });
if (key) { if (key) this.sessionWarm.add(key);
this.sessionWarm.add(key);
}
} }
async search( async search(
@@ -282,9 +263,7 @@ export class MemoryIndexManager {
}); });
} }
const cleaned = query.trim(); const cleaned = query.trim();
if (!cleaned) { if (!cleaned) return [];
return [];
}
const minScore = opts?.minScore ?? this.settings.query.minScore; const minScore = opts?.minScore ?? this.settings.query.minScore;
const maxResults = opts?.maxResults ?? this.settings.query.maxResults; const maxResults = opts?.maxResults ?? this.settings.query.maxResults;
const hybrid = this.settings.query.hybrid; const hybrid = this.settings.query.hybrid;
@@ -343,9 +322,7 @@ export class MemoryIndexManager {
query: string, query: string,
limit: number, limit: number,
): Promise<Array<MemorySearchResult & { id: string; textScore: number }>> { ): Promise<Array<MemorySearchResult & { id: string; textScore: number }>> {
if (!this.fts.enabled || !this.fts.available) { if (!this.fts.enabled || !this.fts.available) return [];
return [];
}
const sourceFilter = this.buildSourceFilter(); const sourceFilter = this.buildSourceFilter();
const results = await searchKeyword({ const results = await searchKeyword({
db: this.db, db: this.db,
@@ -397,9 +374,7 @@ export class MemoryIndexManager {
force?: boolean; force?: boolean;
progress?: (update: MemorySyncProgressUpdate) => void; progress?: (update: MemorySyncProgressUpdate) => void;
}): Promise<void> { }): Promise<void> {
if (this.syncing) { if (this.syncing) return this.syncing;
return this.syncing;
}
this.syncing = this.runSync(params).finally(() => { this.syncing = this.runSync(params).finally(() => {
this.syncing = null; this.syncing = null;
}); });
@@ -411,54 +386,13 @@ export class MemoryIndexManager {
from?: number; from?: number;
lines?: number; lines?: number;
}): Promise<{ text: string; path: string }> { }): Promise<{ text: string; path: string }> {
const rawPath = params.relPath.trim(); const relPath = normalizeRelPath(params.relPath);
if (!rawPath) { if (!relPath || !isMemoryPath(relPath)) {
throw new Error("path required"); throw new Error("path required");
} }
const absPath = path.isAbsolute(rawPath) const absPath = path.resolve(this.workspaceDir, relPath);
? path.resolve(rawPath) if (!absPath.startsWith(this.workspaceDir)) {
: path.resolve(this.workspaceDir, rawPath); throw new Error("path escapes workspace");
const relPath = path.relative(this.workspaceDir, absPath).replace(/\\/g, "/");
const inWorkspace =
relPath.length > 0 && !relPath.startsWith("..") && !path.isAbsolute(relPath);
const allowedWorkspace = inWorkspace && isMemoryPath(relPath);
let allowedAdditional = false;
if (!allowedWorkspace && this.settings.extraPaths.length > 0) {
const additionalPaths = normalizeExtraMemoryPaths(
this.workspaceDir,
this.settings.extraPaths,
);
for (const additionalPath of additionalPaths) {
try {
const stat = await fs.lstat(additionalPath);
if (stat.isSymbolicLink()) {
continue;
}
if (stat.isDirectory()) {
if (absPath === additionalPath || absPath.startsWith(`${additionalPath}${path.sep}`)) {
allowedAdditional = true;
break;
}
continue;
}
if (stat.isFile()) {
if (absPath === additionalPath && absPath.endsWith(".md")) {
allowedAdditional = true;
break;
}
}
} catch {}
}
}
if (!allowedWorkspace && !allowedAdditional) {
throw new Error("path required");
}
if (!absPath.endsWith(".md")) {
throw new Error("path required");
}
const stat = await fs.lstat(absPath);
if (stat.isSymbolicLink() || !stat.isFile()) {
throw new Error("path required");
} }
const content = await fs.readFile(absPath, "utf-8"); const content = await fs.readFile(absPath, "utf-8");
if (!params.from && !params.lines) { if (!params.from && !params.lines) {
@@ -471,40 +405,7 @@ export class MemoryIndexManager {
return { text: slice.join("\n"), path: relPath }; return { text: slice.join("\n"), path: relPath };
} }
status(): { status(): MemoryProviderStatus {
files: number;
chunks: number;
dirty: boolean;
workspaceDir: string;
dbPath: string;
provider: string;
model: string;
requestedProvider: string;
sources: MemorySource[];
extraPaths: string[];
sourceCounts: Array<{ source: MemorySource; files: number; chunks: number }>;
cache?: { enabled: boolean; entries?: number; maxEntries?: number };
fts?: { enabled: boolean; available: boolean; error?: string };
fallback?: { from: string; reason?: string };
vector?: {
enabled: boolean;
available?: boolean;
extensionPath?: string;
loadError?: string;
dims?: number;
};
batch?: {
enabled: boolean;
failures: number;
limit: number;
wait: boolean;
concurrency: number;
pollIntervalMs: number;
timeoutMs: number;
lastError?: string;
lastProvider?: string;
};
} {
const sourceFilter = this.buildSourceFilter(); const sourceFilter = this.buildSourceFilter();
const files = this.db const files = this.db
.prepare(`SELECT COUNT(*) as c FROM files WHERE 1=1${sourceFilter.sql}`) .prepare(`SELECT COUNT(*) as c FROM files WHERE 1=1${sourceFilter.sql}`)
@@ -518,9 +419,7 @@ export class MemoryIndexManager {
}; };
const sourceCounts = (() => { const sourceCounts = (() => {
const sources = Array.from(this.sources); const sources = Array.from(this.sources);
if (sources.length === 0) { if (sources.length === 0) return [];
return [];
}
const bySource = new Map<MemorySource, { files: number; chunks: number }>(); const bySource = new Map<MemorySource, { files: number; chunks: number }>();
for (const source of sources) { for (const source of sources) {
bySource.set(source, { files: 0, chunks: 0 }); bySource.set(source, { files: 0, chunks: 0 });
@@ -545,19 +444,19 @@ export class MemoryIndexManager {
entry.chunks = row.c ?? 0; entry.chunks = row.c ?? 0;
bySource.set(row.source, entry); bySource.set(row.source, entry);
} }
return sources.map((source) => Object.assign({ source }, bySource.get(source)!)); return sources.map((source) => ({ source, ...bySource.get(source)! }));
})(); })();
return { return {
backend: "builtin",
files: files?.c ?? 0, files: files?.c ?? 0,
chunks: chunks?.c ?? 0, chunks: chunks?.c ?? 0,
dirty: this.dirty, dirty: this.dirty || this.sessionsDirty,
workspaceDir: this.workspaceDir, workspaceDir: this.workspaceDir,
dbPath: this.settings.store.path, dbPath: this.settings.store.path,
provider: this.provider.id, provider: this.provider.id,
model: this.provider.model, model: this.provider.model,
requestedProvider: this.requestedProvider, requestedProvider: this.requestedProvider,
sources: Array.from(this.sources), sources: Array.from(this.sources),
extraPaths: this.settings.extraPaths,
sourceCounts, sourceCounts,
cache: this.cache.enabled cache: this.cache.enabled
? { ? {
@@ -601,9 +500,7 @@ export class MemoryIndexManager {
} }
async probeVectorAvailability(): Promise<boolean> { async probeVectorAvailability(): Promise<boolean> {
if (!this.vector.enabled) { if (!this.vector.enabled) return false;
return false;
}
return this.ensureVectorReady(); return this.ensureVectorReady();
} }
@@ -618,9 +515,7 @@ export class MemoryIndexManager {
} }
async close(): Promise<void> { async close(): Promise<void> {
if (this.closed) { if (this.closed) return;
return;
}
this.closed = true; this.closed = true;
if (this.watchTimer) { if (this.watchTimer) {
clearTimeout(this.watchTimer); clearTimeout(this.watchTimer);
@@ -647,9 +542,7 @@ export class MemoryIndexManager {
} }
private async ensureVectorReady(dimensions?: number): Promise<boolean> { private async ensureVectorReady(dimensions?: number): Promise<boolean> {
if (!this.vector.enabled) { if (!this.vector.enabled) return false;
return false;
}
if (!this.vectorReady) { if (!this.vectorReady) {
this.vectorReady = this.withTimeout( this.vectorReady = this.withTimeout(
this.loadVectorExtension(), this.loadVectorExtension(),
@@ -675,9 +568,7 @@ export class MemoryIndexManager {
} }
private async loadVectorExtension(): Promise<boolean> { private async loadVectorExtension(): Promise<boolean> {
if (this.vector.available !== null) { if (this.vector.available !== null) return this.vector.available;
return this.vector.available;
}
if (!this.vector.enabled) { if (!this.vector.enabled) {
this.vector.available = false; this.vector.available = false;
return false; return false;
@@ -687,9 +578,7 @@ export class MemoryIndexManager {
? resolveUserPath(this.vector.extensionPath) ? resolveUserPath(this.vector.extensionPath)
: undefined; : undefined;
const loaded = await loadSqliteVecExtension({ db: this.db, extensionPath: resolvedPath }); const loaded = await loadSqliteVecExtension({ db: this.db, extensionPath: resolvedPath });
if (!loaded.ok) { if (!loaded.ok) throw new Error(loaded.error ?? "unknown sqlite-vec load error");
throw new Error(loaded.error ?? "unknown sqlite-vec load error");
}
this.vector.extensionPath = loaded.extensionPath; this.vector.extensionPath = loaded.extensionPath;
this.vector.available = true; this.vector.available = true;
return true; return true;
@@ -703,9 +592,7 @@ export class MemoryIndexManager {
} }
private ensureVectorTable(dimensions: number): void { private ensureVectorTable(dimensions: number): void {
if (this.vector.dims === dimensions) { if (this.vector.dims === dimensions) return;
return;
}
if (this.vector.dims && this.vector.dims !== dimensions) { if (this.vector.dims && this.vector.dims !== dimensions) {
this.dropVectorTable(); this.dropVectorTable();
} }
@@ -729,9 +616,7 @@ export class MemoryIndexManager {
private buildSourceFilter(alias?: string): { sql: string; params: MemorySource[] } { private buildSourceFilter(alias?: string): { sql: string; params: MemorySource[] } {
const sources = Array.from(this.sources); const sources = Array.from(this.sources);
if (sources.length === 0) { if (sources.length === 0) return { sql: "", params: [] };
return { sql: "", params: [] };
}
const column = alias ? `${alias}.source` : "source"; const column = alias ? `${alias}.source` : "source";
const placeholders = sources.map(() => "?").join(", "); const placeholders = sources.map(() => "?").join(", ");
return { sql: ` AND ${column} IN (${placeholders})`, params: sources }; return { sql: ` AND ${column} IN (${placeholders})`, params: sources };
@@ -750,9 +635,7 @@ export class MemoryIndexManager {
} }
private seedEmbeddingCache(sourceDb: DatabaseSync): void { private seedEmbeddingCache(sourceDb: DatabaseSync): void {
if (!this.cache.enabled) { if (!this.cache.enabled) return;
return;
}
try { try {
const rows = sourceDb const rows = sourceDb
.prepare( .prepare(
@@ -767,9 +650,7 @@ export class MemoryIndexManager {
dims: number | null; dims: number | null;
updated_at: number; updated_at: number;
}>; }>;
if (!rows.length) { if (!rows.length) return;
return;
}
const insert = this.db.prepare( const insert = this.db.prepare(
`INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at) `INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?)
@@ -846,26 +727,12 @@ export class MemoryIndexManager {
} }
private ensureWatcher() { private ensureWatcher() {
if (!this.sources.has("memory") || !this.settings.sync.watch || this.watcher) { if (!this.sources.has("memory") || !this.settings.sync.watch || this.watcher) return;
return; const watchPaths = [
}
const additionalPaths = normalizeExtraMemoryPaths(this.workspaceDir, this.settings.extraPaths)
.map((entry) => {
try {
const stat = fsSync.lstatSync(entry);
return stat.isSymbolicLink() ? null : entry;
} catch {
return null;
}
})
.filter((entry): entry is string => Boolean(entry));
const watchPaths = new Set<string>([
path.join(this.workspaceDir, "MEMORY.md"), path.join(this.workspaceDir, "MEMORY.md"),
path.join(this.workspaceDir, "memory.md"),
path.join(this.workspaceDir, "memory"), path.join(this.workspaceDir, "memory"),
...additionalPaths, ];
]); this.watcher = chokidar.watch(watchPaths, {
this.watcher = chokidar.watch(Array.from(watchPaths), {
ignoreInitial: true, ignoreInitial: true,
awaitWriteFinish: { awaitWriteFinish: {
stabilityThreshold: this.settings.sync.watchDebounceMs, stabilityThreshold: this.settings.sync.watchDebounceMs,
@@ -882,26 +749,18 @@ export class MemoryIndexManager {
} }
private ensureSessionListener() { private ensureSessionListener() {
if (!this.sources.has("sessions") || this.sessionUnsubscribe) { if (!this.sources.has("sessions") || this.sessionUnsubscribe) return;
return;
}
this.sessionUnsubscribe = onSessionTranscriptUpdate((update) => { this.sessionUnsubscribe = onSessionTranscriptUpdate((update) => {
if (this.closed) { if (this.closed) return;
return;
}
const sessionFile = update.sessionFile; const sessionFile = update.sessionFile;
if (!this.isSessionFileForAgent(sessionFile)) { if (!this.isSessionFileForAgent(sessionFile)) return;
return;
}
this.scheduleSessionDirty(sessionFile); this.scheduleSessionDirty(sessionFile);
}); });
} }
private scheduleSessionDirty(sessionFile: string) { private scheduleSessionDirty(sessionFile: string) {
this.sessionPendingFiles.add(sessionFile); this.sessionPendingFiles.add(sessionFile);
if (this.sessionWatchTimer) { if (this.sessionWatchTimer) return;
return;
}
this.sessionWatchTimer = setTimeout(() => { this.sessionWatchTimer = setTimeout(() => {
this.sessionWatchTimer = null; this.sessionWatchTimer = null;
void this.processSessionDeltaBatch().catch((err) => { void this.processSessionDeltaBatch().catch((err) => {
@@ -911,17 +770,13 @@ export class MemoryIndexManager {
} }
private async processSessionDeltaBatch(): Promise<void> { private async processSessionDeltaBatch(): Promise<void> {
if (this.sessionPendingFiles.size === 0) { if (this.sessionPendingFiles.size === 0) return;
return;
}
const pending = Array.from(this.sessionPendingFiles); const pending = Array.from(this.sessionPendingFiles);
this.sessionPendingFiles.clear(); this.sessionPendingFiles.clear();
let shouldSync = false; let shouldSync = false;
for (const sessionFile of pending) { for (const sessionFile of pending) {
const delta = await this.updateSessionDelta(sessionFile); const delta = await this.updateSessionDelta(sessionFile);
if (!delta) { if (!delta) continue;
continue;
}
const bytesThreshold = delta.deltaBytes; const bytesThreshold = delta.deltaBytes;
const messagesThreshold = delta.deltaMessages; const messagesThreshold = delta.deltaMessages;
const bytesHit = const bytesHit =
@@ -930,9 +785,7 @@ export class MemoryIndexManager {
messagesThreshold <= 0 messagesThreshold <= 0
? delta.pendingMessages > 0 ? delta.pendingMessages > 0
: delta.pendingMessages >= messagesThreshold; : delta.pendingMessages >= messagesThreshold;
if (!bytesHit && !messagesHit) { if (!bytesHit && !messagesHit) continue;
continue;
}
this.sessionsDirtyFiles.add(sessionFile); this.sessionsDirtyFiles.add(sessionFile);
this.sessionsDirty = true; this.sessionsDirty = true;
delta.pendingBytes = delta.pendingBytes =
@@ -955,9 +808,7 @@ export class MemoryIndexManager {
pendingMessages: number; pendingMessages: number;
} | null> { } | null> {
const thresholds = this.settings.sync.sessions; const thresholds = this.settings.sync.sessions;
if (!thresholds) { if (!thresholds) return null;
return null;
}
let stat: { size: number }; let stat: { size: number };
try { try {
stat = await fs.stat(sessionFile); stat = await fs.stat(sessionFile);
@@ -1008,9 +859,7 @@ export class MemoryIndexManager {
} }
private async countNewlines(absPath: string, start: number, end: number): Promise<number> { private async countNewlines(absPath: string, start: number, end: number): Promise<number> {
if (end <= start) { if (end <= start) return 0;
return 0;
}
const handle = await fs.open(absPath, "r"); const handle = await fs.open(absPath, "r");
try { try {
let offset = start; let offset = start;
@@ -1019,13 +868,9 @@ export class MemoryIndexManager {
while (offset < end) { while (offset < end) {
const toRead = Math.min(buffer.length, end - offset); const toRead = Math.min(buffer.length, end - offset);
const { bytesRead } = await handle.read(buffer, 0, toRead, offset); const { bytesRead } = await handle.read(buffer, 0, toRead, offset);
if (bytesRead <= 0) { if (bytesRead <= 0) break;
break;
}
for (let i = 0; i < bytesRead; i += 1) { for (let i = 0; i < bytesRead; i += 1) {
if (buffer[i] === 10) { if (buffer[i] === 10) count += 1;
count += 1;
}
} }
offset += bytesRead; offset += bytesRead;
} }
@@ -1037,18 +882,14 @@ export class MemoryIndexManager {
private resetSessionDelta(absPath: string, size: number): void { private resetSessionDelta(absPath: string, size: number): void {
const state = this.sessionDeltas.get(absPath); const state = this.sessionDeltas.get(absPath);
if (!state) { if (!state) return;
return;
}
state.lastSize = size; state.lastSize = size;
state.pendingBytes = 0; state.pendingBytes = 0;
state.pendingMessages = 0; state.pendingMessages = 0;
} }
private isSessionFileForAgent(sessionFile: string): boolean { private isSessionFileForAgent(sessionFile: string): boolean {
if (!sessionFile) { if (!sessionFile) return false;
return false;
}
const sessionsDir = resolveSessionTranscriptsDirForAgent(this.agentId); const sessionsDir = resolveSessionTranscriptsDirForAgent(this.agentId);
const resolvedFile = path.resolve(sessionFile); const resolvedFile = path.resolve(sessionFile);
const resolvedDir = path.resolve(sessionsDir); const resolvedDir = path.resolve(sessionsDir);
@@ -1057,9 +898,7 @@ export class MemoryIndexManager {
private ensureIntervalSync() { private ensureIntervalSync() {
const minutes = this.settings.sync.intervalMinutes; const minutes = this.settings.sync.intervalMinutes;
if (!minutes || minutes <= 0 || this.intervalTimer) { if (!minutes || minutes <= 0 || this.intervalTimer) return;
return;
}
const ms = minutes * 60 * 1000; const ms = minutes * 60 * 1000;
this.intervalTimer = setInterval(() => { this.intervalTimer = setInterval(() => {
void this.sync({ reason: "interval" }).catch((err) => { void this.sync({ reason: "interval" }).catch((err) => {
@@ -1069,12 +908,8 @@ export class MemoryIndexManager {
} }
private scheduleWatchSync() { private scheduleWatchSync() {
if (!this.sources.has("memory") || !this.settings.sync.watch) { if (!this.sources.has("memory") || !this.settings.sync.watch) return;
return; if (this.watchTimer) clearTimeout(this.watchTimer);
}
if (this.watchTimer) {
clearTimeout(this.watchTimer);
}
this.watchTimer = setTimeout(() => { this.watchTimer = setTimeout(() => {
this.watchTimer = null; this.watchTimer = null;
void this.sync({ reason: "watch" }).catch((err) => { void this.sync({ reason: "watch" }).catch((err) => {
@@ -1087,19 +922,11 @@ export class MemoryIndexManager {
params?: { reason?: string; force?: boolean }, params?: { reason?: string; force?: boolean },
needsFullReindex = false, needsFullReindex = false,
) { ) {
if (!this.sources.has("sessions")) { if (!this.sources.has("sessions")) return false;
return false; if (params?.force) return true;
}
if (params?.force) {
return true;
}
const reason = params?.reason; const reason = params?.reason;
if (reason === "session-start" || reason === "watch") { if (reason === "session-start" || reason === "watch") return false;
return false; if (needsFullReindex) return true;
}
if (needsFullReindex) {
return true;
}
return this.sessionsDirty && this.sessionsDirtyFiles.size > 0; return this.sessionsDirty && this.sessionsDirtyFiles.size > 0;
} }
@@ -1107,7 +934,7 @@ export class MemoryIndexManager {
needsFullReindex: boolean; needsFullReindex: boolean;
progress?: MemorySyncProgressState; progress?: MemorySyncProgressState;
}) { }) {
const files = await listMemoryFiles(this.workspaceDir, this.settings.extraPaths); const files = await listMemoryFiles(this.workspaceDir);
const fileEntries = await Promise.all( const fileEntries = await Promise.all(
files.map(async (file) => buildFileEntry(file, this.workspaceDir)), files.map(async (file) => buildFileEntry(file, this.workspaceDir)),
); );
@@ -1156,9 +983,7 @@ export class MemoryIndexManager {
.prepare(`SELECT path FROM files WHERE source = ?`) .prepare(`SELECT path FROM files WHERE source = ?`)
.all("memory") as Array<{ path: string }>; .all("memory") as Array<{ path: string }>;
for (const stale of staleRows) { for (const stale of staleRows) {
if (activePaths.has(stale.path)) { if (activePaths.has(stale.path)) continue;
continue;
}
this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "memory"); this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "memory");
try { try {
this.db this.db
@@ -1253,9 +1078,7 @@ export class MemoryIndexManager {
.prepare(`SELECT path FROM files WHERE source = ?`) .prepare(`SELECT path FROM files WHERE source = ?`)
.all("sessions") as Array<{ path: string }>; .all("sessions") as Array<{ path: string }>;
for (const stale of staleRows) { for (const stale of staleRows) {
if (activePaths.has(stale.path)) { if (activePaths.has(stale.path)) continue;
continue;
}
this.db this.db
.prepare(`DELETE FROM files WHERE path = ? AND source = ?`) .prepare(`DELETE FROM files WHERE path = ? AND source = ?`)
.run(stale.path, "sessions"); .run(stale.path, "sessions");
@@ -1287,9 +1110,7 @@ export class MemoryIndexManager {
total: 0, total: 0,
label: undefined, label: undefined,
report: (update) => { report: (update) => {
if (update.label) { if (update.label) state.label = update.label;
state.label = update.label;
}
const label = const label =
update.total > 0 && state.label update.total > 0 && state.label
? `${state.label} ${update.completed}/${update.total}` ? `${state.label} ${update.completed}/${update.total}`
@@ -1400,12 +1221,8 @@ export class MemoryIndexManager {
private async activateFallbackProvider(reason: string): Promise<boolean> { private async activateFallbackProvider(reason: string): Promise<boolean> {
const fallback = this.settings.fallback; const fallback = this.settings.fallback;
if (!fallback || fallback === "none" || fallback === this.provider.id) { if (!fallback || fallback === "none" || fallback === this.provider.id) return false;
return false; if (this.fallbackFrom) return false;
}
if (this.fallbackFrom) {
return false;
}
const fallbackFrom = this.provider.id as "openai" | "gemini" | "local"; const fallbackFrom = this.provider.id as "openai" | "gemini" | "local";
const fallbackModel = const fallbackModel =
@@ -1557,9 +1374,7 @@ export class MemoryIndexManager {
const row = this.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(META_KEY) as const row = this.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(META_KEY) as
| { value: string } | { value: string }
| undefined; | undefined;
if (!row?.value) { if (!row?.value) return null;
return null;
}
try { try {
return JSON.parse(row.value) as MemoryIndexMeta; return JSON.parse(row.value) as MemoryIndexMeta;
} catch { } catch {
@@ -1606,26 +1421,16 @@ export class MemoryIndexManager {
const normalized = this.normalizeSessionText(content); const normalized = this.normalizeSessionText(content);
return normalized ? normalized : null; return normalized ? normalized : null;
} }
if (!Array.isArray(content)) { if (!Array.isArray(content)) return null;
return null;
}
const parts: string[] = []; const parts: string[] = [];
for (const block of content) { for (const block of content) {
if (!block || typeof block !== "object") { if (!block || typeof block !== "object") continue;
continue;
}
const record = block as { type?: unknown; text?: unknown }; const record = block as { type?: unknown; text?: unknown };
if (record.type !== "text" || typeof record.text !== "string") { if (record.type !== "text" || typeof record.text !== "string") continue;
continue;
}
const normalized = this.normalizeSessionText(record.text); const normalized = this.normalizeSessionText(record.text);
if (normalized) { if (normalized) parts.push(normalized);
parts.push(normalized);
}
}
if (parts.length === 0) {
return null;
} }
if (parts.length === 0) return null;
return parts.join(" "); return parts.join(" ");
} }
@@ -1636,9 +1441,7 @@ export class MemoryIndexManager {
const lines = raw.split("\n"); const lines = raw.split("\n");
const collected: string[] = []; const collected: string[] = [];
for (const line of lines) { for (const line of lines) {
if (!line.trim()) { if (!line.trim()) continue;
continue;
}
let record: unknown; let record: unknown;
try { try {
record = JSON.parse(line); record = JSON.parse(line);
@@ -1655,16 +1458,10 @@ export class MemoryIndexManager {
const message = (record as { message?: unknown }).message as const message = (record as { message?: unknown }).message as
| { role?: unknown; content?: unknown } | { role?: unknown; content?: unknown }
| undefined; | undefined;
if (!message || typeof message.role !== "string") { if (!message || typeof message.role !== "string") continue;
continue; if (message.role !== "user" && message.role !== "assistant") continue;
}
if (message.role !== "user" && message.role !== "assistant") {
continue;
}
const text = this.extractSessionText(message.content); const text = this.extractSessionText(message.content);
if (!text) { if (!text) continue;
continue;
}
const label = message.role === "user" ? "User" : "Assistant"; const label = message.role === "user" ? "User" : "Assistant";
collected.push(`${label}: ${text}`); collected.push(`${label}: ${text}`);
} }
@@ -1684,9 +1481,7 @@ export class MemoryIndexManager {
} }
private estimateEmbeddingTokens(text: string): number { private estimateEmbeddingTokens(text: string): number {
if (!text) { if (!text) return 0;
return 0;
}
return Math.ceil(text.length / EMBEDDING_APPROX_CHARS_PER_TOKEN); return Math.ceil(text.length / EMBEDDING_APPROX_CHARS_PER_TOKEN);
} }
@@ -1719,27 +1514,17 @@ export class MemoryIndexManager {
} }
private loadEmbeddingCache(hashes: string[]): Map<string, number[]> { private loadEmbeddingCache(hashes: string[]): Map<string, number[]> {
if (!this.cache.enabled) { if (!this.cache.enabled) return new Map();
return new Map(); if (hashes.length === 0) return new Map();
}
if (hashes.length === 0) {
return new Map();
}
const unique: string[] = []; const unique: string[] = [];
const seen = new Set<string>(); const seen = new Set<string>();
for (const hash of hashes) { for (const hash of hashes) {
if (!hash) { if (!hash) continue;
continue; if (seen.has(hash)) continue;
}
if (seen.has(hash)) {
continue;
}
seen.add(hash); seen.add(hash);
unique.push(hash); unique.push(hash);
} }
if (unique.length === 0) { if (unique.length === 0) return new Map();
return new Map();
}
const out = new Map<string, number[]>(); const out = new Map<string, number[]>();
const baseParams = [this.provider.id, this.provider.model, this.providerKey]; const baseParams = [this.provider.id, this.provider.model, this.providerKey];
@@ -1761,12 +1546,8 @@ export class MemoryIndexManager {
} }
private upsertEmbeddingCache(entries: Array<{ hash: string; embedding: number[] }>): void { private upsertEmbeddingCache(entries: Array<{ hash: string; embedding: number[] }>): void {
if (!this.cache.enabled) { if (!this.cache.enabled) return;
return; if (entries.length === 0) return;
}
if (entries.length === 0) {
return;
}
const now = Date.now(); const now = Date.now();
const stmt = this.db.prepare( const stmt = this.db.prepare(
`INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at)\n` + `INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at)\n` +
@@ -1791,20 +1572,14 @@ export class MemoryIndexManager {
} }
private pruneEmbeddingCacheIfNeeded(): void { private pruneEmbeddingCacheIfNeeded(): void {
if (!this.cache.enabled) { if (!this.cache.enabled) return;
return;
}
const max = this.cache.maxEntries; const max = this.cache.maxEntries;
if (!max || max <= 0) { if (!max || max <= 0) return;
return;
}
const row = this.db.prepare(`SELECT COUNT(*) as c FROM ${EMBEDDING_CACHE_TABLE}`).get() as const row = this.db.prepare(`SELECT COUNT(*) as c FROM ${EMBEDDING_CACHE_TABLE}`).get() as
| { c: number } | { c: number }
| undefined; | undefined;
const count = row?.c ?? 0; const count = row?.c ?? 0;
if (count <= max) { if (count <= max) return;
return;
}
const excess = count - max; const excess = count - max;
this.db this.db
.prepare( .prepare(
@@ -1819,9 +1594,7 @@ export class MemoryIndexManager {
} }
private async embedChunksInBatches(chunks: MemoryChunk[]): Promise<number[][]> { private async embedChunksInBatches(chunks: MemoryChunk[]): Promise<number[][]> {
if (chunks.length === 0) { if (chunks.length === 0) return [];
return [];
}
const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash)); const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash));
const embeddings: number[][] = Array.from({ length: chunks.length }, () => []); const embeddings: number[][] = Array.from({ length: chunks.length }, () => []);
const missing: Array<{ index: number; chunk: MemoryChunk }> = []; const missing: Array<{ index: number; chunk: MemoryChunk }> = [];
@@ -1836,9 +1609,7 @@ export class MemoryIndexManager {
} }
} }
if (missing.length === 0) { if (missing.length === 0) return embeddings;
return embeddings;
}
const missingChunks = missing.map((m) => m.chunk); const missingChunks = missing.map((m) => m.chunk);
const batches = this.buildEmbeddingBatches(missingChunks); const batches = this.buildEmbeddingBatches(missingChunks);
@@ -1864,7 +1635,7 @@ export class MemoryIndexManager {
if (this.provider.id === "openai" && this.openAi) { if (this.provider.id === "openai" && this.openAi) {
const entries = Object.entries(this.openAi.headers) const entries = Object.entries(this.openAi.headers)
.filter(([key]) => key.toLowerCase() !== "authorization") .filter(([key]) => key.toLowerCase() !== "authorization")
.toSorted(([a], [b]) => a.localeCompare(b)) .sort(([a], [b]) => a.localeCompare(b))
.map(([key, value]) => [key, value]); .map(([key, value]) => [key, value]);
return hashText( return hashText(
JSON.stringify({ JSON.stringify({
@@ -1881,7 +1652,7 @@ export class MemoryIndexManager {
const lower = key.toLowerCase(); const lower = key.toLowerCase();
return lower !== "authorization" && lower !== "x-goog-api-key"; return lower !== "authorization" && lower !== "x-goog-api-key";
}) })
.toSorted(([a], [b]) => a.localeCompare(b)) .sort(([a], [b]) => a.localeCompare(b))
.map(([key, value]) => [key, value]); .map(([key, value]) => [key, value]);
return hashText( return hashText(
JSON.stringify({ JSON.stringify({
@@ -1918,9 +1689,7 @@ export class MemoryIndexManager {
if (!openAi) { if (!openAi) {
return this.embedChunksInBatches(chunks); return this.embedChunksInBatches(chunks);
} }
if (chunks.length === 0) { if (chunks.length === 0) return [];
return [];
}
const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash)); const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash));
const embeddings: number[][] = Array.from({ length: chunks.length }, () => []); const embeddings: number[][] = Array.from({ length: chunks.length }, () => []);
const missing: Array<{ index: number; chunk: MemoryChunk }> = []; const missing: Array<{ index: number; chunk: MemoryChunk }> = [];
@@ -1935,9 +1704,7 @@ export class MemoryIndexManager {
} }
} }
if (missing.length === 0) { if (missing.length === 0) return embeddings;
return embeddings;
}
const requests: OpenAiBatchRequest[] = []; const requests: OpenAiBatchRequest[] = [];
const mapping = new Map<string, { index: number; hash: string }>(); const mapping = new Map<string, { index: number; hash: string }>();
@@ -1972,17 +1739,13 @@ export class MemoryIndexManager {
}), }),
fallback: async () => await this.embedChunksInBatches(chunks), fallback: async () => await this.embedChunksInBatches(chunks),
}); });
if (Array.isArray(batchResult)) { if (Array.isArray(batchResult)) return batchResult;
return batchResult;
}
const byCustomId = batchResult; const byCustomId = batchResult;
const toCache: Array<{ hash: string; embedding: number[] }> = []; const toCache: Array<{ hash: string; embedding: number[] }> = [];
for (const [customId, embedding] of byCustomId.entries()) { for (const [customId, embedding] of byCustomId.entries()) {
const mapped = mapping.get(customId); const mapped = mapping.get(customId);
if (!mapped) { if (!mapped) continue;
continue;
}
embeddings[mapped.index] = embedding; embeddings[mapped.index] = embedding;
toCache.push({ hash: mapped.hash, embedding }); toCache.push({ hash: mapped.hash, embedding });
} }
@@ -1999,9 +1762,7 @@ export class MemoryIndexManager {
if (!gemini) { if (!gemini) {
return this.embedChunksInBatches(chunks); return this.embedChunksInBatches(chunks);
} }
if (chunks.length === 0) { if (chunks.length === 0) return [];
return [];
}
const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash)); const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash));
const embeddings: number[][] = Array.from({ length: chunks.length }, () => []); const embeddings: number[][] = Array.from({ length: chunks.length }, () => []);
const missing: Array<{ index: number; chunk: MemoryChunk }> = []; const missing: Array<{ index: number; chunk: MemoryChunk }> = [];
@@ -2016,9 +1777,7 @@ export class MemoryIndexManager {
} }
} }
if (missing.length === 0) { if (missing.length === 0) return embeddings;
return embeddings;
}
const requests: GeminiBatchRequest[] = []; const requests: GeminiBatchRequest[] = [];
const mapping = new Map<string, { index: number; hash: string }>(); const mapping = new Map<string, { index: number; hash: string }>();
@@ -2050,17 +1809,13 @@ export class MemoryIndexManager {
}), }),
fallback: async () => await this.embedChunksInBatches(chunks), fallback: async () => await this.embedChunksInBatches(chunks),
}); });
if (Array.isArray(batchResult)) { if (Array.isArray(batchResult)) return batchResult;
return batchResult;
}
const byCustomId = batchResult; const byCustomId = batchResult;
const toCache: Array<{ hash: string; embedding: number[] }> = []; const toCache: Array<{ hash: string; embedding: number[] }> = [];
for (const [customId, embedding] of byCustomId.entries()) { for (const [customId, embedding] of byCustomId.entries()) {
const mapped = mapping.get(customId); const mapped = mapping.get(customId);
if (!mapped) { if (!mapped) continue;
continue;
}
embeddings[mapped.index] = embedding; embeddings[mapped.index] = embedding;
toCache.push({ hash: mapped.hash, embedding }); toCache.push({ hash: mapped.hash, embedding });
} }
@@ -2069,9 +1824,7 @@ export class MemoryIndexManager {
} }
private async embedBatchWithRetry(texts: string[]): Promise<number[][]> { private async embedBatchWithRetry(texts: string[]): Promise<number[][]> {
if (texts.length === 0) { if (texts.length === 0) return [];
return [];
}
let attempt = 0; let attempt = 0;
let delayMs = EMBEDDING_RETRY_BASE_DELAY_MS; let delayMs = EMBEDDING_RETRY_BASE_DELAY_MS;
while (true) { while (true) {
@@ -2133,9 +1886,7 @@ export class MemoryIndexManager {
timeoutMs: number, timeoutMs: number,
message: string, message: string,
): Promise<T> { ): Promise<T> {
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return await promise;
return await promise;
}
let timer: NodeJS.Timeout | null = null; let timer: NodeJS.Timeout | null = null;
const timeoutPromise = new Promise<never>((_, reject) => { const timeoutPromise = new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new Error(message)), timeoutMs); timer = setTimeout(() => reject(new Error(message)), timeoutMs);
@@ -2143,16 +1894,12 @@ export class MemoryIndexManager {
try { try {
return (await Promise.race([promise, timeoutPromise])) as T; return (await Promise.race([promise, timeoutPromise])) as T;
} finally { } finally {
if (timer) { if (timer) clearTimeout(timer);
clearTimeout(timer);
}
} }
} }
private async runWithConcurrency<T>(tasks: Array<() => Promise<T>>, limit: number): Promise<T[]> { private async runWithConcurrency<T>(tasks: Array<() => Promise<T>>, limit: number): Promise<T[]> {
if (tasks.length === 0) { if (tasks.length === 0) return [];
return [];
}
const resolvedLimit = Math.max(1, Math.min(limit, tasks.length)); const resolvedLimit = Math.max(1, Math.min(limit, tasks.length));
const results: T[] = Array.from({ length: tasks.length }); const results: T[] = Array.from({ length: tasks.length });
let next = 0; let next = 0;
@@ -2160,14 +1907,10 @@ export class MemoryIndexManager {
const workers = Array.from({ length: resolvedLimit }, async () => { const workers = Array.from({ length: resolvedLimit }, async () => {
while (true) { while (true) {
if (firstError) { if (firstError) return;
return;
}
const index = next; const index = next;
next += 1; next += 1;
if (index >= tasks.length) { if (index >= tasks.length) return;
return;
}
try { try {
results[index] = await tasks[index](); results[index] = await tasks[index]();
} catch (err) { } catch (err) {
@@ -2178,9 +1921,7 @@ export class MemoryIndexManager {
}); });
await Promise.allSettled(workers); await Promise.allSettled(workers);
if (firstError) { if (firstError) throw firstError;
throw firstError;
}
return results; return results;
} }

612
src/memory/qmd-manager.ts Normal file
View File

@@ -0,0 +1,612 @@
import { spawn } from "node:child_process";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import YAML from "yaml";
import type { MoltbotConfig } from "../config/config.js";
import { resolveStateDir } from "../config/paths.js";
import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import {
listSessionFilesForAgent,
buildSessionEntry,
type SessionFileEntry,
} from "./session-files.js";
import { requireNodeSqlite } from "./sqlite.js";
import type {
MemoryProviderStatus,
MemorySearchManager,
MemorySearchResult,
MemorySource,
MemorySyncProgressUpdate,
} from "./types.js";
import type { ResolvedMemoryBackendConfig, ResolvedQmdConfig } from "./backend-config.js";
const log = createSubsystemLogger("memory");
const SNIPPET_HEADER_RE = /@@\s*-([0-9]+),([0-9]+)/;
type QmdQueryResult = {
docid?: string;
score?: number;
file?: string;
snippet?: string;
body?: string;
};
type CollectionRoot = {
path: string;
kind: MemorySource;
};
type SessionExporterConfig = {
dir: string;
retentionMs?: number;
collectionName: string;
};
export class QmdMemoryManager implements MemorySearchManager {
static async create(params: {
cfg: MoltbotConfig;
agentId: string;
resolved: ResolvedMemoryBackendConfig;
}): Promise<QmdMemoryManager | null> {
const resolved = params.resolved.qmd;
if (!resolved) return null;
const manager = new QmdMemoryManager({ cfg: params.cfg, agentId: params.agentId, resolved });
await manager.initialize();
return manager;
}
private readonly cfg: MoltbotConfig;
private readonly agentId: string;
private readonly qmd: ResolvedQmdConfig;
private readonly workspaceDir: string;
private readonly stateDir: string;
private readonly agentStateDir: string;
private readonly qmdDir: string;
private readonly cacheDir: string;
private readonly configDir: string;
private readonly xdgConfigHome: string;
private readonly xdgCacheHome: string;
private readonly collectionsFile: string;
private readonly indexPath: string;
private readonly env: NodeJS.ProcessEnv;
private readonly collectionRoots = new Map<string, CollectionRoot>();
private readonly sources = new Set<MemorySource>();
private readonly docPathCache = new Map<
string,
{ rel: string; abs: string; source: MemorySource }
>();
private readonly sessionExporter: SessionExporterConfig | null;
private updateTimer: NodeJS.Timeout | null = null;
private pendingUpdate: Promise<void> | null = null;
private closed = false;
private db: import("node:sqlite").DatabaseSync | null = null;
private lastUpdateAt: number | null = null;
private constructor(params: {
cfg: MoltbotConfig;
agentId: string;
resolved: ResolvedQmdConfig;
}) {
this.cfg = params.cfg;
this.agentId = params.agentId;
this.qmd = params.resolved;
this.workspaceDir = resolveAgentWorkspaceDir(params.cfg, params.agentId);
this.stateDir = resolveStateDir(process.env, os.homedir);
this.agentStateDir = path.join(this.stateDir, "agents", this.agentId);
this.qmdDir = path.join(this.agentStateDir, "qmd");
this.cacheDir = path.join(this.qmdDir, "cache");
this.configDir = path.join(this.qmdDir, "config");
this.xdgConfigHome = path.join(this.qmdDir, "xdg-config");
this.xdgCacheHome = path.join(this.qmdDir, "xdg-cache");
this.collectionsFile = path.join(this.configDir, "index.yml");
this.indexPath = path.join(this.cacheDir, "index.sqlite");
this.env = {
...process.env,
QMD_CONFIG_DIR: this.configDir,
XDG_CONFIG_HOME: this.xdgConfigHome,
XDG_CACHE_HOME: this.xdgCacheHome,
INDEX_PATH: this.indexPath,
NO_COLOR: "1",
};
this.sessionExporter = this.qmd.sessions.enabled
? {
dir: this.qmd.sessions.exportDir ?? path.join(this.qmdDir, "sessions"),
retentionMs: this.qmd.sessions.retentionDays
? this.qmd.sessions.retentionDays * 24 * 60 * 60 * 1000
: undefined,
collectionName: this.pickSessionCollectionName(),
}
: null;
if (this.sessionExporter) {
this.qmd.collections = [
...this.qmd.collections,
{
name: this.sessionExporter.collectionName,
path: this.sessionExporter.dir,
pattern: "**/*.md",
kind: "sessions",
},
];
}
}
private async initialize(): Promise<void> {
await fs.mkdir(this.cacheDir, { recursive: true });
await fs.mkdir(this.configDir, { recursive: true });
await fs.mkdir(this.xdgConfigHome, { recursive: true });
await fs.mkdir(this.xdgCacheHome, { recursive: true });
this.bootstrapCollections();
await this.writeCollectionsConfig();
if (this.qmd.update.onBoot) {
await this.runUpdate("boot", true);
}
if (this.qmd.update.intervalMs > 0) {
this.updateTimer = setInterval(() => {
void this.runUpdate("interval").catch((err) => {
log.warn(`qmd update failed (${String(err)})`);
});
}, this.qmd.update.intervalMs);
}
}
private bootstrapCollections(): void {
this.collectionRoots.clear();
this.sources.clear();
for (const collection of this.qmd.collections) {
const kind: MemorySource = collection.kind === "sessions" ? "sessions" : "memory";
this.collectionRoots.set(collection.name, { path: collection.path, kind });
this.sources.add(kind);
}
}
private async writeCollectionsConfig(): Promise<void> {
const collections: Record<string, { path: string; pattern: string }> = {};
for (const collection of this.qmd.collections) {
collections[collection.name] = {
path: collection.path,
pattern: collection.pattern,
};
}
const yaml = YAML.stringify({ collections }, { indent: 2, lineWidth: 0 });
await fs.writeFile(this.collectionsFile, yaml, "utf-8");
}
async search(
query: string,
opts?: { maxResults?: number; minScore?: number; sessionKey?: string },
): Promise<MemorySearchResult[]> {
if (!this.isScopeAllowed(opts?.sessionKey)) return [];
const trimmed = query.trim();
if (!trimmed) return [];
await this.pendingUpdate?.catch(() => undefined);
const limit = Math.min(
this.qmd.limits.maxResults,
opts?.maxResults ?? this.qmd.limits.maxResults,
);
const args = ["query", trimmed, "--json", "-n", String(limit)];
let stdout: string;
try {
const result = await this.runQmd(args, { timeoutMs: this.qmd.limits.timeoutMs });
stdout = result.stdout;
} catch (err) {
log.warn(`qmd query failed: ${String(err)}`);
throw err instanceof Error ? err : new Error(String(err));
}
let parsed: QmdQueryResult[] = [];
try {
parsed = JSON.parse(stdout);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
log.warn(`qmd query returned invalid JSON: ${message}`);
throw new Error(`qmd query returned invalid JSON: ${message}`);
}
const results: MemorySearchResult[] = [];
for (const entry of parsed) {
const doc = await this.resolveDocLocation(entry.docid);
if (!doc) continue;
const snippet = entry.snippet?.slice(0, this.qmd.limits.maxSnippetChars) ?? "";
const lines = this.extractSnippetLines(snippet);
const score = typeof entry.score === "number" ? entry.score : 0;
const minScore = opts?.minScore ?? 0;
if (score < minScore) continue;
results.push({
path: doc.rel,
startLine: lines.startLine,
endLine: lines.endLine,
score,
snippet,
source: doc.source,
});
}
return results.slice(0, limit);
}
async sync(params?: {
reason?: string;
force?: boolean;
progress?: (update: MemorySyncProgressUpdate) => void;
}): Promise<void> {
if (params?.progress) {
params.progress({ completed: 0, total: 1, label: "Updating QMD index…" });
}
await this.runUpdate(params?.reason ?? "manual", params?.force);
if (params?.progress) {
params.progress({ completed: 1, total: 1, label: "QMD index updated" });
}
}
async readFile(params: {
relPath: string;
from?: number;
lines?: number;
}): Promise<{ text: string; path: string }> {
const relPath = params.relPath?.trim();
if (!relPath) throw new Error("path required");
const absPath = this.resolveReadPath(relPath);
const content = await fs.readFile(absPath, "utf-8");
if (!params.from && !params.lines) {
return { text: content, path: relPath };
}
const lines = content.split("\n");
const start = Math.max(1, params.from ?? 1);
const count = Math.max(1, params.lines ?? lines.length);
const slice = lines.slice(start - 1, start - 1 + count);
return { text: slice.join("\n"), path: relPath };
}
status(): MemoryProviderStatus {
const counts = this.readCounts();
return {
backend: "qmd",
provider: "qmd",
model: "qmd",
requestedProvider: "qmd",
files: counts.totalDocuments,
chunks: counts.totalDocuments,
dirty: false,
workspaceDir: this.workspaceDir,
dbPath: this.indexPath,
sources: Array.from(this.sources),
sourceCounts: counts.sourceCounts,
vector: { enabled: true, available: true },
batch: {
enabled: false,
failures: 0,
limit: 0,
wait: false,
concurrency: 0,
pollIntervalMs: 0,
timeoutMs: 0,
},
custom: {
qmd: {
collections: this.qmd.collections.length,
lastUpdateAt: this.lastUpdateAt,
},
},
};
}
async probeVectorAvailability(): Promise<boolean> {
return true;
}
async close(): Promise<void> {
if (this.closed) return;
this.closed = true;
if (this.updateTimer) {
clearInterval(this.updateTimer);
this.updateTimer = null;
}
await this.pendingUpdate?.catch(() => undefined);
if (this.db) {
this.db.close();
this.db = null;
}
}
private async runUpdate(reason: string, force?: boolean): Promise<void> {
if (this.pendingUpdate && !force) return this.pendingUpdate;
const run = async () => {
if (this.sessionExporter) {
await this.exportSessions();
}
await this.runQmd(["update"], { timeoutMs: 120_000 });
try {
await this.runQmd(["embed"], { timeoutMs: 120_000 });
} catch (err) {
log.warn(`qmd embed failed (${reason}): ${String(err)}`);
}
this.lastUpdateAt = Date.now();
this.docPathCache.clear();
};
this.pendingUpdate = run().finally(() => {
this.pendingUpdate = null;
});
await this.pendingUpdate;
}
private async runQmd(
args: string[],
opts?: { timeoutMs?: number },
): Promise<{ stdout: string; stderr: string }> {
return await new Promise((resolve, reject) => {
const child = spawn(this.qmd.command, args, {
env: this.env,
cwd: this.workspaceDir,
});
let stdout = "";
let stderr = "";
const timer = opts?.timeoutMs
? setTimeout(() => {
child.kill("SIGKILL");
reject(new Error(`qmd ${args.join(" ")} timed out after ${opts.timeoutMs}ms`));
}, opts.timeoutMs)
: null;
child.stdout.on("data", (data) => {
stdout += data.toString();
});
child.stderr.on("data", (data) => {
stderr += data.toString();
});
child.on("error", (err) => {
if (timer) clearTimeout(timer);
reject(err);
});
child.on("close", (code) => {
if (timer) clearTimeout(timer);
if (code === 0) {
resolve({ stdout, stderr });
} else {
reject(new Error(`qmd ${args.join(" ")} failed (code ${code}): ${stderr || stdout}`));
}
});
});
}
private ensureDb() {
if (this.db) return this.db;
const sqlite = requireNodeSqlite();
this.db = sqlite.open(this.indexPath, { readonly: true });
return this.db;
}
private async exportSessions(): Promise<void> {
if (!this.sessionExporter) return;
const exportDir = this.sessionExporter.dir;
await fs.mkdir(exportDir, { recursive: true });
const files = await listSessionFilesForAgent(this.agentId);
const keep = new Set<string>();
const cutoff = this.sessionExporter.retentionMs
? Date.now() - this.sessionExporter.retentionMs
: null;
for (const sessionFile of files) {
const entry = await buildSessionEntry(sessionFile);
if (!entry) continue;
if (cutoff && entry.mtimeMs < cutoff) continue;
const target = path.join(exportDir, `${path.basename(sessionFile, ".jsonl")}.md`);
await fs.writeFile(target, this.renderSessionMarkdown(entry), "utf-8");
keep.add(target);
}
const exported = await fs.readdir(exportDir).catch(() => []);
for (const name of exported) {
if (!name.endsWith(".md")) continue;
const full = path.join(exportDir, name);
if (!keep.has(full)) {
await fs.rm(full, { force: true });
}
}
}
private renderSessionMarkdown(entry: SessionFileEntry): string {
const header = `# Session ${path.basename(entry.absPath, path.extname(entry.absPath))}`;
const body = entry.content?.trim().length ? entry.content.trim() : "(empty)";
return `${header}\n\n${body}\n`;
}
private pickSessionCollectionName(): string {
const existing = new Set(this.qmd.collections.map((collection) => collection.name));
if (!existing.has("sessions")) return "sessions";
let counter = 2;
let candidate = `sessions-${counter}`;
while (existing.has(candidate)) {
counter += 1;
candidate = `sessions-${counter}`;
}
return candidate;
}
private async resolveDocLocation(
docid?: string,
): Promise<{ rel: string; abs: string; source: MemorySource } | null> {
if (!docid) return null;
const normalized = docid.startsWith("#") ? docid.slice(1) : docid;
if (!normalized) return null;
const cached = this.docPathCache.get(normalized);
if (cached) return cached;
const db = this.ensureDb();
const row = db
.prepare("SELECT collection, path FROM documents WHERE hash LIKE ? AND active = 1 LIMIT 1")
.get(`${normalized}%`) as { collection: string; path: string } | undefined;
if (!row) return null;
const location = this.toDocLocation(row.collection, row.path);
if (!location) return null;
this.docPathCache.set(normalized, location);
return location;
}
private extractSnippetLines(snippet: string): { startLine: number; endLine: number } {
const match = SNIPPET_HEADER_RE.exec(snippet);
if (match) {
const start = Number(match[1]);
const count = Number(match[2]);
if (Number.isFinite(start) && Number.isFinite(count)) {
return { startLine: start, endLine: start + count - 1 };
}
}
const lines = snippet.split("\n").length;
return { startLine: 1, endLine: lines };
}
private readCounts(): {
totalDocuments: number;
sourceCounts: Array<{ source: MemorySource; files: number; chunks: number }>;
} {
try {
const db = this.ensureDb();
const rows = db
.prepare(
"SELECT collection, COUNT(*) as c FROM documents WHERE active = 1 GROUP BY collection",
)
.all() as Array<{ collection: string; c: number }>;
const bySource = new Map<MemorySource, { files: number; chunks: number }>();
for (const source of this.sources) {
bySource.set(source, { files: 0, chunks: 0 });
}
let total = 0;
for (const row of rows) {
const root = this.collectionRoots.get(row.collection);
const source = root?.kind ?? "memory";
const entry = bySource.get(source) ?? { files: 0, chunks: 0 };
entry.files += row.c ?? 0;
entry.chunks += row.c ?? 0;
bySource.set(source, entry);
total += row.c ?? 0;
}
return {
totalDocuments: total,
sourceCounts: Array.from(bySource.entries()).map(([source, value]) => ({
source,
files: value.files,
chunks: value.chunks,
})),
};
} catch (err) {
log.warn(`failed to read qmd index stats: ${String(err)}`);
return {
totalDocuments: 0,
sourceCounts: Array.from(this.sources).map((source) => ({ source, files: 0, chunks: 0 })),
};
}
}
private isScopeAllowed(sessionKey?: string): boolean {
const scope = this.qmd.scope;
if (!scope) return true;
const channel = this.deriveChannelFromKey(sessionKey);
const chatType = this.deriveChatTypeFromKey(sessionKey);
const normalizedKey = sessionKey ?? "";
for (const rule of scope.rules ?? []) {
if (!rule) continue;
const match = rule.match ?? {};
if (match.channel && match.channel !== channel) continue;
if (match.chatType && match.chatType !== chatType) continue;
if (match.keyPrefix && !normalizedKey.startsWith(match.keyPrefix)) continue;
return rule.action === "allow";
}
const fallback = scope.default ?? "allow";
return fallback === "allow";
}
private deriveChannelFromKey(key?: string) {
if (!key) return undefined;
const parts = key.split(":").filter(Boolean);
if (parts.length >= 3 && (parts[1] === "group" || parts[1] === "channel")) {
return parts[0]?.toLowerCase();
}
return undefined;
}
private deriveChatTypeFromKey(key?: string) {
if (!key) return undefined;
if (key.includes(":group:")) return "group";
if (key.includes(":channel:")) return "channel";
return "direct";
}
private toDocLocation(
collection: string,
collectionRelativePath: string,
): { rel: string; abs: string; source: MemorySource } | null {
const root = this.collectionRoots.get(collection);
if (!root) return null;
const normalizedRelative = collectionRelativePath.replace(/\\/g, "/");
const absPath = path.normalize(path.resolve(root.path, collectionRelativePath));
const relativeToWorkspace = path.relative(this.workspaceDir, absPath);
const relPath = this.buildSearchPath(
collection,
normalizedRelative,
relativeToWorkspace,
absPath,
);
return { rel: relPath, abs: absPath, source: root.kind };
}
private buildSearchPath(
collection: string,
collectionRelativePath: string,
relativeToWorkspace: string,
absPath: string,
): string {
const insideWorkspace = this.isInsideWorkspace(relativeToWorkspace);
if (insideWorkspace) {
const normalized = relativeToWorkspace.replace(/\\/g, "/");
if (!normalized) return path.basename(absPath);
return normalized;
}
const sanitized = collectionRelativePath.replace(/^\/+/, "");
return `qmd/${collection}/${sanitized}`;
}
private isInsideWorkspace(relativePath: string): boolean {
if (!relativePath) return true;
if (relativePath.startsWith("..")) return false;
if (relativePath.startsWith(`..${path.sep}`)) return false;
return !path.isAbsolute(relativePath);
}
private resolveReadPath(relPath: string): string {
if (relPath.startsWith("qmd/")) {
const [, collection, ...rest] = relPath.split("/");
if (!collection || rest.length === 0) {
throw new Error("invalid qmd path");
}
const root = this.collectionRoots.get(collection);
if (!root) throw new Error(`unknown qmd collection: ${collection}`);
const joined = rest.join("/");
const resolved = path.resolve(root.path, joined);
if (!this.isWithinRoot(root.path, resolved)) {
throw new Error("qmd path escapes collection");
}
return resolved;
}
const absPath = path.resolve(this.workspaceDir, relPath);
if (!this.isWithinWorkspace(absPath)) {
throw new Error("path escapes workspace");
}
return absPath;
}
private isWithinWorkspace(absPath: string): boolean {
const normalizedWorkspace = this.workspaceDir.endsWith(path.sep)
? this.workspaceDir
: `${this.workspaceDir}${path.sep}`;
if (absPath === this.workspaceDir) return true;
const candidate = absPath.endsWith(path.sep) ? absPath : `${absPath}${path.sep}`;
return candidate.startsWith(normalizedWorkspace);
}
private isWithinRoot(root: string, candidate: string): boolean {
const normalizedRoot = root.endsWith(path.sep) ? root : `${root}${path.sep}`;
if (candidate === root) return true;
const next = candidate.endsWith(path.sep) ? candidate : `${candidate}${path.sep}`;
return next.startsWith(normalizedRoot);
}
}

View File

@@ -0,0 +1,62 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const mockPrimary = {
search: vi.fn(async () => []),
readFile: vi.fn(async () => ({ text: "", path: "MEMORY.md" })),
status: vi.fn(() => ({
backend: "qmd" as const,
provider: "qmd",
model: "qmd",
requestedProvider: "qmd",
files: 0,
chunks: 0,
dirty: false,
workspaceDir: "/tmp",
dbPath: "/tmp/index.sqlite",
sources: ["memory" as const],
sourceCounts: [{ source: "memory" as const, files: 0, chunks: 0 }],
})),
sync: vi.fn(async () => {}),
probeVectorAvailability: vi.fn(async () => true),
close: vi.fn(async () => {}),
};
vi.mock("./qmd-manager.js", () => ({
QmdMemoryManager: {
create: vi.fn(async () => mockPrimary),
},
}));
vi.mock("./manager.js", () => ({
MemoryIndexManager: {
get: vi.fn(async () => null),
},
}));
import { QmdMemoryManager } from "./qmd-manager.js";
import { getMemorySearchManager } from "./search-manager.js";
beforeEach(() => {
mockPrimary.search.mockClear();
mockPrimary.readFile.mockClear();
mockPrimary.status.mockClear();
mockPrimary.sync.mockClear();
mockPrimary.probeVectorAvailability.mockClear();
mockPrimary.close.mockClear();
QmdMemoryManager.create.mockClear();
});
describe("getMemorySearchManager caching", () => {
it("reuses the same QMD manager instance for repeated calls", async () => {
const cfg = {
memory: { backend: "qmd", qmd: {} },
agents: { list: [{ id: "main", default: true, workspace: "/tmp/workspace" }] },
} as const;
const first = await getMemorySearchManager({ cfg, agentId: "main" });
const second = await getMemorySearchManager({ cfg, agentId: "main" });
expect(first.manager).toBe(second.manager);
expect(QmdMemoryManager.create).toHaveBeenCalledTimes(1);
});
});

View File

@@ -1,15 +1,56 @@
import type { OpenClawConfig } from "../config/config.js"; import { createSubsystemLogger } from "../logging/subsystem.js";
import type { MoltbotConfig } from "../config/config.js";
import { resolveMemoryBackendConfig } from "./backend-config.js";
import type { ResolvedQmdConfig } from "./backend-config.js";
import type { MemoryIndexManager } from "./manager.js"; import type { MemoryIndexManager } from "./manager.js";
import type { MemorySearchManager, MemorySyncProgressUpdate } from "./types.js";
const log = createSubsystemLogger("memory");
const QMD_MANAGER_CACHE = new Map<string, MemorySearchManager>();
export type MemorySearchManagerResult = { export type MemorySearchManagerResult = {
manager: MemoryIndexManager | null; manager: MemorySearchManager | null;
error?: string; error?: string;
}; };
export async function getMemorySearchManager(params: { export async function getMemorySearchManager(params: {
cfg: OpenClawConfig; cfg: MoltbotConfig;
agentId: string; agentId: string;
}): Promise<MemorySearchManagerResult> { }): Promise<MemorySearchManagerResult> {
const resolved = resolveMemoryBackendConfig(params);
if (resolved.backend === "qmd" && resolved.qmd) {
const cacheKey = buildQmdCacheKey(params.agentId, resolved.qmd);
const cached = QMD_MANAGER_CACHE.get(cacheKey);
if (cached) {
return { manager: cached };
}
try {
const { QmdMemoryManager } = await import("./qmd-manager.js");
const primary = await QmdMemoryManager.create({
cfg: params.cfg,
agentId: params.agentId,
resolved,
});
if (primary) {
const wrapper = new FallbackMemoryManager(
{
primary,
fallbackFactory: async () => {
const { MemoryIndexManager } = await import("./manager.js");
return await MemoryIndexManager.get(params);
},
},
() => QMD_MANAGER_CACHE.delete(cacheKey),
);
QMD_MANAGER_CACHE.set(cacheKey, wrapper);
return { manager: wrapper };
}
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
log.warn(`qmd memory unavailable; falling back to builtin: ${message}`);
}
}
try { try {
const { MemoryIndexManager } = await import("./manager.js"); const { MemoryIndexManager } = await import("./manager.js");
const manager = await MemoryIndexManager.get(params); const manager = await MemoryIndexManager.get(params);
@@ -19,3 +60,117 @@ export async function getMemorySearchManager(params: {
return { manager: null, error: message }; return { manager: null, error: message };
} }
} }
class FallbackMemoryManager implements MemorySearchManager {
private fallback: MemorySearchManager | null = null;
private primaryFailed = false;
private lastError?: string;
constructor(
private readonly deps: {
primary: MemorySearchManager;
fallbackFactory: () => Promise<MemorySearchManager | null>;
},
private readonly onClose?: () => void,
) {}
async search(
query: string,
opts?: { maxResults?: number; minScore?: number; sessionKey?: string },
) {
if (!this.primaryFailed) {
try {
return await this.deps.primary.search(query, opts);
} catch (err) {
this.primaryFailed = true;
this.lastError = err instanceof Error ? err.message : String(err);
log.warn(`qmd memory failed; switching to builtin index: ${this.lastError}`);
await this.deps.primary.close?.().catch(() => {});
}
}
const fallback = await this.ensureFallback();
if (fallback) {
return await fallback.search(query, opts);
}
throw new Error(this.lastError ?? "memory search unavailable");
}
async readFile(params: { relPath: string; from?: number; lines?: number }) {
if (!this.primaryFailed) {
return await this.deps.primary.readFile(params);
}
const fallback = await this.ensureFallback();
if (fallback) {
return await fallback.readFile(params);
}
throw new Error(this.lastError ?? "memory read unavailable");
}
status() {
if (!this.primaryFailed) {
return this.deps.primary.status();
}
const fallbackStatus = this.fallback?.status();
if (fallbackStatus) {
const custom = fallbackStatus.custom ?? {};
return {
...fallbackStatus,
custom: {
...custom,
fallback: { disabled: true, reason: this.lastError ?? "unknown" },
},
};
}
const primaryStatus = this.deps.primary.status();
const custom = primaryStatus.custom ?? {};
return {
...primaryStatus,
custom: {
...custom,
fallback: { disabled: true, reason: this.lastError ?? "unknown" },
},
};
}
async sync(params?: {
reason?: string;
force?: boolean;
progress?: (update: MemorySyncProgressUpdate) => void;
}) {
if (!this.primaryFailed) {
await this.deps.primary.sync?.(params);
return;
}
const fallback = await this.ensureFallback();
await fallback?.sync?.(params);
}
async probeVectorAvailability() {
if (!this.primaryFailed) {
return await this.deps.primary.probeVectorAvailability();
}
const fallback = await this.ensureFallback();
return (await fallback?.probeVectorAvailability()) ?? false;
}
async close() {
await this.deps.primary.close?.();
await this.fallback?.close?.();
this.onClose?.();
}
private async ensureFallback(): Promise<MemorySearchManager | null> {
if (this.fallback) return this.fallback;
const fallback = await this.deps.fallbackFactory();
if (!fallback) {
log.warn("memory fallback requested but builtin index is unavailable");
return null;
}
this.fallback = fallback;
return this.fallback;
}
}
function buildQmdCacheKey(agentId: string, config: ResolvedQmdConfig): string {
return `${agentId}:${JSON.stringify(config)}`;
}

72
src/memory/types.ts Normal file
View File

@@ -0,0 +1,72 @@
export type MemorySource = "memory" | "sessions";
export type MemorySearchResult = {
path: string;
startLine: number;
endLine: number;
score: number;
snippet: string;
source: MemorySource;
citation?: string;
};
export type MemorySyncProgressUpdate = {
completed: number;
total: number;
label?: string;
};
export type MemoryProviderStatus = {
backend: "builtin" | "qmd";
provider: string;
model?: string;
requestedProvider?: string;
files?: number;
chunks?: number;
dirty?: boolean;
workspaceDir?: string;
dbPath?: string;
sources?: MemorySource[];
cache?: { enabled: boolean; entries?: number; maxEntries?: number };
fts?: { enabled: boolean; available: boolean; error?: string };
fallback?: { from: string; reason?: string };
vector?: {
enabled: boolean;
available?: boolean;
extensionPath?: string;
loadError?: string;
dims?: number;
};
batch?: {
enabled: boolean;
failures: number;
limit: number;
wait: boolean;
concurrency: number;
pollIntervalMs: number;
timeoutMs: number;
lastError?: string;
lastProvider?: string;
};
custom?: Record<string, unknown>;
};
export interface MemorySearchManager {
search(
query: string,
opts?: { maxResults?: number; minScore?: number; sessionKey?: string },
): Promise<MemorySearchResult[]>;
readFile(params: {
relPath: string;
from?: number;
lines?: number;
}): Promise<{ text: string; path: string }>;
status(): MemoryProviderStatus;
sync?(params?: {
reason?: string;
force?: boolean;
progress?: (update: MemorySyncProgressUpdate) => void;
}): Promise<void>;
probeVectorAvailability(): Promise<boolean>;
close?(): Promise<void>;
}