Compare commits
74 Commits
secrets/pr
...
adabot
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bc4e8d46d2 | ||
|
|
1b64548caf | ||
|
|
9a6f7c9b23 | ||
|
|
7a7f8e480c | ||
|
|
dc7063af88 | ||
|
|
6ff248fd4e | ||
|
|
18b8007d23 | ||
|
|
f093be7b3a | ||
|
|
fee43d505d | ||
|
|
1bc6cdd00c | ||
|
|
85ae75882c | ||
|
|
8d88f2f8de | ||
|
|
c33a0f21cc | ||
|
|
1a8e46b037 | ||
|
|
0a55711110 | ||
|
|
fc92b05046 | ||
|
|
624ba65554 | ||
|
|
a5e0487647 | ||
|
|
68d4ca1e0d | ||
|
|
ed5d6db833 | ||
|
|
a170e25494 | ||
|
|
4d54736b98 | ||
|
|
08b08c66f1 | ||
|
|
e85dd19092 | ||
|
|
7704e5cc44 | ||
|
|
4082d2657e | ||
|
|
8762697d22 | ||
|
|
8086915187 | ||
|
|
0149f39e72 | ||
|
|
0f6a15deca | ||
|
|
e9b9da5a1f | ||
|
|
e562ff4e31 | ||
|
|
a5ebbe4b55 | ||
|
|
e0e98c2c0d | ||
|
|
309c5b6029 | ||
|
|
d4e3549ed2 | ||
|
|
806c5e2d13 | ||
|
|
03e4768732 | ||
|
|
cb1c0658fc | ||
|
|
b70cecc307 | ||
|
|
1f80d4f0d2 | ||
|
|
1ae3afbd6b | ||
|
|
d311438cb4 | ||
|
|
27cb766209 | ||
|
|
4a3d424890 | ||
|
|
fff48a146d | ||
|
|
9f6372241c | ||
|
|
9cfb56696f | ||
|
|
6747967b83 | ||
|
|
7674fa8c15 | ||
|
|
91efe2e432 | ||
|
|
ae1d35aab3 | ||
|
|
bcbeba400e | ||
|
|
c002574371 | ||
|
|
f1753aa336 | ||
|
|
516459395c | ||
|
|
b0a9eb9407 | ||
|
|
f1f32d5723 | ||
|
|
5761b23760 | ||
|
|
370adb0f4b | ||
|
|
bf5a7a05dd | ||
|
|
50f095ecb0 | ||
|
|
8e5fe5fc14 | ||
|
|
e65b052d27 | ||
|
|
f5859e09ab | ||
|
|
d096055a4b | ||
|
|
0908731c54 | ||
|
|
3082c53a76 | ||
|
|
c1371b639e | ||
|
|
66f9f972b2 | ||
|
|
1e4ffdcec8 | ||
|
|
007daf3c27 | ||
|
|
e7ac300b7e | ||
|
|
e65d1deedd |
12
.github/workflows/ci.yml
vendored
12
.github/workflows/ci.yml
vendored
@@ -53,11 +53,17 @@ jobs:
|
||||
|
||||
if [ "${{ github.event_name }}" = "push" ]; then
|
||||
BASE="${{ github.event.before }}"
|
||||
CHANGED="$(git diff --name-only "$BASE" HEAD 2>/dev/null || echo "UNKNOWN")"
|
||||
else
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
# pull_request runs use a merge commit checkout. Diffing parent branches is
|
||||
# more reliable than relying on base SHA availability in rerun attempts.
|
||||
if git rev-parse --verify HEAD^1 >/dev/null 2>&1 && git rev-parse --verify HEAD^2 >/dev/null 2>&1; then
|
||||
CHANGED="$(git diff --name-only HEAD^1...HEAD^2 2>/dev/null || echo "UNKNOWN")"
|
||||
else
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
CHANGED="$(git diff --name-only "$BASE" HEAD 2>/dev/null || echo "UNKNOWN")"
|
||||
fi
|
||||
fi
|
||||
|
||||
CHANGED="$(git diff --name-only "$BASE" HEAD 2>/dev/null || echo "UNKNOWN")"
|
||||
if [ "$CHANGED" = "UNKNOWN" ] || [ -z "$CHANGED" ]; then
|
||||
# Fail-safe: run broad checks if detection fails.
|
||||
echo "run_node=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -86,3 +86,6 @@ USER.md
|
||||
!.agent/workflows/
|
||||
/local/
|
||||
package-lock.json
|
||||
|
||||
# Claude Code local settings
|
||||
.claude/
|
||||
|
||||
667
docs/assets/docs-chat-widget.js
Normal file
667
docs/assets/docs-chat-widget.js
Normal file
@@ -0,0 +1,667 @@
|
||||
(() => {
|
||||
if (document.getElementById("docs-chat-root")) return;
|
||||
|
||||
// Determine if we're on the docs site or embedded elsewhere
|
||||
const hostname = window.location.hostname;
|
||||
const isDocsSite = hostname === "localhost" || hostname === "127.0.0.1" ||
|
||||
hostname.includes("docs.openclaw") || hostname.endsWith(".mintlify.app");
|
||||
const assetsBase = isDocsSite ? "" : "https://docs.openclaw.ai";
|
||||
const apiBase = "https://claw-api.openknot.ai/api";
|
||||
|
||||
// Load marked for markdown rendering (via CDN)
|
||||
let markedReady = false;
|
||||
const loadMarkdownLib = () => {
|
||||
if (window.marked) {
|
||||
markedReady = true;
|
||||
return;
|
||||
}
|
||||
const script = document.createElement("script");
|
||||
script.src = "https://cdn.jsdelivr.net/npm/marked@15.0.6/marked.min.js";
|
||||
script.onload = () => {
|
||||
if (window.marked) {
|
||||
markedReady = true;
|
||||
}
|
||||
};
|
||||
script.onerror = () => console.warn("Failed to load marked library");
|
||||
document.head.appendChild(script);
|
||||
};
|
||||
loadMarkdownLib();
|
||||
|
||||
// Markdown renderer with fallback before module loads
|
||||
const renderMarkdown = (text) => {
|
||||
if (markedReady && window.marked) {
|
||||
// Configure marked for security: disable HTML pass-through
|
||||
const html = window.marked.parse(text, { async: false, gfm: true, breaks: true });
|
||||
// Open links in new tab by rewriting <a> tags
|
||||
return html.replace(/<a href="/g, '<a target="_blank" rel="noopener" href="');
|
||||
}
|
||||
// Fallback: escape HTML and preserve newlines
|
||||
return text
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/\n/g, "<br>");
|
||||
};
|
||||
|
||||
const style = document.createElement("style");
|
||||
style.textContent = `
|
||||
#docs-chat-root { position: fixed; right: 20px; bottom: 20px; z-index: 9999; font-family: var(--font-body, system-ui, -apple-system, sans-serif); }
|
||||
#docs-chat-root.docs-chat-expanded { right: 0; bottom: 0; top: 0; }
|
||||
/* Thin scrollbar styling */
|
||||
#docs-chat-root ::-webkit-scrollbar { width: 6px; height: 6px; }
|
||||
#docs-chat-root ::-webkit-scrollbar-track { background: transparent; }
|
||||
#docs-chat-root ::-webkit-scrollbar-thumb { background: var(--docs-chat-panel-border); border-radius: 3px; }
|
||||
#docs-chat-root ::-webkit-scrollbar-thumb:hover { background: var(--docs-chat-muted); }
|
||||
#docs-chat-root * { scrollbar-width: thin; scrollbar-color: var(--docs-chat-panel-border) transparent; }
|
||||
:root {
|
||||
--docs-chat-accent: var(--accent, #ff7d60);
|
||||
--docs-chat-text: #1a1a1a;
|
||||
--docs-chat-muted: #555;
|
||||
--docs-chat-panel: rgba(255, 255, 255, 0.92);
|
||||
--docs-chat-panel-border: rgba(0, 0, 0, 0.1);
|
||||
--docs-chat-surface: rgba(250, 250, 250, 0.95);
|
||||
--docs-chat-shadow: 0 18px 50px rgba(0,0,0,0.15);
|
||||
--docs-chat-code-bg: rgba(0, 0, 0, 0.05);
|
||||
--docs-chat-assistant-bg: #f5f5f5;
|
||||
}
|
||||
html[data-theme="dark"] {
|
||||
--docs-chat-text: #e8e8e8;
|
||||
--docs-chat-muted: #aaa;
|
||||
--docs-chat-panel: rgba(28, 28, 30, 0.95);
|
||||
--docs-chat-panel-border: rgba(255, 255, 255, 0.12);
|
||||
--docs-chat-surface: rgba(38, 38, 40, 0.95);
|
||||
--docs-chat-shadow: 0 18px 50px rgba(0,0,0,0.5);
|
||||
--docs-chat-code-bg: rgba(255, 255, 255, 0.08);
|
||||
--docs-chat-assistant-bg: #2a2a2c;
|
||||
}
|
||||
#docs-chat-button {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
background: linear-gradient(140deg, rgba(255,90,54,0.25), rgba(255,90,54,0.06));
|
||||
color: var(--docs-chat-text);
|
||||
border: 1px solid rgba(255,90,54,0.4);
|
||||
border-radius: 999px;
|
||||
padding: 10px 14px;
|
||||
cursor: pointer;
|
||||
box-shadow: 0 8px 30px rgba(255,90,54, 0.08);
|
||||
backdrop-filter: blur(12px);
|
||||
-webkit-backdrop-filter: blur(12px);
|
||||
font-family: var(--font-pixel, var(--font-body, system-ui, sans-serif));
|
||||
}
|
||||
#docs-chat-button span { font-weight: 600; letter-spacing: 0.04em; font-size: 14px; }
|
||||
.docs-chat-logo { width: 20px; height: 20px; }
|
||||
#docs-chat-panel {
|
||||
width: min(440px, calc(100vw - 40px));
|
||||
height: min(696px, calc(100vh - 80px));
|
||||
background: var(--docs-chat-panel);
|
||||
color: var(--docs-chat-text);
|
||||
border-radius: 16px;
|
||||
border: 1px solid var(--docs-chat-panel-border);
|
||||
box-shadow: var(--docs-chat-shadow);
|
||||
display: none;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
backdrop-filter: blur(16px);
|
||||
-webkit-backdrop-filter: blur(16px);
|
||||
}
|
||||
#docs-chat-root.docs-chat-expanded #docs-chat-panel {
|
||||
width: min(512px, 100vw);
|
||||
height: 100vh;
|
||||
height: 100dvh;
|
||||
border-radius: 18px 0 0 18px;
|
||||
padding-top: env(safe-area-inset-top, 0);
|
||||
padding-bottom: env(safe-area-inset-bottom, 0);
|
||||
}
|
||||
@media (max-width: 520px) {
|
||||
#docs-chat-root.docs-chat-expanded #docs-chat-panel {
|
||||
width: 100vw;
|
||||
border-radius: 0;
|
||||
}
|
||||
#docs-chat-root.docs-chat-expanded { right: 0; left: 0; bottom: 0; top: 0; }
|
||||
}
|
||||
#docs-chat-header {
|
||||
padding: 12px 14px;
|
||||
font-weight: 600;
|
||||
font-family: var(--font-pixel, var(--font-body, system-ui, sans-serif));
|
||||
letter-spacing: 0.03em;
|
||||
border-bottom: 1px solid var(--docs-chat-panel-border);
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
}
|
||||
#docs-chat-header-title { display: inline-flex; align-items: center; gap: 8px; }
|
||||
#docs-chat-header-title span { color: var(--docs-chat-text); font-size: 15px; }
|
||||
#docs-chat-header-actions { display: inline-flex; align-items: center; gap: 6px; }
|
||||
.docs-chat-icon-button {
|
||||
border: 1px solid var(--docs-chat-panel-border);
|
||||
background: transparent;
|
||||
color: inherit;
|
||||
border-radius: 8px;
|
||||
width: 30px;
|
||||
height: 30px;
|
||||
cursor: pointer;
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
}
|
||||
#docs-chat-messages { flex: 1; padding: 12px 14px; overflow: auto; background: transparent; }
|
||||
#docs-chat-input {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
padding: 12px 14px;
|
||||
border-top: 1px solid var(--docs-chat-panel-border);
|
||||
background: var(--docs-chat-surface);
|
||||
backdrop-filter: blur(10px);
|
||||
-webkit-backdrop-filter: blur(10px);
|
||||
}
|
||||
#docs-chat-input textarea {
|
||||
flex: 1;
|
||||
resize: none;
|
||||
border: 1px solid var(--docs-chat-panel-border);
|
||||
border-radius: 10px;
|
||||
padding: 9px 10px;
|
||||
font-size: 14px;
|
||||
line-height: 1.5;
|
||||
font-family: inherit;
|
||||
color: var(--docs-chat-text);
|
||||
background: var(--docs-chat-surface);
|
||||
min-height: 42px;
|
||||
max-height: 120px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
#docs-chat-input textarea::placeholder { color: var(--docs-chat-muted); }
|
||||
#docs-chat-send {
|
||||
background: var(--docs-chat-accent);
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: 10px;
|
||||
padding: 8px 14px;
|
||||
cursor: pointer;
|
||||
font-weight: 600;
|
||||
font-family: inherit;
|
||||
font-size: 14px;
|
||||
transition: opacity 0.15s ease;
|
||||
}
|
||||
#docs-chat-send:hover { opacity: 0.9; }
|
||||
#docs-chat-send:active { opacity: 0.8; }
|
||||
.docs-chat-bubble {
|
||||
margin-bottom: 10px;
|
||||
padding: 10px 14px;
|
||||
border-radius: 12px;
|
||||
font-size: 14px;
|
||||
line-height: 1.6;
|
||||
max-width: 92%;
|
||||
}
|
||||
.docs-chat-user {
|
||||
background: rgba(255, 125, 96, 0.15);
|
||||
color: var(--docs-chat-text);
|
||||
border: 1px solid rgba(255, 125, 96, 0.3);
|
||||
align-self: flex-end;
|
||||
white-space: pre-wrap;
|
||||
margin-left: auto;
|
||||
}
|
||||
html[data-theme="dark"] .docs-chat-user {
|
||||
background: rgba(255, 125, 96, 0.18);
|
||||
border-color: rgba(255, 125, 96, 0.35);
|
||||
}
|
||||
.docs-chat-assistant {
|
||||
background: var(--docs-chat-assistant-bg);
|
||||
color: var(--docs-chat-text);
|
||||
border: 1px solid var(--docs-chat-panel-border);
|
||||
}
|
||||
/* Markdown content styling for chat bubbles */
|
||||
.docs-chat-assistant p { margin: 0 0 10px 0; }
|
||||
.docs-chat-assistant p:last-child { margin-bottom: 0; }
|
||||
.docs-chat-assistant code {
|
||||
background: var(--docs-chat-code-bg);
|
||||
padding: 2px 6px;
|
||||
border-radius: 5px;
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
.docs-chat-assistant pre {
|
||||
background: var(--docs-chat-code-bg);
|
||||
padding: 10px 12px;
|
||||
border-radius: 8px;
|
||||
overflow-x: auto;
|
||||
margin: 6px 0;
|
||||
font-size: 0.9em;
|
||||
max-width: 100%;
|
||||
white-space: pre;
|
||||
word-wrap: normal;
|
||||
}
|
||||
.docs-chat-assistant pre::-webkit-scrollbar-thumb { background: transparent; }
|
||||
.docs-chat-assistant pre:hover::-webkit-scrollbar-thumb { background: var(--docs-chat-panel-border); }
|
||||
@media (hover: none) {
|
||||
.docs-chat-assistant pre { -webkit-overflow-scrolling: touch; }
|
||||
.docs-chat-assistant pre::-webkit-scrollbar-thumb { background: var(--docs-chat-panel-border); }
|
||||
}
|
||||
.docs-chat-assistant pre code {
|
||||
background: transparent;
|
||||
padding: 0;
|
||||
font-size: inherit;
|
||||
white-space: pre;
|
||||
word-wrap: normal;
|
||||
display: block;
|
||||
}
|
||||
/* Compact single-line code blocks */
|
||||
.docs-chat-assistant pre.compact {
|
||||
margin: 4px 0;
|
||||
padding: 6px 10px;
|
||||
}
|
||||
/* Longer code blocks with copy button need extra top padding */
|
||||
.docs-chat-assistant pre:not(.compact) {
|
||||
padding-top: 28px;
|
||||
}
|
||||
.docs-chat-assistant a {
|
||||
color: var(--docs-chat-accent);
|
||||
text-decoration: underline;
|
||||
text-underline-offset: 2px;
|
||||
}
|
||||
.docs-chat-assistant a:hover { opacity: 0.8; }
|
||||
.docs-chat-assistant ul, .docs-chat-assistant ol {
|
||||
margin: 8px 0;
|
||||
padding-left: 18px;
|
||||
list-style: none;
|
||||
}
|
||||
.docs-chat-assistant li {
|
||||
margin: 4px 0;
|
||||
position: relative;
|
||||
padding-left: 14px;
|
||||
}
|
||||
.docs-chat-assistant li::before {
|
||||
content: "•";
|
||||
position: absolute;
|
||||
left: 0;
|
||||
color: var(--docs-chat-muted);
|
||||
}
|
||||
.docs-chat-assistant strong { font-weight: 600; }
|
||||
.docs-chat-assistant em { font-style: italic; }
|
||||
.docs-chat-assistant h1, .docs-chat-assistant h2, .docs-chat-assistant h3 {
|
||||
font-weight: 600;
|
||||
margin: 12px 0 6px 0;
|
||||
line-height: 1.3;
|
||||
}
|
||||
.docs-chat-assistant h1 { font-size: 1.2em; }
|
||||
.docs-chat-assistant h2 { font-size: 1.1em; }
|
||||
.docs-chat-assistant h3 { font-size: 1.05em; }
|
||||
.docs-chat-assistant blockquote {
|
||||
border-left: 3px solid var(--docs-chat-accent);
|
||||
margin: 10px 0;
|
||||
padding: 4px 12px;
|
||||
color: var(--docs-chat-muted);
|
||||
background: var(--docs-chat-code-bg);
|
||||
border-radius: 0 6px 6px 0;
|
||||
}
|
||||
.docs-chat-assistant hr {
|
||||
border: none;
|
||||
height: 1px;
|
||||
background: var(--docs-chat-panel-border);
|
||||
margin: 12px 0;
|
||||
}
|
||||
/* Copy buttons */
|
||||
.docs-chat-assistant { position: relative; padding-top: 28px; }
|
||||
.docs-chat-copy-response {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 8px;
|
||||
background: var(--docs-chat-surface);
|
||||
border: 1px solid var(--docs-chat-panel-border);
|
||||
border-radius: 5px;
|
||||
padding: 4px 8px;
|
||||
font-size: 11px;
|
||||
cursor: pointer;
|
||||
color: var(--docs-chat-muted);
|
||||
transition: color 0.15s ease, background 0.15s ease;
|
||||
}
|
||||
.docs-chat-copy-response:hover {
|
||||
color: var(--docs-chat-text);
|
||||
background: var(--docs-chat-code-bg);
|
||||
}
|
||||
.docs-chat-assistant pre {
|
||||
position: relative;
|
||||
}
|
||||
.docs-chat-copy-code {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 8px;
|
||||
background: var(--docs-chat-surface);
|
||||
border: 1px solid var(--docs-chat-panel-border);
|
||||
border-radius: 4px;
|
||||
padding: 3px 7px;
|
||||
font-size: 10px;
|
||||
cursor: pointer;
|
||||
color: var(--docs-chat-muted);
|
||||
transition: color 0.15s ease, background 0.15s ease;
|
||||
z-index: 1;
|
||||
}
|
||||
.docs-chat-copy-code:hover {
|
||||
color: var(--docs-chat-text);
|
||||
background: var(--docs-chat-code-bg);
|
||||
}
|
||||
/* Resize handle - left edge of expanded panel */
|
||||
#docs-chat-resize-handle {
|
||||
position: absolute;
|
||||
left: 0;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
width: 6px;
|
||||
cursor: ew-resize;
|
||||
z-index: 10;
|
||||
display: none;
|
||||
}
|
||||
#docs-chat-root.docs-chat-expanded #docs-chat-resize-handle { display: block; }
|
||||
#docs-chat-resize-handle::after {
|
||||
content: "";
|
||||
position: absolute;
|
||||
left: 1px;
|
||||
top: 50%;
|
||||
transform: translateY(-50%);
|
||||
width: 4px;
|
||||
height: 40px;
|
||||
border-radius: 2px;
|
||||
background: var(--docs-chat-panel-border);
|
||||
opacity: 0;
|
||||
transition: opacity 0.15s ease, background 0.15s ease;
|
||||
}
|
||||
#docs-chat-resize-handle:hover::after,
|
||||
#docs-chat-resize-handle.docs-chat-dragging::after {
|
||||
opacity: 1;
|
||||
background: var(--docs-chat-accent);
|
||||
}
|
||||
@media (max-width: 520px) {
|
||||
#docs-chat-resize-handle { display: none !important; }
|
||||
}
|
||||
`;
|
||||
document.head.appendChild(style);
|
||||
|
||||
const root = document.createElement("div");
|
||||
root.id = "docs-chat-root";
|
||||
|
||||
const button = document.createElement("button");
|
||||
button.id = "docs-chat-button";
|
||||
button.type = "button";
|
||||
button.innerHTML =
|
||||
`<img class="docs-chat-logo" src="${assetsBase}/assets/pixel-lobster.svg" alt="OpenClaw">` +
|
||||
`<span>Ask Molty</span>`;
|
||||
|
||||
const panel = document.createElement("div");
|
||||
panel.id = "docs-chat-panel";
|
||||
panel.style.display = "none";
|
||||
|
||||
// Resize handle for expandable sidebar width (desktop only)
|
||||
const resizeHandle = document.createElement("div");
|
||||
resizeHandle.id = "docs-chat-resize-handle";
|
||||
|
||||
const header = document.createElement("div");
|
||||
header.id = "docs-chat-header";
|
||||
header.innerHTML =
|
||||
`<div id="docs-chat-header-title">` +
|
||||
`<img class="docs-chat-logo" src="${assetsBase}/assets/pixel-lobster.svg" alt="OpenClaw">` +
|
||||
`<span>OpenClaw Docs</span>` +
|
||||
`</div>` +
|
||||
`<div id="docs-chat-header-actions"></div>`;
|
||||
const headerActions = header.querySelector("#docs-chat-header-actions");
|
||||
const expand = document.createElement("button");
|
||||
expand.type = "button";
|
||||
expand.className = "docs-chat-icon-button";
|
||||
expand.setAttribute("aria-label", "Expand");
|
||||
expand.textContent = "⤢";
|
||||
const clear = document.createElement("button");
|
||||
clear.type = "button";
|
||||
clear.className = "docs-chat-icon-button";
|
||||
clear.setAttribute("aria-label", "Clear chat");
|
||||
clear.textContent = "⌫";
|
||||
const close = document.createElement("button");
|
||||
close.type = "button";
|
||||
close.className = "docs-chat-icon-button";
|
||||
close.setAttribute("aria-label", "Close");
|
||||
close.textContent = "×";
|
||||
headerActions.appendChild(expand);
|
||||
headerActions.appendChild(clear);
|
||||
headerActions.appendChild(close);
|
||||
|
||||
const messages = document.createElement("div");
|
||||
messages.id = "docs-chat-messages";
|
||||
|
||||
const inputWrap = document.createElement("div");
|
||||
inputWrap.id = "docs-chat-input";
|
||||
const textarea = document.createElement("textarea");
|
||||
textarea.rows = 1;
|
||||
textarea.placeholder = "Ask about OpenClaw Docs...";
|
||||
|
||||
// Auto-expand textarea as user types (up to max-height set in CSS)
|
||||
const autoExpand = () => {
|
||||
textarea.style.height = "auto";
|
||||
textarea.style.height = Math.min(textarea.scrollHeight, 224) + "px";
|
||||
};
|
||||
textarea.addEventListener("input", autoExpand);
|
||||
|
||||
const send = document.createElement("button");
|
||||
send.id = "docs-chat-send";
|
||||
send.type = "button";
|
||||
send.textContent = "Send";
|
||||
|
||||
inputWrap.appendChild(textarea);
|
||||
inputWrap.appendChild(send);
|
||||
|
||||
panel.appendChild(resizeHandle);
|
||||
panel.appendChild(header);
|
||||
panel.appendChild(messages);
|
||||
panel.appendChild(inputWrap);
|
||||
|
||||
root.appendChild(button);
|
||||
root.appendChild(panel);
|
||||
document.body.appendChild(root);
|
||||
|
||||
// Add copy buttons to assistant bubble
|
||||
const addCopyButtons = (bubble, rawText) => {
|
||||
// Add copy response button
|
||||
const copyResponse = document.createElement("button");
|
||||
copyResponse.className = "docs-chat-copy-response";
|
||||
copyResponse.textContent = "Copy";
|
||||
copyResponse.type = "button";
|
||||
copyResponse.addEventListener("click", async () => {
|
||||
try {
|
||||
await navigator.clipboard.writeText(rawText);
|
||||
copyResponse.textContent = "Copied!";
|
||||
setTimeout(() => (copyResponse.textContent = "Copy"), 1500);
|
||||
} catch (e) {
|
||||
copyResponse.textContent = "Failed";
|
||||
}
|
||||
});
|
||||
bubble.appendChild(copyResponse);
|
||||
|
||||
// Add copy buttons to code blocks (skip short/single-line blocks)
|
||||
bubble.querySelectorAll("pre").forEach((pre) => {
|
||||
const code = pre.querySelector("code") || pre;
|
||||
const text = code.textContent || "";
|
||||
const lineCount = text.split("\n").length;
|
||||
const isShort = lineCount <= 2 && text.length < 100;
|
||||
|
||||
if (isShort) {
|
||||
pre.classList.add("compact");
|
||||
return; // Skip copy button for compact blocks
|
||||
}
|
||||
|
||||
const copyCode = document.createElement("button");
|
||||
copyCode.className = "docs-chat-copy-code";
|
||||
copyCode.textContent = "Copy";
|
||||
copyCode.type = "button";
|
||||
copyCode.addEventListener("click", async (e) => {
|
||||
e.stopPropagation();
|
||||
try {
|
||||
await navigator.clipboard.writeText(text);
|
||||
copyCode.textContent = "Copied!";
|
||||
setTimeout(() => (copyCode.textContent = "Copy"), 1500);
|
||||
} catch (err) {
|
||||
copyCode.textContent = "Failed";
|
||||
}
|
||||
});
|
||||
pre.appendChild(copyCode);
|
||||
});
|
||||
};
|
||||
|
||||
const addBubble = (text, role, isMarkdown = false) => {
|
||||
const bubble = document.createElement("div");
|
||||
bubble.className =
|
||||
"docs-chat-bubble " +
|
||||
(role === "user" ? "docs-chat-user" : "docs-chat-assistant");
|
||||
if (isMarkdown && role === "assistant") {
|
||||
bubble.innerHTML = renderMarkdown(text);
|
||||
} else {
|
||||
bubble.textContent = text;
|
||||
}
|
||||
messages.appendChild(bubble);
|
||||
messages.scrollTop = messages.scrollHeight;
|
||||
return bubble;
|
||||
};
|
||||
|
||||
let isExpanded = false;
|
||||
let customWidth = null; // User-set width via drag
|
||||
const MIN_WIDTH = 320;
|
||||
const MAX_WIDTH = 800;
|
||||
|
||||
// Drag-to-resize logic
|
||||
let isDragging = false;
|
||||
let startX, startWidth;
|
||||
|
||||
resizeHandle.addEventListener("mousedown", (e) => {
|
||||
if (!isExpanded) return;
|
||||
isDragging = true;
|
||||
startX = e.clientX;
|
||||
startWidth = panel.offsetWidth;
|
||||
resizeHandle.classList.add("docs-chat-dragging");
|
||||
document.body.style.cursor = "ew-resize";
|
||||
document.body.style.userSelect = "none";
|
||||
e.preventDefault();
|
||||
});
|
||||
|
||||
document.addEventListener("mousemove", (e) => {
|
||||
if (!isDragging) return;
|
||||
// Panel is on right, so dragging left increases width
|
||||
const delta = startX - e.clientX;
|
||||
const newWidth = Math.min(MAX_WIDTH, Math.max(MIN_WIDTH, startWidth + delta));
|
||||
customWidth = newWidth;
|
||||
panel.style.width = newWidth + "px";
|
||||
});
|
||||
|
||||
document.addEventListener("mouseup", () => {
|
||||
if (!isDragging) return;
|
||||
isDragging = false;
|
||||
resizeHandle.classList.remove("docs-chat-dragging");
|
||||
document.body.style.cursor = "";
|
||||
document.body.style.userSelect = "";
|
||||
});
|
||||
|
||||
const setOpen = (isOpen) => {
|
||||
panel.style.display = isOpen ? "flex" : "none";
|
||||
button.style.display = isOpen ? "none" : "inline-flex";
|
||||
root.classList.toggle("docs-chat-expanded", isOpen && isExpanded);
|
||||
if (!isOpen) {
|
||||
panel.style.width = ""; // Reset to CSS default when closed
|
||||
} else if (isExpanded && customWidth) {
|
||||
panel.style.width = customWidth + "px";
|
||||
}
|
||||
if (isOpen) textarea.focus();
|
||||
};
|
||||
|
||||
const setExpanded = (next) => {
|
||||
isExpanded = next;
|
||||
expand.textContent = isExpanded ? "⤡" : "⤢";
|
||||
expand.setAttribute("aria-label", isExpanded ? "Collapse" : "Expand");
|
||||
if (panel.style.display !== "none") {
|
||||
root.classList.toggle("docs-chat-expanded", isExpanded);
|
||||
if (isExpanded && customWidth) {
|
||||
panel.style.width = customWidth + "px";
|
||||
} else if (!isExpanded) {
|
||||
panel.style.width = ""; // Reset to CSS default
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
button.addEventListener("click", () => setOpen(true));
|
||||
expand.addEventListener("click", () => setExpanded(!isExpanded));
|
||||
clear.addEventListener("click", () => {
|
||||
messages.innerHTML = "";
|
||||
});
|
||||
close.addEventListener("click", () => {
|
||||
setOpen(false);
|
||||
root.classList.remove("docs-chat-expanded");
|
||||
});
|
||||
|
||||
const sendMessage = async () => {
|
||||
const text = textarea.value.trim();
|
||||
if (!text) return;
|
||||
textarea.value = "";
|
||||
textarea.style.height = "auto"; // Reset height after sending
|
||||
addBubble(text, "user");
|
||||
const assistantBubble = addBubble("...", "assistant");
|
||||
assistantBubble.innerHTML = "";
|
||||
|
||||
let fullText = "";
|
||||
try {
|
||||
const response = await fetch(`${apiBase}/chat`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ message: text }),
|
||||
});
|
||||
|
||||
// Handle rate limiting
|
||||
if (response.status === 429) {
|
||||
const retryAfter = response.headers.get("Retry-After") || "60";
|
||||
fullText = `You're asking questions too quickly. Please wait ${retryAfter} seconds before trying again.`;
|
||||
assistantBubble.innerHTML = renderMarkdown(fullText);
|
||||
addCopyButtons(assistantBubble, fullText);
|
||||
return;
|
||||
}
|
||||
|
||||
// Handle other errors
|
||||
if (!response.ok) {
|
||||
try {
|
||||
const errorData = await response.json();
|
||||
fullText = errorData.error || "Something went wrong. Please try again.";
|
||||
} catch {
|
||||
fullText = "Something went wrong. Please try again.";
|
||||
}
|
||||
assistantBubble.innerHTML = renderMarkdown(fullText);
|
||||
addCopyButtons(assistantBubble, fullText);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!response.body) {
|
||||
fullText = await response.text();
|
||||
assistantBubble.innerHTML = renderMarkdown(fullText);
|
||||
addCopyButtons(assistantBubble, fullText);
|
||||
return;
|
||||
}
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
fullText += decoder.decode(value, { stream: true });
|
||||
// Re-render markdown on each chunk for live preview
|
||||
assistantBubble.innerHTML = renderMarkdown(fullText);
|
||||
messages.scrollTop = messages.scrollHeight;
|
||||
}
|
||||
// Flush any remaining buffered bytes (partial UTF-8 sequences)
|
||||
fullText += decoder.decode();
|
||||
assistantBubble.innerHTML = renderMarkdown(fullText);
|
||||
// Add copy buttons after streaming completes
|
||||
addCopyButtons(assistantBubble, fullText);
|
||||
} catch (err) {
|
||||
fullText = "Failed to reach docs chat API.";
|
||||
assistantBubble.innerHTML = renderMarkdown(fullText);
|
||||
addCopyButtons(assistantBubble, fullText);
|
||||
}
|
||||
};
|
||||
|
||||
send.addEventListener("click", sendMessage);
|
||||
textarea.addEventListener("keydown", (event) => {
|
||||
if (event.key === "Enter" && !event.shiftKey) {
|
||||
event.preventDefault();
|
||||
sendMessage();
|
||||
}
|
||||
});
|
||||
})();
|
||||
@@ -99,7 +99,8 @@ Text + native (when enabled):
|
||||
- `/reasoning on|off|stream` (alias: `/reason`; when on, sends a separate message prefixed `Reasoning:`; `stream` = Telegram draft only)
|
||||
- `/elevated on|off|ask|full` (alias: `/elev`; `full` skips exec approvals)
|
||||
- `/exec host=<sandbox|gateway|node> security=<deny|allowlist|full> ask=<off|on-miss|always> node=<id>` (send `/exec` to show current)
|
||||
- `/model <name>` (alias: `/models`; or `/<alias>` from `agents.defaults.models.*.alias`)
|
||||
- `/model <name>` (alias: `.model`; or `/<alias>` from `agents.defaults.models.*.alias`)
|
||||
- `/models [provider]` (alias: `.models`)
|
||||
- `/queue <mode>` (plus options like `debounce:2s cap:25 drop:summarize`; send `/queue` to see current settings)
|
||||
- `/bash <command>` (host-only; alias for `! <command>`; requires `commands.bash: true` + `tools.elevated` allowlists)
|
||||
|
||||
|
||||
85
extensions/memory-lancedb/benchmark.mjs
Normal file
85
extensions/memory-lancedb/benchmark.mjs
Normal file
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* LanceDB performance benchmark
|
||||
*/
|
||||
import * as lancedb from "@lancedb/lancedb";
|
||||
import OpenAI from "openai";
|
||||
|
||||
const LANCEDB_PATH = "/home/tsukhani/.openclaw/memory/lancedb";
|
||||
const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
|
||||
|
||||
const openai = new OpenAI({ apiKey: OPENAI_API_KEY });
|
||||
|
||||
async function embed(text) {
|
||||
const start = Date.now();
|
||||
const response = await openai.embeddings.create({
|
||||
model: "text-embedding-3-small",
|
||||
input: text,
|
||||
});
|
||||
const embedTime = Date.now() - start;
|
||||
return { vector: response.data[0].embedding, embedTime };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("📊 LanceDB Performance Benchmark");
|
||||
console.log("================================\n");
|
||||
|
||||
// Connect
|
||||
const connectStart = Date.now();
|
||||
const db = await lancedb.connect(LANCEDB_PATH);
|
||||
const table = await db.openTable("memories");
|
||||
const connectTime = Date.now() - connectStart;
|
||||
console.log(`Connection time: ${connectTime}ms`);
|
||||
|
||||
const count = await table.countRows();
|
||||
console.log(`Total memories: ${count}\n`);
|
||||
|
||||
// Test queries
|
||||
const queries = [
|
||||
"Tarun's preferences",
|
||||
"What is the OpenRouter API key location?",
|
||||
"meeting schedule",
|
||||
"Abundent Academy training",
|
||||
"slate blue",
|
||||
];
|
||||
|
||||
console.log("Search benchmarks (5 runs each, limit=5):\n");
|
||||
|
||||
for (const query of queries) {
|
||||
const times = [];
|
||||
let embedTime = 0;
|
||||
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const { vector, embedTime: et } = await embed(query);
|
||||
embedTime = et; // Last one
|
||||
|
||||
const searchStart = Date.now();
|
||||
const _results = await table.vectorSearch(vector).limit(5).toArray();
|
||||
const searchTime = Date.now() - searchStart;
|
||||
times.push(searchTime);
|
||||
}
|
||||
|
||||
const avg = Math.round(times.reduce((a, b) => a + b, 0) / times.length);
|
||||
const min = Math.min(...times);
|
||||
const max = Math.max(...times);
|
||||
|
||||
console.log(`"${query}"`);
|
||||
console.log(` Embedding: ${embedTime}ms`);
|
||||
console.log(` Search: avg=${avg}ms, min=${min}ms, max=${max}ms`);
|
||||
console.log("");
|
||||
}
|
||||
|
||||
// Raw vector search (no embedding)
|
||||
console.log("\nRaw vector search (pre-computed embedding):");
|
||||
const { vector } = await embed("test query");
|
||||
const rawTimes = [];
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const start = Date.now();
|
||||
await table.vectorSearch(vector).limit(5).toArray();
|
||||
rawTimes.push(Date.now() - start);
|
||||
}
|
||||
const avgRaw = Math.round(rawTimes.reduce((a, b) => a + b, 0) / rawTimes.length);
|
||||
console.log(` avg=${avgRaw}ms, min=${Math.min(...rawTimes)}ms, max=${Math.max(...rawTimes)}ms`);
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
@@ -2,6 +2,20 @@ import fs from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
export type AutoCaptureConfig = {
|
||||
enabled: boolean;
|
||||
/** LLM provider for memory extraction: "openrouter" (default) or "openai" */
|
||||
provider?: "openrouter" | "openai";
|
||||
/** LLM model for memory extraction (default: google/gemini-2.0-flash-001) */
|
||||
model?: string;
|
||||
/** API key for the LLM provider (supports ${ENV_VAR} syntax) */
|
||||
apiKey?: string;
|
||||
/** Base URL for the LLM provider (default: https://openrouter.ai/api/v1) */
|
||||
baseUrl?: string;
|
||||
/** Maximum messages to send for extraction (default: 10) */
|
||||
maxMessages?: number;
|
||||
};
|
||||
|
||||
export type MemoryConfig = {
|
||||
embedding: {
|
||||
provider: "openai";
|
||||
@@ -9,12 +23,27 @@ export type MemoryConfig = {
|
||||
apiKey: string;
|
||||
};
|
||||
dbPath?: string;
|
||||
autoCapture?: boolean;
|
||||
/** @deprecated Use autoCapture object instead. Boolean true enables with defaults. */
|
||||
autoCapture?: boolean | AutoCaptureConfig;
|
||||
autoRecall?: boolean;
|
||||
captureMaxChars?: number;
|
||||
coreMemory?: {
|
||||
enabled?: boolean;
|
||||
/** Maximum number of core memories to load */
|
||||
maxEntries?: number;
|
||||
/** Minimum importance threshold for core memories */
|
||||
minImportance?: number;
|
||||
};
|
||||
};
|
||||
|
||||
export const MEMORY_CATEGORIES = ["preference", "fact", "decision", "entity", "other"] as const;
|
||||
export const MEMORY_CATEGORIES = [
|
||||
"preference",
|
||||
"fact",
|
||||
"decision",
|
||||
"entity",
|
||||
"other",
|
||||
"core",
|
||||
] as const;
|
||||
export type MemoryCategory = (typeof MEMORY_CATEGORIES)[number];
|
||||
|
||||
const DEFAULT_MODEL = "text-embedding-3-small";
|
||||
@@ -93,7 +122,7 @@ export const memoryConfigSchema = {
|
||||
const cfg = value as Record<string, unknown>;
|
||||
assertAllowedKeys(
|
||||
cfg,
|
||||
["embedding", "dbPath", "autoCapture", "autoRecall", "captureMaxChars"],
|
||||
["embedding", "dbPath", "autoCapture", "autoRecall", "captureMaxChars", "coreMemory"],
|
||||
"memory config",
|
||||
);
|
||||
|
||||
@@ -114,6 +143,43 @@ export const memoryConfigSchema = {
|
||||
throw new Error("captureMaxChars must be between 100 and 10000");
|
||||
}
|
||||
|
||||
// Parse autoCapture (supports boolean for backward compat, or object for LLM config)
|
||||
let autoCapture: MemoryConfig["autoCapture"];
|
||||
if (cfg.autoCapture === false || cfg.autoCapture === undefined) {
|
||||
autoCapture = false;
|
||||
} else if (cfg.autoCapture === true) {
|
||||
// Legacy boolean true — enable with defaults
|
||||
autoCapture = { enabled: true };
|
||||
} else if (typeof cfg.autoCapture === "object" && !Array.isArray(cfg.autoCapture)) {
|
||||
const ac = cfg.autoCapture as Record<string, unknown>;
|
||||
assertAllowedKeys(
|
||||
ac,
|
||||
["enabled", "provider", "model", "apiKey", "baseUrl", "maxMessages"],
|
||||
"autoCapture config",
|
||||
);
|
||||
autoCapture = {
|
||||
enabled: ac.enabled !== false,
|
||||
provider:
|
||||
ac.provider === "openai" || ac.provider === "openrouter" ? ac.provider : "openrouter",
|
||||
model: typeof ac.model === "string" ? ac.model : undefined,
|
||||
apiKey: typeof ac.apiKey === "string" ? resolveEnvVars(ac.apiKey) : undefined,
|
||||
baseUrl: typeof ac.baseUrl === "string" ? ac.baseUrl : undefined,
|
||||
maxMessages: typeof ac.maxMessages === "number" ? ac.maxMessages : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse coreMemory
|
||||
let coreMemory: MemoryConfig["coreMemory"];
|
||||
if (cfg.coreMemory && typeof cfg.coreMemory === "object" && !Array.isArray(cfg.coreMemory)) {
|
||||
const bc = cfg.coreMemory as Record<string, unknown>;
|
||||
assertAllowedKeys(bc, ["enabled", "maxEntries", "minImportance"], "coreMemory config");
|
||||
coreMemory = {
|
||||
enabled: bc.enabled === true,
|
||||
maxEntries: typeof bc.maxEntries === "number" ? bc.maxEntries : 50,
|
||||
minImportance: typeof bc.minImportance === "number" ? bc.minImportance : 0.5,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
embedding: {
|
||||
provider: "openai",
|
||||
@@ -121,9 +187,11 @@ export const memoryConfigSchema = {
|
||||
apiKey: resolveEnvVars(embedding.apiKey),
|
||||
},
|
||||
dbPath: typeof cfg.dbPath === "string" ? cfg.dbPath : DEFAULT_DB_PATH,
|
||||
autoCapture: cfg.autoCapture === true,
|
||||
autoCapture: autoCapture ?? false,
|
||||
autoRecall: cfg.autoRecall !== false,
|
||||
captureMaxChars: captureMaxChars ?? DEFAULT_CAPTURE_MAX_CHARS,
|
||||
// Default coreMemory to enabled for consistency with autoCapture/autoRecall
|
||||
coreMemory: coreMemory ?? { enabled: true, maxEntries: 50, minImportance: 0.5 },
|
||||
};
|
||||
},
|
||||
uiHints: {
|
||||
@@ -143,19 +211,47 @@ export const memoryConfigSchema = {
|
||||
placeholder: "~/.openclaw/memory/lancedb",
|
||||
advanced: true,
|
||||
},
|
||||
autoCapture: {
|
||||
"autoCapture.enabled": {
|
||||
label: "Auto-Capture",
|
||||
help: "Automatically capture important information from conversations",
|
||||
help: "Automatically capture important information from conversations using LLM extraction",
|
||||
},
|
||||
"autoCapture.provider": {
|
||||
label: "Capture LLM Provider",
|
||||
placeholder: "openrouter",
|
||||
advanced: true,
|
||||
help: "LLM provider for memory extraction (openrouter or openai)",
|
||||
},
|
||||
"autoCapture.model": {
|
||||
label: "Capture Model",
|
||||
placeholder: "google/gemini-2.0-flash-001",
|
||||
advanced: true,
|
||||
help: "LLM model for memory extraction (use a fast/cheap model)",
|
||||
},
|
||||
"autoCapture.apiKey": {
|
||||
label: "Capture API Key",
|
||||
sensitive: true,
|
||||
advanced: true,
|
||||
help: "API key for capture LLM (defaults to OpenRouter key from provider config)",
|
||||
},
|
||||
autoRecall: {
|
||||
label: "Auto-Recall",
|
||||
help: "Automatically inject relevant memories into context",
|
||||
},
|
||||
captureMaxChars: {
|
||||
label: "Capture Max Chars",
|
||||
help: "Maximum message length eligible for auto-capture",
|
||||
"coreMemory.enabled": {
|
||||
label: "Core Memory",
|
||||
help: "Inject core memories as virtual MEMORY.md at session start (replaces MEMORY.md file)",
|
||||
},
|
||||
"coreMemory.maxEntries": {
|
||||
label: "Max Core Entries",
|
||||
placeholder: "50",
|
||||
advanced: true,
|
||||
placeholder: String(DEFAULT_CAPTURE_MAX_CHARS),
|
||||
help: "Maximum number of core memories to load",
|
||||
},
|
||||
"coreMemory.minImportance": {
|
||||
label: "Min Core Importance",
|
||||
placeholder: "0.5",
|
||||
advanced: true,
|
||||
help: "Minimum importance threshold for core memories (0-1)",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
102
extensions/memory-lancedb/export-memories.mjs
Normal file
102
extensions/memory-lancedb/export-memories.mjs
Normal file
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Export memories from LanceDB for migration to memory-neo4j
|
||||
*
|
||||
* Usage:
|
||||
* pnpm exec node export-memories.mjs [output-file.json]
|
||||
*
|
||||
* Default output: memories-export.json
|
||||
*/
|
||||
|
||||
import * as lancedb from "@lancedb/lancedb";
|
||||
import { writeFileSync } from "fs";
|
||||
|
||||
const LANCEDB_PATH = process.env.LANCEDB_PATH || "/home/tsukhani/.openclaw/memory/lancedb";
|
||||
const AGENT_ID = process.env.AGENT_ID || "main";
|
||||
const outputFile = process.argv[2] || "memories-export.json";
|
||||
|
||||
console.log("📦 Memory Export Tool (LanceDB)");
|
||||
console.log(` LanceDB path: ${LANCEDB_PATH}`);
|
||||
console.log(` Output: ${outputFile}`);
|
||||
console.log("");
|
||||
|
||||
// Transform for neo4j format
|
||||
function transformMemory(lanceEntry) {
|
||||
const createdAtISO = new Date(lanceEntry.createdAt).toISOString();
|
||||
|
||||
return {
|
||||
id: lanceEntry.id,
|
||||
text: lanceEntry.text,
|
||||
embedding: lanceEntry.vector,
|
||||
importance: lanceEntry.importance,
|
||||
category: lanceEntry.category,
|
||||
createdAt: createdAtISO,
|
||||
updatedAt: createdAtISO,
|
||||
source: "import",
|
||||
extractionStatus: "skipped",
|
||||
agentId: AGENT_ID,
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// Load from LanceDB
|
||||
console.log("📥 Loading from LanceDB...");
|
||||
const db = await lancedb.connect(LANCEDB_PATH);
|
||||
const table = await db.openTable("memories");
|
||||
const count = await table.countRows();
|
||||
console.log(` Found ${count} memories`);
|
||||
|
||||
const memories = await table
|
||||
.query()
|
||||
.limit(count + 100)
|
||||
.toArray();
|
||||
console.log(` Loaded ${memories.length} memories`);
|
||||
|
||||
// Transform
|
||||
console.log("🔄 Transforming...");
|
||||
const transformed = memories.map(transformMemory);
|
||||
|
||||
// Stats
|
||||
const stats = {};
|
||||
transformed.forEach((m) => {
|
||||
stats[m.category] = (stats[m.category] || 0) + 1;
|
||||
});
|
||||
console.log(" Categories:", stats);
|
||||
|
||||
// Export
|
||||
console.log(`📤 Exporting to ${outputFile}...`);
|
||||
const exportData = {
|
||||
exportedAt: new Date().toISOString(),
|
||||
sourcePlugin: "memory-lancedb",
|
||||
targetPlugin: "memory-neo4j",
|
||||
agentId: AGENT_ID,
|
||||
vectorDim: transformed[0]?.embedding?.length || 1536,
|
||||
count: transformed.length,
|
||||
stats,
|
||||
memories: transformed,
|
||||
};
|
||||
|
||||
writeFileSync(outputFile, JSON.stringify(exportData, null, 2));
|
||||
|
||||
// Also write a preview without embeddings
|
||||
const previewFile = outputFile.replace(".json", "-preview.json");
|
||||
const preview = {
|
||||
...exportData,
|
||||
memories: transformed.map((m) => ({
|
||||
...m,
|
||||
embedding: `[${m.embedding?.length} dims]`,
|
||||
})),
|
||||
};
|
||||
writeFileSync(previewFile, JSON.stringify(preview, null, 2));
|
||||
|
||||
console.log(`✅ Exported ${transformed.length} memories`);
|
||||
console.log(
|
||||
` Full export: ${outputFile} (${(JSON.stringify(exportData).length / 1024 / 1024).toFixed(2)} MB)`,
|
||||
);
|
||||
console.log(` Preview: ${previewFile}`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Error:", err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
26
extensions/memory-lancedb/inspect.mjs
Normal file
26
extensions/memory-lancedb/inspect.mjs
Normal file
@@ -0,0 +1,26 @@
|
||||
import * as lancedb from "@lancedb/lancedb";
|
||||
|
||||
const db = await lancedb.connect("/home/tsukhani/.openclaw/memory/lancedb");
|
||||
const tables = await db.tableNames();
|
||||
console.log("Tables:", tables);
|
||||
|
||||
if (tables.includes("memories")) {
|
||||
const table = await db.openTable("memories");
|
||||
const count = await table.countRows();
|
||||
console.log("Memory count:", count);
|
||||
|
||||
const all = await table.query().limit(200).toArray();
|
||||
|
||||
const stats = { preference: 0, fact: 0, decision: 0, entity: 0, other: 0, core: 0 };
|
||||
|
||||
all.forEach((e) => {
|
||||
stats[e.category] = (stats[e.category] || 0) + 1;
|
||||
});
|
||||
|
||||
console.log("\nCategory breakdown:", stats);
|
||||
console.log("\nSample entries:");
|
||||
all.slice(0, 5).forEach((e, i) => {
|
||||
console.log(`${i + 1}. [${e.category}] ${(e.text || "").substring(0, 100)}...`);
|
||||
console.log(` id: ${e.id}, importance: ${e.importance}, vectorDim: ${e.vector?.length}`);
|
||||
});
|
||||
}
|
||||
@@ -26,11 +26,21 @@
|
||||
"label": "Auto-Recall",
|
||||
"help": "Automatically inject relevant memories into context"
|
||||
},
|
||||
"captureMaxChars": {
|
||||
"label": "Capture Max Chars",
|
||||
"help": "Maximum message length eligible for auto-capture",
|
||||
"coreMemory.enabled": {
|
||||
"label": "Core Memory",
|
||||
"help": "Inject core memories as virtual MEMORY.md at session start (replaces MEMORY.md file)"
|
||||
},
|
||||
"coreMemory.maxEntries": {
|
||||
"label": "Max Core Entries",
|
||||
"placeholder": "50",
|
||||
"advanced": true,
|
||||
"placeholder": "500"
|
||||
"help": "Maximum number of core memories to load"
|
||||
},
|
||||
"coreMemory.minImportance": {
|
||||
"label": "Min Core Importance",
|
||||
"placeholder": "0.5",
|
||||
"advanced": true,
|
||||
"help": "Minimum importance threshold for core memories (0-1)"
|
||||
}
|
||||
},
|
||||
"configSchema": {
|
||||
@@ -60,10 +70,20 @@
|
||||
"autoRecall": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"captureMaxChars": {
|
||||
"type": "number",
|
||||
"minimum": 100,
|
||||
"maximum": 10000
|
||||
"coreMemory": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"maxEntries": {
|
||||
"type": "number"
|
||||
},
|
||||
"minImportance": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["embedding"]
|
||||
|
||||
252
extensions/memory-neo4j/attention-gate.ts
Normal file
252
extensions/memory-neo4j/attention-gate.ts
Normal file
@@ -0,0 +1,252 @@
|
||||
/**
|
||||
* Attention gate — lightweight heuristic filter (phase 1 of memory pipeline).
|
||||
*
|
||||
* Rejects obvious noise without any LLM call, analogous to how the brain's
|
||||
* sensory gating filters out irrelevant stimuli before they enter working
|
||||
* memory. Everything that passes gets stored; the sleep cycle decides what
|
||||
* matters.
|
||||
*/
|
||||
|
||||
const NOISE_PATTERNS = [
|
||||
// Greetings / acknowledgments (exact match, with optional punctuation)
|
||||
/^(hi|hey|hello|yo|sup|ok|okay|sure|thanks|thank you|thx|ty|yep|yup|nope|no|yes|yeah|cool|nice|great|got it|sounds good|perfect|alright|fine|noted|ack|kk|k)\s*[.!?]*$/i,
|
||||
// Two-word affirmations: "ok great", "sounds good", "yes please", etc.
|
||||
/^(ok|okay|yes|yeah|yep|sure|no|nope|alright|right|fine|cool|nice|great)\s+(great|good|sure|thanks|please|ok|fine|cool|yeah|perfect|noted|absolutely|definitely|exactly)\s*[.!?]*$/i,
|
||||
// Deictic: messages that are only pronouns/articles/common verbs — no standalone meaning
|
||||
// e.g. "I need those", "let me do it", "ok let me test it out", "I got it"
|
||||
/^(ok[,.]?\s+)?(i('ll|'m|'d|'ve)?\s+)?(just\s+)?(need|want|got|have|let|let's|let me|give me|send|do|did|try|check|see|look at|test|take|get|go|use)\s+(it|that|this|those|these|them|some|one|the|a|an|me|him|her|us)\s*(out|up|now|then|too|again|later|first|here|there|please)?\s*[.!?]*$/i,
|
||||
// Short acknowledgments with trailing context: "ok, ..." / "yes, ..." when total is brief
|
||||
/^(ok|okay|yes|yeah|yep|sure|no|nope|right|alright|fine|cool|nice|great|perfect)[,.]?\s+.{0,20}$/i,
|
||||
// Conversational filler / noise phrases (standalone, with optional punctuation)
|
||||
/^(hmm+|huh|haha|ha|lol|lmao|rofl|nah|meh|idk|brb|ttyl|omg|wow|whoa|welp|oops|ooh|aah|ugh|bleh|pfft|smh|ikr|tbh|imo|fwiw|np|nvm|nm|wut|wat|wha|heh|tsk|sigh|yay|woo+|boo|dang|darn|geez|gosh|sheesh|oof)\s*[.!?]*$/i,
|
||||
// Single-word or near-empty
|
||||
/^\S{0,3}$/,
|
||||
// Pure emoji
|
||||
/^[\p{Emoji}\s]+$/u,
|
||||
// System/XML markup
|
||||
/^<[a-z-]+>[\s\S]*<\/[a-z-]+>$/i,
|
||||
|
||||
// --- Session reset prompts (from /new and /reset commands) ---
|
||||
/^A new session was started via/i,
|
||||
|
||||
// --- Raw chat messages with channel metadata (autocaptured noise) ---
|
||||
/\[slack message id:/i,
|
||||
/\[message_id:/i,
|
||||
/\[telegram message id:/i,
|
||||
|
||||
// --- System infrastructure messages (never user-generated) ---
|
||||
// Heartbeat prompts
|
||||
/Read HEARTBEAT\.md if it exists/i,
|
||||
// Pre-compaction flush prompts
|
||||
/^Pre-compaction memory flush/i,
|
||||
// System timestamp messages (cron outputs, reminders, exec reports)
|
||||
/^System:\s*\[/i,
|
||||
// Cron job wrappers
|
||||
/^\[cron:[0-9a-f-]+/i,
|
||||
// Gateway restart JSON payloads
|
||||
/^GatewayRestart:\s*\{/i,
|
||||
// Background task completion reports
|
||||
/^\[\w{3}\s+\d{4}-\d{2}-\d{2}\s.*\]\s*A background task/i,
|
||||
|
||||
// --- Conversation metadata that survived stripping ---
|
||||
/^Conversation info\s*\(/i,
|
||||
/^\[Queued messages/i,
|
||||
|
||||
// --- Cron delivery outputs & scheduled reminders ---
|
||||
// Scheduled reminder injection text (appears mid-message)
|
||||
/A scheduled reminder has been triggered/i,
|
||||
// Cron delivery instruction to agent (summarize for user)
|
||||
/Summarize this naturally for the user/i,
|
||||
// Relay instruction from cron announcements
|
||||
/Please relay this reminder to the user/i,
|
||||
// Subagent completion announcements (date-stamped)
|
||||
/^\[.*\d{4}-\d{2}-\d{2}.*\]\s*A sub-?agent task/i,
|
||||
// Formatted urgency/priority reports (email summaries, briefings)
|
||||
/(\*\*)?🔴\s*(URGENT|Priority)/i,
|
||||
// Subagent findings header
|
||||
/^Findings:\s*$/im,
|
||||
// "Stats:" lines from subagent completions
|
||||
/^Stats:\s*runtime\s/im,
|
||||
];
|
||||
|
||||
/** Maximum message length — code dumps, logs, etc. are not memories. */
|
||||
const MAX_CAPTURE_CHARS = 2000;
|
||||
|
||||
/** Minimum message length — too short to be meaningful. */
|
||||
const MIN_CAPTURE_CHARS = 30;
|
||||
|
||||
/** Minimum word count — short contextual phrases lack standalone meaning. */
|
||||
const MIN_WORD_COUNT = 8;
|
||||
|
||||
/** Shared checks applied by both user and assistant attention gates. */
|
||||
function failsSharedGateChecks(trimmed: string): boolean {
|
||||
// Injected context from the memory system itself
|
||||
if (trimmed.includes("<relevant-memories>") || trimmed.includes("<core-memory-refresh>")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Noise patterns
|
||||
if (NOISE_PATTERNS.some((r) => r.test(trimmed))) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Excessive emoji (likely reaction, not substance)
|
||||
const emojiCount = (
|
||||
trimmed.match(/[\u{1F300}-\u{1F9FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}\u{1FA00}-\u{1FAFF}]/gu) ||
|
||||
[]
|
||||
).length;
|
||||
if (emojiCount > 3) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export function passesAttentionGate(text: string): boolean {
|
||||
const trimmed = text.trim();
|
||||
|
||||
// Length bounds
|
||||
if (trimmed.length < MIN_CAPTURE_CHARS || trimmed.length > MAX_CAPTURE_CHARS) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Word count — short phrases ("I need those") lack context for recall
|
||||
const wordCount = trimmed.split(/\s+/).length;
|
||||
if (wordCount < MIN_WORD_COUNT) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (failsSharedGateChecks(trimmed)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Passes gate — retain for short-term storage
|
||||
return true;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Assistant attention gate — stricter filter for assistant messages
|
||||
// ============================================================================
|
||||
|
||||
/** Maximum assistant message length — shorter than user to avoid code dumps. */
|
||||
const MAX_ASSISTANT_CAPTURE_CHARS = 1000;
|
||||
|
||||
/** Minimum word count for assistant messages — higher than user. */
|
||||
const MIN_ASSISTANT_WORD_COUNT = 10;
|
||||
|
||||
/**
|
||||
* Patterns that reject assistant self-narration — play-by-play commentary
|
||||
* that reads like thinking out loud rather than a conclusion or fact.
|
||||
* These are the single biggest source of noise in auto-captured assistant memories.
|
||||
*/
|
||||
const ASSISTANT_NARRATION_PATTERNS = [
|
||||
// "Let me ..." / "Now let me ..." / "I'll ..." action narration
|
||||
/^(ok[,.]?\s+)?(now\s+)?let me\s+(check|look|see|try|run|start|test|read|update|verify|fix|search|process|create|build|set up|examine|investigate|query|fetch|pull|scan|clean|install|download|configure|make|select|click|type|fill|open|close|switch|send|post|submit|edit|change|add|remove|write|save|upload)/i,
|
||||
// "I'll ..." action narration
|
||||
/^I('ll| will)\s+(check|look|see|try|run|start|test|read|update|verify|fix|search|process|create|build|set up|examine|investigate|query|fetch|pull|scan|clean|install|download|configure|execute|help|handle|make|select|click|type|fill|open|close|switch|send|post|submit|edit|change|add|remove|write|save|upload|use|grab|get|do)/i,
|
||||
// "Starting ..." / "Running ..." / "Processing ..." status updates
|
||||
/^(starting|running|processing|checking|fetching|scanning|building|installing|downloading|configuring|executing|loading|updating|filling|selecting|clicking|typing|opening|closing|switching|navigating|uploading|saving|sending|posting|submitting)\s/i,
|
||||
// "Good!" / "Great!" / "Perfect!" / "Done!" as opener followed by narration
|
||||
/^(good|great|perfect|nice|excellent|awesome|done)[!.]?\s+(i |the |now |let |we |that |here)/i,
|
||||
// Progress narration: "Now I have..." / "Now I can see..." / "Now let me..."
|
||||
/^now\s+(i\s+(have|can|need|see|understand)|we\s+(have|can|need)|the\s|on\s)/i,
|
||||
// Step narration: "Step 1:" / "**Step 1:**"
|
||||
/^\*?\*?step\s+\d/i,
|
||||
// Page/section progress narration: "Page 1 done!", "Page 3 — final page!"
|
||||
/^Page\s+\d/i,
|
||||
// Narration of what was found/done: "Found it." / "Found X." / "I see — ..."
|
||||
/^(found it|found the|i see\s*[—–-])/i,
|
||||
// Sub-agent task descriptions (workflow narration)
|
||||
/^\[?(mon|tue|wed|thu|fri|sat|sun)\s+\d{4}-\d{2}-\d{2}/i,
|
||||
// Context compaction self-announcements
|
||||
/^🔄\s*\*?\*?context reset/i,
|
||||
// Filename slug generation prompts (internal tool use)
|
||||
/^based on this conversation,?\s*generate a short/i,
|
||||
|
||||
// --- Conversational filler responses (not knowledge) ---
|
||||
// "I'm here" / "I am here" filler: "I'm here to help", "I am here and listening", etc.
|
||||
/^I('m| am) here\b/i,
|
||||
// Ready-state: "Sure, (just) tell me what you want..."
|
||||
/^Sure[,!.]?\s+(just\s+)?(tell|let)\s+me/i,
|
||||
// Observational UI narration: "I can see the picker", "I can see the button"
|
||||
/^I can see\s/i,
|
||||
// A sub-agent task report (quoted or inline)
|
||||
/^A sub-?agent task\b/i,
|
||||
|
||||
// --- Injected system/voice context (not user knowledge) ---
|
||||
// Voice mode formatting instructions injected into sessions
|
||||
/^\[VOICE\s*(MODE|OUTPUT)/i,
|
||||
/^\[voice[-\s]?context\]/i,
|
||||
// Voice tag prefix
|
||||
/^\[voice\]\s/i,
|
||||
|
||||
// --- Session completion summaries (ephemeral, not long-term knowledge) ---
|
||||
// "Done ✅ ..." completion messages (assistant summarizing what it just did)
|
||||
/^Done\s*[✅✓☑️]\s/i,
|
||||
// "All good" / "All set" wrap-ups
|
||||
/^All (good|set|done)[!.]/i,
|
||||
// "Here's what changed" / "Summary of changes" (session-specific)
|
||||
/^(here'?s\s+(what|the|a)\s+(changed?|summary|breakdown|recap))/i,
|
||||
|
||||
// --- Open proposals / action items (cause rogue actions when recalled) ---
|
||||
// These are dangerous in memory: when auto-recalled, other sessions interpret
|
||||
// them as active instructions and attempt to carry them out.
|
||||
// "Want me to...?" / "Should I...?" / "Shall I...?" / "Would you like me to...?"
|
||||
/want me to\s.+\?/i,
|
||||
/should I\s.+\?/i,
|
||||
/shall I\s.+\?/i,
|
||||
/would you like me to\s.+\?/i,
|
||||
// "Do you want me to...?"
|
||||
/do you want me to\s.+\?/i,
|
||||
// "Can I...?" / "May I...?" assistant proposals
|
||||
/^(can|may) I\s.+\?/i,
|
||||
// "Ready to...?" / "Proceed with...?"
|
||||
/ready to\s.+\?/i,
|
||||
/proceed with\s.+\?/i,
|
||||
];
|
||||
|
||||
export function passesAssistantAttentionGate(text: string): boolean {
|
||||
const trimmed = text.trim();
|
||||
|
||||
// Length bounds (stricter than user)
|
||||
if (trimmed.length < MIN_CAPTURE_CHARS || trimmed.length > MAX_ASSISTANT_CAPTURE_CHARS) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Word count — higher threshold than user messages
|
||||
const wordCount = trimmed.split(/\s+/).length;
|
||||
if (wordCount < MIN_ASSISTANT_WORD_COUNT) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Reject messages that are mostly code (>50% inside triple-backtick fences)
|
||||
const codeBlockRegex = /```[\s\S]*?```/g;
|
||||
let codeChars = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = codeBlockRegex.exec(trimmed)) !== null) {
|
||||
codeChars += match[0].length;
|
||||
}
|
||||
if (codeChars > trimmed.length * 0.5) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Reject messages that are mostly tool output
|
||||
if (
|
||||
trimmed.includes("<tool_result>") ||
|
||||
trimmed.includes("<tool_use>") ||
|
||||
trimmed.includes("<function_call>")
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (failsSharedGateChecks(trimmed)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Assistant-specific narration patterns (play-by-play self-talk)
|
||||
if (ASSISTANT_NARRATION_PATTERNS.some((r) => r.test(trimmed))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
573
extensions/memory-neo4j/auto-capture.test.ts
Normal file
573
extensions/memory-neo4j/auto-capture.test.ts
Normal file
@@ -0,0 +1,573 @@
|
||||
/**
|
||||
* Tests for the auto-capture pipeline: captureMessage and runAutoCapture.
|
||||
*
|
||||
* Tests the embed → dedup → rate → store pipeline including:
|
||||
* - Pre-computed vector usage (batch embedding optimization)
|
||||
* - Exact dedup (≥0.95 score band)
|
||||
* - Semantic dedup (0.75-0.95 score band via LLM)
|
||||
* - Importance pre-screening for assistant messages
|
||||
* - Batch embedding in runAutoCapture
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ExtractionConfig } from "./config.js";
|
||||
import type { Embeddings } from "./embeddings.js";
|
||||
import type { Neo4jMemoryClient } from "./neo4j-client.js";
|
||||
import { _captureMessage as captureMessage, _runAutoCapture as runAutoCapture } from "./index.js";
|
||||
|
||||
// ============================================================================
|
||||
// Mocks
|
||||
// ============================================================================
|
||||
|
||||
const enabledConfig: ExtractionConfig = {
|
||||
enabled: true,
|
||||
apiKey: "test-key",
|
||||
model: "test-model",
|
||||
baseUrl: "https://test.ai/api/v1",
|
||||
temperature: 0.0,
|
||||
maxRetries: 0,
|
||||
};
|
||||
|
||||
const disabledConfig: ExtractionConfig = {
|
||||
...enabledConfig,
|
||||
enabled: false,
|
||||
};
|
||||
|
||||
const mockLogger = {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
};
|
||||
|
||||
function createMockDb(overrides?: Partial<Neo4jMemoryClient>): Neo4jMemoryClient {
|
||||
return {
|
||||
findSimilar: vi.fn().mockResolvedValue([]),
|
||||
storeMemory: vi.fn().mockResolvedValue(undefined),
|
||||
...overrides,
|
||||
} as unknown as Neo4jMemoryClient;
|
||||
}
|
||||
|
||||
function createMockEmbeddings(overrides?: Partial<Embeddings>): Embeddings {
|
||||
return {
|
||||
embed: vi.fn().mockResolvedValue([0.1, 0.2, 0.3]),
|
||||
embedBatch: vi.fn().mockResolvedValue([[0.1, 0.2, 0.3]]),
|
||||
...overrides,
|
||||
} as unknown as Embeddings;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// captureMessage
|
||||
// ============================================================================
|
||||
|
||||
describe("captureMessage", () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
it("should store a new memory when no duplicates exist", async () => {
|
||||
const db = createMockDb();
|
||||
const embeddings = createMockEmbeddings();
|
||||
|
||||
// Mock rateImportance (LLM call via fetch)
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () =>
|
||||
Promise.resolve({
|
||||
choices: [{ message: { content: JSON.stringify({ score: 7 }) } }],
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await captureMessage(
|
||||
"I prefer TypeScript over JavaScript",
|
||||
"auto-capture",
|
||||
0.5,
|
||||
1.0,
|
||||
"test-agent",
|
||||
"session-1",
|
||||
db,
|
||||
embeddings,
|
||||
enabledConfig,
|
||||
mockLogger,
|
||||
);
|
||||
|
||||
expect(result.stored).toBe(true);
|
||||
expect(result.semanticDeduped).toBe(false);
|
||||
expect(db.storeMemory).toHaveBeenCalledOnce();
|
||||
expect(embeddings.embed).toHaveBeenCalledWith("I prefer TypeScript over JavaScript");
|
||||
});
|
||||
|
||||
it("should use pre-computed vector when provided", async () => {
|
||||
const db = createMockDb();
|
||||
const embeddings = createMockEmbeddings();
|
||||
const precomputedVector = [0.5, 0.6, 0.7];
|
||||
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () =>
|
||||
Promise.resolve({
|
||||
choices: [{ message: { content: JSON.stringify({ score: 7 }) } }],
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await captureMessage(
|
||||
"test text",
|
||||
"auto-capture",
|
||||
0.5,
|
||||
1.0,
|
||||
"test-agent",
|
||||
undefined,
|
||||
db,
|
||||
embeddings,
|
||||
enabledConfig,
|
||||
mockLogger,
|
||||
precomputedVector,
|
||||
);
|
||||
|
||||
expect(result.stored).toBe(true);
|
||||
// Should NOT call embed() since pre-computed vector was provided
|
||||
expect(embeddings.embed).not.toHaveBeenCalled();
|
||||
// Should use the pre-computed vector for findSimilar
|
||||
expect(db.findSimilar).toHaveBeenCalledWith(precomputedVector, 0.75, 3, "test-agent");
|
||||
});
|
||||
|
||||
it("should skip storage when exact duplicate found (score >= 0.95)", async () => {
|
||||
const db = createMockDb({
|
||||
findSimilar: vi
|
||||
.fn()
|
||||
.mockResolvedValue([{ id: "existing-1", text: "duplicate text", score: 0.97 }]),
|
||||
});
|
||||
const embeddings = createMockEmbeddings();
|
||||
|
||||
const result = await captureMessage(
|
||||
"duplicate text",
|
||||
"auto-capture",
|
||||
0.5,
|
||||
1.0,
|
||||
"test-agent",
|
||||
undefined,
|
||||
db,
|
||||
embeddings,
|
||||
enabledConfig,
|
||||
mockLogger,
|
||||
);
|
||||
|
||||
expect(result.stored).toBe(false);
|
||||
expect(result.semanticDeduped).toBe(false);
|
||||
expect(db.storeMemory).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should semantic dedup when candidate in 0.75-0.95 band is LLM-confirmed duplicate", async () => {
|
||||
const db = createMockDb({
|
||||
findSimilar: vi
|
||||
.fn()
|
||||
.mockResolvedValue([{ id: "candidate-1", text: "User prefers TypeScript", score: 0.88 }]),
|
||||
});
|
||||
const embeddings = createMockEmbeddings();
|
||||
|
||||
// First call: rateImportance, second call: isSemanticDuplicate
|
||||
let callCount = 0;
|
||||
globalThis.fetch = vi.fn().mockImplementation(() => {
|
||||
callCount++;
|
||||
if (callCount === 1) {
|
||||
// rateImportance response
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
json: () =>
|
||||
Promise.resolve({
|
||||
choices: [{ message: { content: JSON.stringify({ score: 7 }) } }],
|
||||
}),
|
||||
});
|
||||
}
|
||||
// isSemanticDuplicate response
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
json: () =>
|
||||
Promise.resolve({
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
content: JSON.stringify({
|
||||
verdict: "duplicate",
|
||||
reason: "same preference",
|
||||
}),
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
const result = await captureMessage(
|
||||
"I like TypeScript",
|
||||
"auto-capture",
|
||||
0.5,
|
||||
1.0,
|
||||
"test-agent",
|
||||
undefined,
|
||||
db,
|
||||
embeddings,
|
||||
enabledConfig,
|
||||
mockLogger,
|
||||
);
|
||||
|
||||
expect(result.stored).toBe(false);
|
||||
expect(result.semanticDeduped).toBe(true);
|
||||
expect(db.storeMemory).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should skip importance check when extraction is disabled", async () => {
|
||||
const db = createMockDb();
|
||||
const embeddings = createMockEmbeddings();
|
||||
|
||||
// With extraction disabled, rateImportance returns 0.5 fallback,
|
||||
// so the threshold check is skipped entirely
|
||||
const result = await captureMessage(
|
||||
"some text to store",
|
||||
"auto-capture",
|
||||
0.5,
|
||||
1.0,
|
||||
"test-agent",
|
||||
undefined,
|
||||
db,
|
||||
embeddings,
|
||||
disabledConfig,
|
||||
mockLogger,
|
||||
);
|
||||
|
||||
expect(result.stored).toBe(true);
|
||||
expect(db.storeMemory).toHaveBeenCalledOnce();
|
||||
// Verify stored with fallback importance * discount
|
||||
const storeCall = (db.storeMemory as ReturnType<typeof vi.fn>).mock.calls[0][0];
|
||||
expect(storeCall.importance).toBe(0.5); // 0.5 fallback * 1.0 discount
|
||||
expect(storeCall.extractionStatus).toBe("skipped");
|
||||
});
|
||||
|
||||
it("should apply importance discount for assistant messages", async () => {
|
||||
const db = createMockDb();
|
||||
const embeddings = createMockEmbeddings();
|
||||
|
||||
// For assistant messages, importance is rated first
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () =>
|
||||
Promise.resolve({
|
||||
choices: [{ message: { content: JSON.stringify({ score: 8 }) } }],
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await captureMessage(
|
||||
"Here's what I know about Neo4j graph databases...",
|
||||
"auto-capture-assistant",
|
||||
0.8, // higher threshold for assistant
|
||||
0.75, // 25% discount
|
||||
"test-agent",
|
||||
undefined,
|
||||
db,
|
||||
embeddings,
|
||||
enabledConfig,
|
||||
mockLogger,
|
||||
);
|
||||
|
||||
expect(result.stored).toBe(true);
|
||||
const storeCall = (db.storeMemory as ReturnType<typeof vi.fn>).mock.calls[0][0];
|
||||
// importance 0.8 (score 8/10) * 0.75 discount ≈ 0.6
|
||||
expect(storeCall.importance).toBeCloseTo(0.6);
|
||||
expect(storeCall.source).toBe("auto-capture-assistant");
|
||||
});
|
||||
|
||||
it("should reject assistant messages below importance threshold", async () => {
|
||||
const db = createMockDb();
|
||||
const embeddings = createMockEmbeddings();
|
||||
|
||||
// Low importance score
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () =>
|
||||
Promise.resolve({
|
||||
choices: [{ message: { content: JSON.stringify({ score: 3 }) } }],
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await captureMessage(
|
||||
"Sure, I can help with that.",
|
||||
"auto-capture-assistant",
|
||||
0.8, // threshold 0.8
|
||||
0.75,
|
||||
"test-agent",
|
||||
undefined,
|
||||
db,
|
||||
embeddings,
|
||||
enabledConfig,
|
||||
mockLogger,
|
||||
);
|
||||
|
||||
expect(result.stored).toBe(false);
|
||||
// Should not even embed since importance pre-screen failed
|
||||
expect(embeddings.embed).not.toHaveBeenCalled();
|
||||
expect(db.storeMemory).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should reject user messages below importance threshold", async () => {
|
||||
const db = createMockDb();
|
||||
const embeddings = createMockEmbeddings();
|
||||
|
||||
// Low importance score
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () =>
|
||||
Promise.resolve({
|
||||
choices: [{ message: { content: JSON.stringify({ score: 2 }) } }],
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await captureMessage(
|
||||
"okay thanks",
|
||||
"auto-capture",
|
||||
0.5, // threshold 0.5
|
||||
1.0,
|
||||
"test-agent",
|
||||
undefined,
|
||||
db,
|
||||
embeddings,
|
||||
enabledConfig,
|
||||
mockLogger,
|
||||
);
|
||||
|
||||
expect(result.stored).toBe(false);
|
||||
expect(db.storeMemory).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// runAutoCapture
|
||||
// ============================================================================
|
||||
|
||||
describe("runAutoCapture", () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
it("should batch-embed all retained messages at once", async () => {
|
||||
const db = createMockDb();
|
||||
const embedBatchMock = vi.fn().mockResolvedValue([
|
||||
[0.1, 0.2],
|
||||
[0.3, 0.4],
|
||||
]);
|
||||
const embeddings = createMockEmbeddings({ embedBatch: embedBatchMock });
|
||||
|
||||
// Mock rateImportance calls
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () =>
|
||||
Promise.resolve({
|
||||
choices: [{ message: { content: JSON.stringify({ score: 7 }) } }],
|
||||
}),
|
||||
});
|
||||
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content: "I prefer TypeScript over JavaScript for backend development",
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content:
|
||||
"TypeScript is great for type safety and developer experience, especially with Node.js projects",
|
||||
},
|
||||
];
|
||||
|
||||
await runAutoCapture(
|
||||
messages,
|
||||
"test-agent",
|
||||
"session-1",
|
||||
db,
|
||||
embeddings,
|
||||
enabledConfig,
|
||||
mockLogger,
|
||||
);
|
||||
|
||||
// Should call embedBatch once with both texts
|
||||
expect(embedBatchMock).toHaveBeenCalledOnce();
|
||||
const batchTexts = embedBatchMock.mock.calls[0][0];
|
||||
expect(batchTexts.length).toBe(2);
|
||||
});
|
||||
|
||||
it("should not call embedBatch when no messages pass the gate", async () => {
|
||||
const db = createMockDb();
|
||||
const embedBatchMock = vi.fn().mockResolvedValue([]);
|
||||
const embeddings = createMockEmbeddings({ embedBatch: embedBatchMock });
|
||||
|
||||
// Short messages that won't pass attention gate
|
||||
const messages = [
|
||||
{ role: "user", content: "ok" },
|
||||
{ role: "assistant", content: "yes" },
|
||||
];
|
||||
|
||||
await runAutoCapture(
|
||||
messages,
|
||||
"test-agent",
|
||||
"session-1",
|
||||
db,
|
||||
embeddings,
|
||||
enabledConfig,
|
||||
mockLogger,
|
||||
);
|
||||
|
||||
expect(embedBatchMock).not.toHaveBeenCalled();
|
||||
expect(db.storeMemory).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should handle empty messages array", async () => {
|
||||
const db = createMockDb();
|
||||
const embeddings = createMockEmbeddings();
|
||||
|
||||
await runAutoCapture([], "test-agent", undefined, db, embeddings, enabledConfig, mockLogger);
|
||||
|
||||
expect(db.storeMemory).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should continue processing if one message fails", async () => {
|
||||
const db = createMockDb();
|
||||
// First embed call fails, second succeeds
|
||||
let embedCallCount = 0;
|
||||
const findSimilarMock = vi.fn().mockImplementation(() => {
|
||||
embedCallCount++;
|
||||
if (embedCallCount === 1) {
|
||||
return Promise.reject(new Error("DB connection failed"));
|
||||
}
|
||||
return Promise.resolve([]);
|
||||
});
|
||||
const embedBatchMock = vi.fn().mockResolvedValue([
|
||||
[0.1, 0.2],
|
||||
[0.3, 0.4],
|
||||
]);
|
||||
const dbWithError = createMockDb({
|
||||
findSimilar: findSimilarMock,
|
||||
});
|
||||
const embeddings = createMockEmbeddings({ embedBatch: embedBatchMock });
|
||||
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () =>
|
||||
Promise.resolve({
|
||||
choices: [{ message: { content: JSON.stringify({ score: 7 }) } }],
|
||||
}),
|
||||
});
|
||||
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content: "First message that is long enough to pass the attention gate filter",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: "Second message that is also long enough to pass the attention gate",
|
||||
},
|
||||
];
|
||||
|
||||
// Should not throw — errors are caught per-message
|
||||
await runAutoCapture(
|
||||
messages,
|
||||
"test-agent",
|
||||
"session-1",
|
||||
dbWithError,
|
||||
embeddings,
|
||||
enabledConfig,
|
||||
mockLogger,
|
||||
);
|
||||
|
||||
// The second message should still have been attempted
|
||||
expect(findSimilarMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("should use different thresholds for user vs assistant messages", async () => {
|
||||
const db = createMockDb();
|
||||
const storeMemoryMock = vi.fn().mockResolvedValue(undefined);
|
||||
const dbWithStore = createMockDb({ storeMemory: storeMemoryMock });
|
||||
const embedBatchMock = vi.fn().mockResolvedValue([
|
||||
[0.1, 0.2],
|
||||
[0.3, 0.4],
|
||||
]);
|
||||
const embeddings = createMockEmbeddings({ embedBatch: embedBatchMock });
|
||||
|
||||
// Always return high importance so both pass
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () =>
|
||||
Promise.resolve({
|
||||
choices: [{ message: { content: JSON.stringify({ score: 9 }) } }],
|
||||
}),
|
||||
});
|
||||
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content: "I really love working with graph databases like Neo4j for my projects",
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content:
|
||||
"Graph databases like Neo4j excel at modeling connected data and relationship queries",
|
||||
},
|
||||
];
|
||||
|
||||
await runAutoCapture(
|
||||
messages,
|
||||
"test-agent",
|
||||
"session-1",
|
||||
dbWithStore,
|
||||
embeddings,
|
||||
enabledConfig,
|
||||
mockLogger,
|
||||
);
|
||||
|
||||
// Both should be stored
|
||||
const storeCalls = storeMemoryMock.mock.calls;
|
||||
if (storeCalls.length === 2) {
|
||||
// User message: importance * 1.0 discount
|
||||
expect(storeCalls[0][0].source).toBe("auto-capture");
|
||||
// Assistant message: importance * 0.75 discount
|
||||
expect(storeCalls[1][0].source).toBe("auto-capture-assistant");
|
||||
expect(storeCalls[1][0].importance).toBeLessThan(storeCalls[0][0].importance);
|
||||
}
|
||||
});
|
||||
|
||||
it("should log capture errors without throwing", async () => {
|
||||
const embedBatchMock = vi.fn().mockRejectedValue(new Error("embedding service down"));
|
||||
const embeddings = createMockEmbeddings({ embedBatch: embedBatchMock });
|
||||
const db = createMockDb();
|
||||
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content: "A long enough message to pass the attention gate for testing purposes",
|
||||
},
|
||||
];
|
||||
|
||||
// Should not throw
|
||||
await runAutoCapture(
|
||||
messages,
|
||||
"test-agent",
|
||||
"session-1",
|
||||
db,
|
||||
embeddings,
|
||||
enabledConfig,
|
||||
mockLogger,
|
||||
);
|
||||
|
||||
// Should have logged the error
|
||||
expect(mockLogger.warn).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
817
extensions/memory-neo4j/cli.ts
Normal file
817
extensions/memory-neo4j/cli.ts
Normal file
@@ -0,0 +1,817 @@
|
||||
/**
|
||||
* CLI command registration for memory-neo4j.
|
||||
*
|
||||
* Registers the `openclaw memory neo4j` subcommand group with commands:
|
||||
* - list: List memory counts by agent and category
|
||||
* - search: Search memories via hybrid search
|
||||
* - stats: Show memory statistics and configuration
|
||||
* - sleep: Run sleep cycle (six-phase memory consolidation)
|
||||
* - index: Re-embed all memories after changing embedding model
|
||||
* - cleanup: Retroactively apply attention gate to stored memories
|
||||
*/
|
||||
|
||||
import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
|
||||
import type { ExtractionConfig, MemoryNeo4jConfig } from "./config.js";
|
||||
import type { Embeddings } from "./embeddings.js";
|
||||
import type { Neo4jMemoryClient } from "./neo4j-client.js";
|
||||
import { passesAttentionGate } from "./attention-gate.js";
|
||||
import { stripMessageWrappers } from "./message-utils.js";
|
||||
import { hybridSearch } from "./search.js";
|
||||
import { runSleepCycle } from "./sleep-cycle.js";
|
||||
|
||||
export type CliDeps = {
|
||||
db: Neo4jMemoryClient;
|
||||
embeddings: Embeddings;
|
||||
cfg: MemoryNeo4jConfig;
|
||||
extractionConfig: ExtractionConfig;
|
||||
vectorDim: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Register the `openclaw memory neo4j` CLI subcommand group.
|
||||
*/
|
||||
export function registerCli(api: OpenClawPluginApi, deps: CliDeps): void {
|
||||
const { db, embeddings, cfg, extractionConfig, vectorDim } = deps;
|
||||
|
||||
api.registerCli(
|
||||
({ program }) => {
|
||||
// Find existing memory command or create fallback
|
||||
let memoryCmd = program.commands.find((cmd) => cmd.name() === "memory");
|
||||
if (!memoryCmd) {
|
||||
// Fallback if core memory CLI not registered yet
|
||||
memoryCmd = program.command("memory").description("Memory commands");
|
||||
}
|
||||
|
||||
// Add neo4j memory subcommand group
|
||||
const memory = memoryCmd.command("neo4j").description("Neo4j graph memory commands");
|
||||
|
||||
memory
|
||||
.command("list")
|
||||
.description("List memories grouped by agent and category")
|
||||
.option("--agent <id>", "Filter by agent id")
|
||||
.option("--category <name>", "Filter by category")
|
||||
.option("--limit <n>", "Max memories per category (default: 20)")
|
||||
.option("--json", "Output as JSON")
|
||||
.action(
|
||||
async (opts: { agent?: string; category?: string; limit?: string; json?: boolean }) => {
|
||||
try {
|
||||
await db.ensureInitialized();
|
||||
const perCategoryLimit = opts.limit ? parseInt(opts.limit, 10) : 20;
|
||||
if (Number.isNaN(perCategoryLimit) || perCategoryLimit <= 0) {
|
||||
console.error("Error: --limit must be greater than 0");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// Build query with optional filters
|
||||
const conditions: string[] = [];
|
||||
const params: Record<string, unknown> = {};
|
||||
if (opts.agent) {
|
||||
conditions.push("m.agentId = $agentId");
|
||||
params.agentId = opts.agent;
|
||||
}
|
||||
if (opts.category) {
|
||||
conditions.push("m.category = $category");
|
||||
params.category = opts.category;
|
||||
}
|
||||
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
||||
|
||||
const rows = await db.runQuery<{
|
||||
agentId: string;
|
||||
category: string;
|
||||
id: string;
|
||||
text: string;
|
||||
importance: number;
|
||||
createdAt: string;
|
||||
source: string;
|
||||
}>(
|
||||
`MATCH (m:Memory) ${where}
|
||||
WITH m.agentId AS agentId, m.category AS category, m
|
||||
ORDER BY m.importance DESC
|
||||
WITH agentId, category, collect({
|
||||
id: m.id, text: m.text, importance: m.importance,
|
||||
createdAt: m.createdAt, source: coalesce(m.source, 'unknown')
|
||||
}) AS memories
|
||||
UNWIND memories[0..${perCategoryLimit}] AS mem
|
||||
RETURN agentId, category,
|
||||
mem.id AS id, mem.text AS text,
|
||||
mem.importance AS importance,
|
||||
mem.createdAt AS createdAt,
|
||||
mem.source AS source
|
||||
ORDER BY agentId, category, importance DESC`,
|
||||
params,
|
||||
);
|
||||
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(rows, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
if (rows.length === 0) {
|
||||
console.log("No memories found.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Group by agent → category → memories
|
||||
const byAgent = new Map<
|
||||
string,
|
||||
Map<
|
||||
string,
|
||||
Array<{
|
||||
id: string;
|
||||
text: string;
|
||||
importance: number;
|
||||
createdAt: string;
|
||||
source: string;
|
||||
}>
|
||||
>
|
||||
>();
|
||||
for (const row of rows) {
|
||||
const agent = (row.agentId as string) ?? "default";
|
||||
const cat = (row.category as string) ?? "other";
|
||||
if (!byAgent.has(agent)) byAgent.set(agent, new Map());
|
||||
const catMap = byAgent.get(agent)!;
|
||||
if (!catMap.has(cat)) catMap.set(cat, []);
|
||||
catMap.get(cat)!.push({
|
||||
id: row.id as string,
|
||||
text: row.text as string,
|
||||
importance: row.importance as number,
|
||||
createdAt: row.createdAt as string,
|
||||
source: row.source as string,
|
||||
});
|
||||
}
|
||||
|
||||
const impBar = (ratio: number) => {
|
||||
const W = 10;
|
||||
const filled = Math.round(ratio * W);
|
||||
return "█".repeat(filled) + "░".repeat(W - filled);
|
||||
};
|
||||
|
||||
for (const [agentId, categories] of byAgent) {
|
||||
const agentTotal = [...categories.values()].reduce((s, m) => s + m.length, 0);
|
||||
console.log(`\n┌─ ${agentId} (${agentTotal} shown)`);
|
||||
|
||||
for (const [category, memories] of categories) {
|
||||
console.log(`│\n│ ── ${category} (${memories.length}) ──`);
|
||||
for (const mem of memories) {
|
||||
const pct = ((mem.importance * 100).toFixed(0) + "%").padStart(4);
|
||||
const preview = mem.text.length > 72 ? `${mem.text.slice(0, 69)}...` : mem.text;
|
||||
console.log(`│ ${impBar(mem.importance)} ${pct} ${preview}`);
|
||||
}
|
||||
}
|
||||
console.log("└");
|
||||
}
|
||||
console.log("");
|
||||
} catch (err) {
|
||||
console.error(`Error: ${err instanceof Error ? err.message : String(err)}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
memory
|
||||
.command("search")
|
||||
.description("Search memories")
|
||||
.argument("<query>", "Search query")
|
||||
.option("--limit <n>", "Max results", "5")
|
||||
.option("--agent <id>", "Agent id (default: default)")
|
||||
.action(async (query: string, opts: { limit: string; agent?: string }) => {
|
||||
try {
|
||||
const results = await hybridSearch(
|
||||
db,
|
||||
embeddings,
|
||||
query,
|
||||
parseInt(opts.limit, 10),
|
||||
opts.agent ?? "default",
|
||||
extractionConfig.enabled,
|
||||
{ graphSearchDepth: cfg.graphSearchDepth },
|
||||
);
|
||||
const output = results.map((r) => ({
|
||||
id: r.id,
|
||||
text: r.text,
|
||||
category: r.category,
|
||||
importance: r.importance,
|
||||
score: r.score,
|
||||
}));
|
||||
console.log(JSON.stringify(output, null, 2));
|
||||
} catch (err) {
|
||||
console.error(`Error: ${err instanceof Error ? err.message : String(err)}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
});
|
||||
|
||||
memory
|
||||
.command("stats")
|
||||
.description("Show memory statistics and configuration")
|
||||
.action(async () => {
|
||||
try {
|
||||
await db.ensureInitialized();
|
||||
const stats = await db.getMemoryStats();
|
||||
const total = stats.reduce((sum, s) => sum + s.count, 0);
|
||||
|
||||
console.log("\nMemory (Neo4j) Statistics");
|
||||
console.log("─────────────────────────");
|
||||
console.log(`Total memories: ${total}`);
|
||||
console.log(`Neo4j URI: ${cfg.neo4j.uri}`);
|
||||
console.log(`Embedding: ${cfg.embedding.provider}/${cfg.embedding.model}`);
|
||||
console.log(
|
||||
`Extraction: ${extractionConfig.enabled ? extractionConfig.model : "disabled"}`,
|
||||
);
|
||||
console.log(`Auto-capture: ${cfg.autoCapture ? "enabled" : "disabled"}`);
|
||||
console.log(`Auto-recall: ${cfg.autoRecall ? "enabled" : "disabled"}`);
|
||||
console.log(`Core memory: ${cfg.coreMemory.enabled ? "enabled" : "disabled"}`);
|
||||
|
||||
if (stats.length > 0) {
|
||||
const BAR_WIDTH = 20;
|
||||
const bar = (ratio: number) => {
|
||||
const filled = Math.round(ratio * BAR_WIDTH);
|
||||
return "█".repeat(filled) + "░".repeat(BAR_WIDTH - filled);
|
||||
};
|
||||
|
||||
// Group by agentId
|
||||
const byAgent = new Map<
|
||||
string,
|
||||
Array<{ category: string; count: number; avgImportance: number }>
|
||||
>();
|
||||
for (const row of stats) {
|
||||
const list = byAgent.get(row.agentId) || [];
|
||||
list.push({
|
||||
category: row.category,
|
||||
count: row.count,
|
||||
avgImportance: row.avgImportance,
|
||||
});
|
||||
byAgent.set(row.agentId, list);
|
||||
}
|
||||
|
||||
for (const [agentId, categories] of byAgent) {
|
||||
const agentTotal = categories.reduce((sum, c) => sum + c.count, 0);
|
||||
const maxCatCount = Math.max(...categories.map((c) => c.count));
|
||||
const catLabelLen = Math.max(...categories.map((c) => c.category.length));
|
||||
|
||||
console.log(`\n┌─ ${agentId} (${agentTotal} memories)`);
|
||||
console.log("│");
|
||||
console.log(
|
||||
`│ ${"Category".padEnd(catLabelLen)} ${"Count".padStart(5)} ${"".padEnd(BAR_WIDTH)} ${"Importance".padStart(10)}`,
|
||||
);
|
||||
console.log(`│ ${"─".repeat(catLabelLen + 5 + BAR_WIDTH * 2 + 18)}`);
|
||||
for (const { category, count, avgImportance } of categories) {
|
||||
const cat = category.padEnd(catLabelLen);
|
||||
const cnt = String(count).padStart(5);
|
||||
const pct = ((avgImportance * 100).toFixed(0) + "%").padStart(10);
|
||||
console.log(
|
||||
`│ ${cat} ${cnt} ${bar(count / maxCatCount)} ${pct} ${bar(avgImportance)}`,
|
||||
);
|
||||
}
|
||||
console.log("└");
|
||||
}
|
||||
|
||||
console.log(`\nAgents: ${byAgent.size} (${[...byAgent.keys()].join(", ")})`);
|
||||
}
|
||||
console.log("");
|
||||
} catch (err) {
|
||||
console.error(`Error: ${err instanceof Error ? err.message : String(err)}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
});
|
||||
|
||||
memory
|
||||
.command("sleep")
|
||||
.description("Run sleep cycle — consolidate memories")
|
||||
.option("--agent <id>", "Agent id (default: all agents)")
|
||||
.option("--dedup-threshold <n>", "Vector similarity threshold for dedup (default: 0.95)")
|
||||
.option("--decay-threshold <n>", "Decay score threshold for pruning (default: 0.1)")
|
||||
.option("--decay-half-life <days>", "Base half-life in days (default: 30)")
|
||||
.option("--batch-size <n>", "Extraction batch size (default: 50)")
|
||||
.option("--delay <ms>", "Delay between extraction batches in ms (default: 1000)")
|
||||
.option("--max-semantic-pairs <n>", "Max LLM-checked semantic dedup pairs (default: 500)")
|
||||
.option("--concurrency <n>", "Parallel LLM calls — match OLLAMA_NUM_PARALLEL (default: 8)")
|
||||
.option(
|
||||
"--skip-semantic",
|
||||
"Skip LLM-based semantic dedup (Phase 1b) and conflict detection (Phase 1c)",
|
||||
)
|
||||
.option("--workspace <dir>", "Workspace directory for TASKS.md cleanup")
|
||||
.option("--report", "Show quality metrics after sleep cycle completes")
|
||||
.action(
|
||||
async (opts: {
|
||||
agent?: string;
|
||||
dedupThreshold?: string;
|
||||
decayThreshold?: string;
|
||||
decayHalfLife?: string;
|
||||
batchSize?: string;
|
||||
delay?: string;
|
||||
maxSemanticPairs?: string;
|
||||
concurrency?: string;
|
||||
skipSemantic?: boolean;
|
||||
workspace?: string;
|
||||
report?: boolean;
|
||||
}) => {
|
||||
console.log("\n🌙 Memory Sleep Cycle");
|
||||
console.log("═════════════════════════════════════════════════════════════");
|
||||
console.log("Multi-phase memory consolidation:\n");
|
||||
console.log(" Phase 1: Deduplication — Merge near-duplicate memories");
|
||||
console.log(
|
||||
" Phase 1b: Semantic Dedup — LLM-based paraphrase detection (0.75–0.95 band)",
|
||||
);
|
||||
console.log(" Phase 1c: Conflict Detection — Resolve contradictory memories");
|
||||
console.log(" Phase 1d: Entity Dedup — Merge duplicate entity nodes");
|
||||
console.log(" Phase 2: Extraction — Extract entities and categorize");
|
||||
console.log(" Phase 2b: Retroactive Tagging — Tag memories missing topic tags");
|
||||
console.log(" Phase 3: Decay & Pruning — Remove stale low-importance memories");
|
||||
console.log(" Phase 4: Orphan Cleanup — Remove disconnected nodes");
|
||||
console.log(" Phase 5: Noise Cleanup — Remove dangerous pattern memories");
|
||||
console.log(" Phase 5b: Credential Scan — Remove memories with leaked secrets");
|
||||
console.log(" Phase 6: Task Ledger Cleanup — Archive stale tasks in TASKS.md\n");
|
||||
|
||||
try {
|
||||
// Validate sleep cycle CLI parameters before running
|
||||
const batchSize = opts.batchSize ? parseInt(opts.batchSize, 10) : undefined;
|
||||
const delay = opts.delay ? parseInt(opts.delay, 10) : undefined;
|
||||
const decayHalfLife = opts.decayHalfLife
|
||||
? parseInt(opts.decayHalfLife, 10)
|
||||
: undefined;
|
||||
const decayThreshold = opts.decayThreshold
|
||||
? parseFloat(opts.decayThreshold)
|
||||
: undefined;
|
||||
|
||||
if (batchSize != null && (Number.isNaN(batchSize) || batchSize <= 0)) {
|
||||
console.error("Error: --batch-size must be greater than 0");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
if (delay != null && (Number.isNaN(delay) || delay < 0)) {
|
||||
console.error("Error: --delay must be >= 0");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
if (decayHalfLife != null && (Number.isNaN(decayHalfLife) || decayHalfLife <= 0)) {
|
||||
console.error("Error: --decay-half-life must be greater than 0");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
if (
|
||||
decayThreshold != null &&
|
||||
(Number.isNaN(decayThreshold) || decayThreshold < 0 || decayThreshold > 1)
|
||||
) {
|
||||
console.error("Error: --decay-threshold must be between 0 and 1");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const maxSemanticPairs = opts.maxSemanticPairs
|
||||
? parseInt(opts.maxSemanticPairs, 10)
|
||||
: undefined;
|
||||
if (
|
||||
maxSemanticPairs != null &&
|
||||
(Number.isNaN(maxSemanticPairs) || maxSemanticPairs <= 0)
|
||||
) {
|
||||
console.error("Error: --max-semantic-pairs must be greater than 0");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const concurrency = opts.concurrency ? parseInt(opts.concurrency, 10) : undefined;
|
||||
if (concurrency != null && (Number.isNaN(concurrency) || concurrency <= 0)) {
|
||||
console.error("Error: --concurrency must be greater than 0");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
await db.ensureInitialized();
|
||||
|
||||
// Resolve workspace dir for task ledger cleanup
|
||||
const resolvedWorkspace = opts.workspace?.trim() || undefined;
|
||||
|
||||
const result = await runSleepCycle(db, embeddings, extractionConfig, api.logger, {
|
||||
agentId: opts.agent,
|
||||
dedupThreshold: opts.dedupThreshold ? parseFloat(opts.dedupThreshold) : undefined,
|
||||
skipSemanticDedup: opts.skipSemantic === true,
|
||||
maxSemanticDedupPairs: maxSemanticPairs,
|
||||
llmConcurrency: concurrency,
|
||||
decayRetentionThreshold: decayThreshold,
|
||||
decayBaseHalfLifeDays: decayHalfLife,
|
||||
decayCurves: Object.keys(cfg.decayCurves).length > 0 ? cfg.decayCurves : undefined,
|
||||
extractionBatchSize: batchSize,
|
||||
extractionDelayMs: delay,
|
||||
workspaceDir: resolvedWorkspace,
|
||||
onPhaseStart: (phase) => {
|
||||
const phaseNames: Record<string, string> = {
|
||||
dedup: "Phase 1: Deduplication",
|
||||
semanticDedup: "Phase 1b: Semantic Deduplication",
|
||||
conflict: "Phase 1c: Conflict Detection",
|
||||
entityDedup: "Phase 1d: Entity Deduplication",
|
||||
extraction: "Phase 2: Extraction",
|
||||
retroactiveTagging: "Phase 2b: Retroactive Tagging",
|
||||
decay: "Phase 3: Decay & Pruning",
|
||||
cleanup: "Phase 4: Orphan Cleanup",
|
||||
noiseCleanup: "Phase 5: Noise Cleanup",
|
||||
credentialScan: "Phase 5b: Credential Scan",
|
||||
taskLedger: "Phase 6: Task Ledger Cleanup",
|
||||
};
|
||||
console.log(`\n▶ ${phaseNames[phase] ?? phase}`);
|
||||
console.log("─────────────────────────────────────────────────────────────");
|
||||
},
|
||||
onProgress: (_phase, message) => {
|
||||
console.log(` ${message}`);
|
||||
},
|
||||
});
|
||||
|
||||
console.log("\n═════════════════════════════════════════════════════════════");
|
||||
console.log(`✅ Sleep cycle complete in ${(result.durationMs / 1000).toFixed(1)}s`);
|
||||
console.log("─────────────────────────────────────────────────────────────");
|
||||
console.log(
|
||||
` Deduplication: ${result.dedup.clustersFound} clusters → ${result.dedup.memoriesMerged} merged`,
|
||||
);
|
||||
console.log(
|
||||
` Conflicts: ${result.conflict.pairsFound} pairs, ${result.conflict.resolved} resolved, ${result.conflict.invalidated} invalidated`,
|
||||
);
|
||||
console.log(
|
||||
` Semantic Dedup: ${result.semanticDedup.pairsChecked} pairs checked, ${result.semanticDedup.duplicatesMerged} merged`,
|
||||
);
|
||||
console.log(` Decay/Pruning: ${result.decay.memoriesPruned} memories pruned`);
|
||||
console.log(
|
||||
` Extraction: ${result.extraction.succeeded}/${result.extraction.total} extracted` +
|
||||
(result.extraction.failed > 0 ? ` (${result.extraction.failed} failed)` : ""),
|
||||
);
|
||||
console.log(
|
||||
` Retro-Tagging: ${result.retroactiveTagging.tagged}/${result.retroactiveTagging.total} tagged` +
|
||||
(result.retroactiveTagging.failed > 0
|
||||
? ` (${result.retroactiveTagging.failed} failed)`
|
||||
: ""),
|
||||
);
|
||||
console.log(
|
||||
` Cleanup: ${result.cleanup.entitiesRemoved} entities, ${result.cleanup.tagsRemoved} tags removed`,
|
||||
);
|
||||
console.log(
|
||||
` Task Ledger: ${result.taskLedger.archivedCount} stale tasks archived` +
|
||||
(result.taskLedger.archivedIds.length > 0
|
||||
? ` (${result.taskLedger.archivedIds.join(", ")})`
|
||||
: ""),
|
||||
);
|
||||
if (result.aborted) {
|
||||
console.log("\n⚠️ Sleep cycle was aborted before completion.");
|
||||
}
|
||||
|
||||
// Quality report (optional)
|
||||
if (opts.report) {
|
||||
console.log("\n═════════════════════════════════════════════════════════════");
|
||||
console.log("📊 Quality Report");
|
||||
console.log("─────────────────────────────────────────────────────────────");
|
||||
|
||||
try {
|
||||
// Extraction coverage
|
||||
const statusCounts = await db.countByExtractionStatus(opts.agent);
|
||||
const totalMems =
|
||||
statusCounts.pending +
|
||||
statusCounts.complete +
|
||||
statusCounts.failed +
|
||||
statusCounts.skipped;
|
||||
const coveragePct =
|
||||
totalMems > 0 ? ((statusCounts.complete / totalMems) * 100).toFixed(1) : "0.0";
|
||||
console.log(
|
||||
`\n Extraction Coverage: ${coveragePct}% (${statusCounts.complete}/${totalMems})`,
|
||||
);
|
||||
console.log(
|
||||
` pending=${statusCounts.pending} complete=${statusCounts.complete} failed=${statusCounts.failed} skipped=${statusCounts.skipped}`,
|
||||
);
|
||||
|
||||
// Entity graph stats
|
||||
const graphStats = await db.getEntityGraphStats(opts.agent);
|
||||
console.log(`\n Entity Graph:`);
|
||||
console.log(
|
||||
` Entities: ${graphStats.entityCount} Mentions: ${graphStats.mentionCount} Density: ${graphStats.density.toFixed(2)}`,
|
||||
);
|
||||
|
||||
// Decay distribution
|
||||
const decayDist = await db.getDecayDistribution(opts.agent);
|
||||
if (decayDist.length > 0) {
|
||||
const maxCount = Math.max(...decayDist.map((d) => d.count));
|
||||
const BAR_W = 20;
|
||||
console.log(`\n Decay Distribution:`);
|
||||
for (const { bucket, count } of decayDist) {
|
||||
const filled = maxCount > 0 ? Math.round((count / maxCount) * BAR_W) : 0;
|
||||
const bar = "█".repeat(filled) + "░".repeat(BAR_W - filled);
|
||||
console.log(` ${bucket.padEnd(13)} ${bar} ${count}`);
|
||||
}
|
||||
}
|
||||
} catch (reportErr) {
|
||||
console.log(`\n ⚠️ Could not generate quality report: ${String(reportErr)}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("");
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`\n❌ Sleep cycle failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
memory
|
||||
.command("index")
|
||||
.description(
|
||||
"Re-embed all memories and entities — use after changing embedding model/provider",
|
||||
)
|
||||
.option("--batch-size <n>", "Embedding batch size (default: 50)")
|
||||
.action(async (opts: { batchSize?: string }) => {
|
||||
const batchSize = opts.batchSize ? parseInt(opts.batchSize, 10) : 50;
|
||||
if (Number.isNaN(batchSize) || batchSize <= 0) {
|
||||
console.error("Error: --batch-size must be greater than 0");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("\nMemory Neo4j — Reindex Embeddings");
|
||||
console.log("═════════════════════════════════════════════════════════════");
|
||||
console.log(`Model: ${cfg.embedding.provider}/${cfg.embedding.model}`);
|
||||
console.log(`Dimensions: ${vectorDim}`);
|
||||
console.log(`Batch size: ${batchSize}\n`);
|
||||
|
||||
try {
|
||||
const startedAt = Date.now();
|
||||
const result = await db.reindex((texts) => embeddings.embedBatch(texts), {
|
||||
batchSize,
|
||||
onProgress: (phase, done, total) => {
|
||||
if (phase === "drop-indexes" && done === 0) {
|
||||
console.log("▶ Dropping old vector index…");
|
||||
} else if (phase === "memories") {
|
||||
console.log(` Memories: ${done}/${total}`);
|
||||
} else if (phase === "create-indexes" && done === 0) {
|
||||
console.log("▶ Recreating vector index…");
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const elapsed = ((Date.now() - startedAt) / 1000).toFixed(1);
|
||||
console.log("\n═════════════════════════════════════════════════════════════");
|
||||
console.log(`✅ Reindex complete in ${elapsed}s — ${result.memories} memories`);
|
||||
console.log("");
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`\n❌ Reindex failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
});
|
||||
|
||||
memory
|
||||
.command("cleanup")
|
||||
.description(
|
||||
"Retroactively apply the attention gate — find and remove low-substance memories",
|
||||
)
|
||||
.option("--execute", "Actually delete (default: dry-run preview)")
|
||||
.option("--all", "Include explicitly-stored memories (default: auto-capture only)")
|
||||
.option("--agent <id>", "Only clean up memories for a specific agent")
|
||||
.action(async (opts: { execute?: boolean; all?: boolean; agent?: string }) => {
|
||||
try {
|
||||
await db.ensureInitialized();
|
||||
|
||||
// Fetch memories — by default only auto-capture (explicit stores are trusted)
|
||||
const conditions: string[] = [];
|
||||
if (!opts.all) {
|
||||
conditions.push("m.source = 'auto-capture'");
|
||||
}
|
||||
if (opts.agent) {
|
||||
conditions.push("m.agentId = $agentId");
|
||||
}
|
||||
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
||||
const allMemories = await db.runQuery<{
|
||||
id: string;
|
||||
text: string;
|
||||
source: string;
|
||||
}>(
|
||||
`MATCH (m:Memory) ${where}
|
||||
RETURN m.id AS id, m.text AS text, COALESCE(m.source, 'unknown') AS source
|
||||
ORDER BY m.createdAt ASC`,
|
||||
opts.agent ? { agentId: opts.agent } : {},
|
||||
);
|
||||
|
||||
// Strip channel metadata wrappers (same as the real pipeline) then gate
|
||||
const noise: Array<{ id: string; text: string; source: string }> = [];
|
||||
for (const mem of allMemories) {
|
||||
const stripped = stripMessageWrappers(mem.text);
|
||||
if (!passesAttentionGate(stripped)) {
|
||||
noise.push(mem);
|
||||
}
|
||||
}
|
||||
|
||||
if (noise.length === 0) {
|
||||
console.log("\nNo low-substance memories found. Everything passes the gate.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`\nFound ${noise.length}/${allMemories.length} memories that fail the attention gate:\n`,
|
||||
);
|
||||
|
||||
for (const mem of noise) {
|
||||
const preview = mem.text.length > 80 ? `${mem.text.slice(0, 77)}...` : mem.text;
|
||||
console.log(` [${mem.source}] "${preview}"`);
|
||||
}
|
||||
|
||||
if (!opts.execute) {
|
||||
console.log(
|
||||
`\nDry run — ${noise.length} memories would be removed. Re-run with --execute to delete.\n`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Delete in batch
|
||||
const deleted = await db.pruneMemories(noise.map((m) => m.id));
|
||||
console.log(`\nDeleted ${deleted} low-substance memories.\n`);
|
||||
} catch (err) {
|
||||
console.error(`Error: ${err instanceof Error ? err.message : String(err)}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
});
|
||||
memory
|
||||
.command("health")
|
||||
.description("Memory system health dashboard")
|
||||
.option("--agent <id>", "Scope to a specific agent")
|
||||
.option("--json", "Output all sections as JSON")
|
||||
.action(async (opts: { agent?: string; json?: boolean }) => {
|
||||
try {
|
||||
await db.ensureInitialized();
|
||||
|
||||
const agentId = opts.agent;
|
||||
|
||||
// Gather all data in parallel
|
||||
const [
|
||||
memoryStats,
|
||||
totalCount,
|
||||
statusCounts,
|
||||
graphStats,
|
||||
decayDist,
|
||||
orphanEntities,
|
||||
orphanTags,
|
||||
singleUseTags,
|
||||
] = await Promise.all([
|
||||
db.getMemoryStats(),
|
||||
db.countMemories(agentId),
|
||||
db.countByExtractionStatus(agentId),
|
||||
db.getEntityGraphStats(agentId),
|
||||
db.getDecayDistribution(agentId),
|
||||
db.findOrphanEntities(500),
|
||||
db.findOrphanTags(500),
|
||||
db.findSingleUseTags(14, 500),
|
||||
]);
|
||||
|
||||
// Filter stats by agent if specified
|
||||
const filteredStats = agentId
|
||||
? memoryStats.filter((s) => s.agentId === agentId)
|
||||
: memoryStats;
|
||||
|
||||
if (opts.json) {
|
||||
const totalExtraction =
|
||||
statusCounts.pending +
|
||||
statusCounts.complete +
|
||||
statusCounts.failed +
|
||||
statusCounts.skipped;
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
memoryOverview: {
|
||||
total: totalCount,
|
||||
byAgentCategory: filteredStats,
|
||||
},
|
||||
extractionHealth: {
|
||||
...statusCounts,
|
||||
total: totalExtraction,
|
||||
coveragePercent:
|
||||
totalExtraction > 0
|
||||
? Number(((statusCounts.complete / totalExtraction) * 100).toFixed(1))
|
||||
: 0,
|
||||
},
|
||||
entityGraph: {
|
||||
...graphStats,
|
||||
orphanCount: orphanEntities.length,
|
||||
},
|
||||
tagHealth: {
|
||||
orphanCount: orphanTags.length,
|
||||
singleUseCount: singleUseTags.length,
|
||||
},
|
||||
decayDistribution: decayDist,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const BAR_W = 20;
|
||||
const bar = (ratio: number) => {
|
||||
const filled = Math.round(Math.min(1, Math.max(0, ratio)) * BAR_W);
|
||||
return "█".repeat(filled) + "░".repeat(BAR_W - filled);
|
||||
};
|
||||
|
||||
console.log("\n╔═══════════════════════════════════════════════════════════╗");
|
||||
console.log("║ Memory (Neo4j) Health Dashboard ║");
|
||||
if (agentId) {
|
||||
console.log(`║ Agent: ${agentId.padEnd(49)}║`);
|
||||
}
|
||||
console.log("╚═══════════════════════════════════════════════════════════╝");
|
||||
|
||||
// Section 1: Memory Overview
|
||||
console.log("\n┌─ Memory Overview");
|
||||
console.log("│");
|
||||
console.log(`│ Total: ${totalCount} memories`);
|
||||
|
||||
if (filteredStats.length > 0) {
|
||||
// Group by agent
|
||||
const byAgent = new Map<
|
||||
string,
|
||||
Array<{ category: string; count: number; avgImportance: number }>
|
||||
>();
|
||||
for (const row of filteredStats) {
|
||||
const list = byAgent.get(row.agentId) || [];
|
||||
list.push({
|
||||
category: row.category,
|
||||
count: row.count,
|
||||
avgImportance: row.avgImportance,
|
||||
});
|
||||
byAgent.set(row.agentId, list);
|
||||
}
|
||||
|
||||
for (const [agent, categories] of byAgent) {
|
||||
const agentTotal = categories.reduce((s, c) => s + c.count, 0);
|
||||
const maxCat = Math.max(...categories.map((c) => c.count));
|
||||
console.log(`│`);
|
||||
console.log(`│ ${agent} (${agentTotal}):`);
|
||||
for (const { category, count } of categories) {
|
||||
const ratio = maxCat > 0 ? count / maxCat : 0;
|
||||
console.log(`│ ${category.padEnd(12)} ${bar(ratio)} ${count}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log("└");
|
||||
|
||||
// Section 2: Extraction Health
|
||||
const totalExtraction =
|
||||
statusCounts.pending +
|
||||
statusCounts.complete +
|
||||
statusCounts.failed +
|
||||
statusCounts.skipped;
|
||||
const coveragePct =
|
||||
totalExtraction > 0
|
||||
? ((statusCounts.complete / totalExtraction) * 100).toFixed(1)
|
||||
: "0.0";
|
||||
|
||||
console.log("\n┌─ Extraction Health");
|
||||
console.log("│");
|
||||
console.log(
|
||||
`│ Coverage: ${coveragePct}% (${statusCounts.complete}/${totalExtraction})`,
|
||||
);
|
||||
console.log(`│`);
|
||||
const statusEntries: Array<[string, number]> = [
|
||||
["pending", statusCounts.pending],
|
||||
["complete", statusCounts.complete],
|
||||
["failed", statusCounts.failed],
|
||||
["skipped", statusCounts.skipped],
|
||||
];
|
||||
const maxStatus = Math.max(...statusEntries.map(([, c]) => c));
|
||||
for (const [label, count] of statusEntries) {
|
||||
const ratio = maxStatus > 0 ? count / maxStatus : 0;
|
||||
console.log(`│ ${label.padEnd(10)} ${bar(ratio)} ${count}`);
|
||||
}
|
||||
console.log("└");
|
||||
|
||||
// Section 3: Entity Graph
|
||||
console.log("\n┌─ Entity Graph");
|
||||
console.log("│");
|
||||
console.log(`│ Entities: ${graphStats.entityCount}`);
|
||||
console.log(`│ Mentions: ${graphStats.mentionCount}`);
|
||||
console.log(`│ Density: ${graphStats.density.toFixed(2)} mentions/entity`);
|
||||
console.log(`│ Orphans: ${orphanEntities.length}`);
|
||||
console.log("└");
|
||||
|
||||
// Section 4: Tag Health
|
||||
console.log("\n┌─ Tag Health");
|
||||
console.log("│");
|
||||
console.log(`│ Orphan tags: ${orphanTags.length}`);
|
||||
console.log(`│ Single-use tags: ${singleUseTags.length}`);
|
||||
console.log("└");
|
||||
|
||||
// Section 5: Decay Distribution
|
||||
console.log("\n┌─ Decay Distribution");
|
||||
console.log("│");
|
||||
if (decayDist.length > 0) {
|
||||
const maxDecay = Math.max(...decayDist.map((d) => d.count));
|
||||
for (const { bucket, count } of decayDist) {
|
||||
const ratio = maxDecay > 0 ? count / maxDecay : 0;
|
||||
console.log(`│ ${bucket.padEnd(13)} ${bar(ratio)} ${count}`);
|
||||
}
|
||||
} else {
|
||||
console.log("│ No non-core memories found.");
|
||||
}
|
||||
console.log("└\n");
|
||||
} catch (err) {
|
||||
console.error(`Error: ${err instanceof Error ? err.message : String(err)}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
});
|
||||
},
|
||||
{ commands: [] }, // Adds subcommands to existing "memory" command, no conflict
|
||||
);
|
||||
}
|
||||
728
extensions/memory-neo4j/config.test.ts
Normal file
728
extensions/memory-neo4j/config.test.ts
Normal file
@@ -0,0 +1,728 @@
|
||||
/**
|
||||
* Tests for config.ts — Configuration Parsing.
|
||||
*
|
||||
* Tests memoryNeo4jConfigSchema.parse(), vectorDimsForModel(), and resolveExtractionConfig().
|
||||
*/
|
||||
|
||||
import { describe, it, expect, afterEach } from "vitest";
|
||||
import {
|
||||
memoryNeo4jConfigSchema,
|
||||
vectorDimsForModel,
|
||||
contextLengthForModel,
|
||||
DEFAULT_EMBEDDING_CONTEXT_LENGTH,
|
||||
resolveExtractionConfig,
|
||||
} from "./config.js";
|
||||
|
||||
// ============================================================================
|
||||
// memoryNeo4jConfigSchema.parse()
|
||||
// ============================================================================
|
||||
|
||||
describe("memoryNeo4jConfigSchema.parse", () => {
|
||||
// Store original env vars so we can restore them
|
||||
const originalEnv = { ...process.env };
|
||||
|
||||
afterEach(() => {
|
||||
process.env = { ...originalEnv };
|
||||
});
|
||||
|
||||
describe("valid complete configs", () => {
|
||||
it("should parse a minimal valid config with ollama provider", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", user: "neo4j", password: "test" },
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
|
||||
expect(config.neo4j.uri).toBe("bolt://localhost:7687");
|
||||
expect(config.neo4j.username).toBe("neo4j");
|
||||
expect(config.neo4j.password).toBe("test");
|
||||
expect(config.embedding.provider).toBe("ollama");
|
||||
expect(config.embedding.model).toBe("mxbai-embed-large");
|
||||
expect(config.embedding.apiKey).toBeUndefined();
|
||||
expect(config.autoCapture).toBe(true);
|
||||
expect(config.autoRecall).toBe(true);
|
||||
expect(config.coreMemory.enabled).toBe(true);
|
||||
});
|
||||
|
||||
it("should parse a full config with openai provider", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: {
|
||||
uri: "neo4j+s://cloud.neo4j.io:7687",
|
||||
username: "admin",
|
||||
password: "secret",
|
||||
},
|
||||
embedding: {
|
||||
provider: "openai",
|
||||
apiKey: "sk-test-key",
|
||||
model: "text-embedding-3-large",
|
||||
},
|
||||
autoCapture: false,
|
||||
autoRecall: false,
|
||||
coreMemory: {
|
||||
enabled: false,
|
||||
refreshAtContextPercent: 75,
|
||||
},
|
||||
});
|
||||
|
||||
expect(config.neo4j.uri).toBe("neo4j+s://cloud.neo4j.io:7687");
|
||||
expect(config.neo4j.username).toBe("admin");
|
||||
expect(config.neo4j.password).toBe("secret");
|
||||
expect(config.embedding.provider).toBe("openai");
|
||||
expect(config.embedding.apiKey).toBe("sk-test-key");
|
||||
expect(config.embedding.model).toBe("text-embedding-3-large");
|
||||
expect(config.autoCapture).toBe(false);
|
||||
expect(config.autoRecall).toBe(false);
|
||||
expect(config.coreMemory.enabled).toBe(false);
|
||||
expect(config.coreMemory.refreshAtContextPercent).toBe(75);
|
||||
});
|
||||
|
||||
it("should support 'user' field as alias for 'username' in neo4j config", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", user: "custom-user", password: "pass" },
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.neo4j.username).toBe("custom-user");
|
||||
});
|
||||
|
||||
it("should support 'username' field in neo4j config", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", username: "custom-user", password: "pass" },
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.neo4j.username).toBe("custom-user");
|
||||
});
|
||||
|
||||
it("should default neo4j username to 'neo4j' when not specified", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "pass" },
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.neo4j.username).toBe("neo4j");
|
||||
});
|
||||
});
|
||||
|
||||
describe("missing required fields", () => {
|
||||
it("should throw when config is null", () => {
|
||||
expect(() => memoryNeo4jConfigSchema.parse(null)).toThrow("memory-neo4j config required");
|
||||
});
|
||||
|
||||
it("should throw when config is undefined", () => {
|
||||
expect(() => memoryNeo4jConfigSchema.parse(undefined)).toThrow(
|
||||
"memory-neo4j config required",
|
||||
);
|
||||
});
|
||||
|
||||
it("should throw when config is not an object", () => {
|
||||
expect(() => memoryNeo4jConfigSchema.parse("string")).toThrow("memory-neo4j config required");
|
||||
});
|
||||
|
||||
it("should throw when config is an array", () => {
|
||||
expect(() => memoryNeo4jConfigSchema.parse([])).toThrow("memory-neo4j config required");
|
||||
});
|
||||
|
||||
it("should throw when neo4j section is missing", () => {
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
embedding: { provider: "ollama" },
|
||||
}),
|
||||
).toThrow("neo4j config section is required");
|
||||
});
|
||||
|
||||
it("should throw when neo4j.uri is missing", () => {
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { password: "test" },
|
||||
embedding: { provider: "ollama" },
|
||||
}),
|
||||
).toThrow("neo4j.uri is required");
|
||||
});
|
||||
|
||||
it("should throw when neo4j.uri is empty string", () => {
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "", password: "test" },
|
||||
embedding: { provider: "ollama" },
|
||||
}),
|
||||
).toThrow("neo4j.uri is required");
|
||||
});
|
||||
});
|
||||
|
||||
describe("environment variable resolution", () => {
|
||||
it("should resolve ${ENV_VAR} in neo4j.password", () => {
|
||||
process.env.TEST_NEO4J_PASSWORD = "resolved-password";
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: {
|
||||
uri: "bolt://localhost:7687",
|
||||
password: "${TEST_NEO4J_PASSWORD}",
|
||||
},
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.neo4j.password).toBe("resolved-password");
|
||||
});
|
||||
|
||||
it("should resolve ${ENV_VAR} in embedding.apiKey", () => {
|
||||
process.env.TEST_OPENAI_KEY = "sk-from-env";
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "openai", apiKey: "${TEST_OPENAI_KEY}" },
|
||||
});
|
||||
expect(config.embedding.apiKey).toBe("sk-from-env");
|
||||
});
|
||||
|
||||
it("should resolve ${ENV_VAR} in neo4j.user (username)", () => {
|
||||
process.env.TEST_NEO4J_USER = "resolved-user";
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: {
|
||||
uri: "bolt://localhost:7687",
|
||||
user: "${TEST_NEO4J_USER}",
|
||||
password: "",
|
||||
},
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.neo4j.username).toBe("resolved-user");
|
||||
});
|
||||
|
||||
it("should resolve ${ENV_VAR} in neo4j.username", () => {
|
||||
process.env.TEST_NEO4J_USERNAME = "resolved-username";
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: {
|
||||
uri: "bolt://localhost:7687",
|
||||
username: "${TEST_NEO4J_USERNAME}",
|
||||
password: "",
|
||||
},
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.neo4j.username).toBe("resolved-username");
|
||||
});
|
||||
|
||||
it("should throw when referenced env var is not set", () => {
|
||||
delete process.env.NONEXISTENT_VAR;
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
neo4j: {
|
||||
uri: "bolt://localhost:7687",
|
||||
password: "${NONEXISTENT_VAR}",
|
||||
},
|
||||
embedding: { provider: "ollama" },
|
||||
}),
|
||||
).toThrow("Environment variable NONEXISTENT_VAR is not set");
|
||||
});
|
||||
});
|
||||
|
||||
describe("default values", () => {
|
||||
it("should default autoCapture to true", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.autoCapture).toBe(true);
|
||||
});
|
||||
|
||||
it("should default autoRecall to true", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.autoRecall).toBe(true);
|
||||
});
|
||||
|
||||
it("should default coreMemory.enabled to true", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.coreMemory.enabled).toBe(true);
|
||||
});
|
||||
|
||||
it("should default refreshAtContextPercent to undefined", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.coreMemory.refreshAtContextPercent).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should default embedding model to mxbai-embed-large for ollama", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.embedding.model).toBe("mxbai-embed-large");
|
||||
});
|
||||
|
||||
it("should default embedding model to text-embedding-3-small for openai", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "openai", apiKey: "sk-test" },
|
||||
});
|
||||
expect(config.embedding.model).toBe("text-embedding-3-small");
|
||||
});
|
||||
|
||||
it("should default neo4j.password to empty string when not provided", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687" },
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.neo4j.password).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("provider validation", () => {
|
||||
it("should require apiKey for openai provider", () => {
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "openai" },
|
||||
}),
|
||||
).toThrow("embedding.apiKey is required for OpenAI provider");
|
||||
});
|
||||
|
||||
it("should not require apiKey for ollama provider", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.embedding.apiKey).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should default to openai when no provider is specified", () => {
|
||||
// No provider but has apiKey — should default to openai
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { apiKey: "sk-test" },
|
||||
});
|
||||
expect(config.embedding.provider).toBe("openai");
|
||||
});
|
||||
|
||||
it("should accept embedding.baseUrl", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama", baseUrl: "http://my-ollama:11434" },
|
||||
});
|
||||
expect(config.embedding.baseUrl).toBe("http://my-ollama:11434");
|
||||
});
|
||||
});
|
||||
|
||||
describe("unknown keys rejected", () => {
|
||||
it("should reject unknown top-level keys", () => {
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
unknownKey: "value",
|
||||
}),
|
||||
).toThrow("unknown keys: unknownKey");
|
||||
});
|
||||
|
||||
it("should reject unknown neo4j keys", () => {
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "", port: 7687 },
|
||||
embedding: { provider: "ollama" },
|
||||
}),
|
||||
).toThrow("unknown keys: port");
|
||||
});
|
||||
|
||||
it("should reject unknown embedding keys", () => {
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama", temperature: 0.5 },
|
||||
}),
|
||||
).toThrow("unknown keys: temperature");
|
||||
});
|
||||
|
||||
it("should reject unknown coreMemory keys", () => {
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
coreMemory: { unknownField: true },
|
||||
}),
|
||||
).toThrow("unknown keys: unknownField");
|
||||
});
|
||||
});
|
||||
|
||||
describe("refreshAtContextPercent edge cases", () => {
|
||||
it("should accept refreshAtContextPercent of 1 (minimum valid)", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
coreMemory: { refreshAtContextPercent: 1 },
|
||||
});
|
||||
expect(config.coreMemory.refreshAtContextPercent).toBe(1);
|
||||
});
|
||||
|
||||
it("should accept refreshAtContextPercent of 100 (maximum valid)", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
coreMemory: { refreshAtContextPercent: 100 },
|
||||
});
|
||||
expect(config.coreMemory.refreshAtContextPercent).toBe(100);
|
||||
});
|
||||
|
||||
it("should reject refreshAtContextPercent of 0", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
coreMemory: { refreshAtContextPercent: 0 },
|
||||
});
|
||||
expect(config.coreMemory.refreshAtContextPercent).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should reject refreshAtContextPercent over 100 by throwing", () => {
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
coreMemory: { refreshAtContextPercent: 150 },
|
||||
}),
|
||||
).toThrow("coreMemory.refreshAtContextPercent must be between 1 and 100");
|
||||
});
|
||||
|
||||
it("should reject negative refreshAtContextPercent", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
coreMemory: { refreshAtContextPercent: -10 },
|
||||
});
|
||||
expect(config.coreMemory.refreshAtContextPercent).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should ignore non-number refreshAtContextPercent", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
coreMemory: { refreshAtContextPercent: "50" },
|
||||
});
|
||||
expect(config.coreMemory.refreshAtContextPercent).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("autoRecallMinScore", () => {
|
||||
it("should default autoRecallMinScore to 0.25 when not specified", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.autoRecallMinScore).toBe(0.25);
|
||||
});
|
||||
|
||||
it("should accept an explicit autoRecallMinScore value", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
autoRecallMinScore: 0.5,
|
||||
});
|
||||
expect(config.autoRecallMinScore).toBe(0.5);
|
||||
});
|
||||
|
||||
it("should accept autoRecallMinScore of 0", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
autoRecallMinScore: 0,
|
||||
});
|
||||
expect(config.autoRecallMinScore).toBe(0);
|
||||
});
|
||||
|
||||
it("should accept autoRecallMinScore of 1", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
autoRecallMinScore: 1,
|
||||
});
|
||||
expect(config.autoRecallMinScore).toBe(1);
|
||||
});
|
||||
|
||||
it("should throw when autoRecallMinScore is negative", () => {
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
autoRecallMinScore: -0.1,
|
||||
}),
|
||||
).toThrow("autoRecallMinScore must be between 0 and 1");
|
||||
});
|
||||
|
||||
it("should throw when autoRecallMinScore is greater than 1", () => {
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
autoRecallMinScore: 1.5,
|
||||
}),
|
||||
).toThrow("autoRecallMinScore must be between 0 and 1");
|
||||
});
|
||||
|
||||
it("should default to 0.25 when autoRecallMinScore is a non-number type", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
autoRecallMinScore: "0.5",
|
||||
});
|
||||
expect(config.autoRecallMinScore).toBe(0.25);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sleepCycle config section", () => {
|
||||
it("should default sleepCycle.auto to true", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.sleepCycle.auto).toBe(true);
|
||||
});
|
||||
|
||||
it("should respect explicit sleepCycle.auto = false", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
sleepCycle: { auto: false },
|
||||
});
|
||||
expect(config.sleepCycle.auto).toBe(false);
|
||||
});
|
||||
|
||||
it("should still accept autoIntervalMs without error (backwards compat)", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
sleepCycle: { autoIntervalMs: 3600000 },
|
||||
});
|
||||
expect(config.sleepCycle.auto).toBe(true);
|
||||
});
|
||||
|
||||
it("should reject unknown sleepCycle keys", () => {
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
sleepCycle: { unknownKey: true },
|
||||
}),
|
||||
).toThrow("unknown keys: unknownKey");
|
||||
});
|
||||
});
|
||||
|
||||
describe("extraction config section", () => {
|
||||
it("should parse extraction config when provided", () => {
|
||||
process.env.EXTRACTION_DUMMY = ""; // avoid env var issues
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
extraction: {
|
||||
apiKey: "or-test-key",
|
||||
model: "google/gemini-2.0-flash-001",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
},
|
||||
});
|
||||
expect(config.extraction).toBeDefined();
|
||||
expect(config.extraction!.apiKey).toBe("or-test-key");
|
||||
expect(config.extraction!.model).toBe("google/gemini-2.0-flash-001");
|
||||
});
|
||||
|
||||
it("should not include extraction when section is empty", () => {
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
extraction: {},
|
||||
});
|
||||
expect(config.extraction).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should reject unknown keys in extraction section", () => {
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", password: "" },
|
||||
embedding: { provider: "ollama" },
|
||||
extraction: { badKey: "value" },
|
||||
}),
|
||||
).toThrow("unknown keys: badKey");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// vectorDimsForModel()
|
||||
// ============================================================================
|
||||
|
||||
describe("vectorDimsForModel", () => {
|
||||
describe("known models", () => {
|
||||
it("should return 1536 for text-embedding-3-small", () => {
|
||||
expect(vectorDimsForModel("text-embedding-3-small")).toBe(1536);
|
||||
});
|
||||
|
||||
it("should return 3072 for text-embedding-3-large", () => {
|
||||
expect(vectorDimsForModel("text-embedding-3-large")).toBe(3072);
|
||||
});
|
||||
|
||||
it("should return 1024 for mxbai-embed-large", () => {
|
||||
expect(vectorDimsForModel("mxbai-embed-large")).toBe(1024);
|
||||
});
|
||||
|
||||
it("should return 768 for nomic-embed-text", () => {
|
||||
expect(vectorDimsForModel("nomic-embed-text")).toBe(768);
|
||||
});
|
||||
|
||||
it("should return 384 for all-minilm", () => {
|
||||
expect(vectorDimsForModel("all-minilm")).toBe(384);
|
||||
});
|
||||
});
|
||||
|
||||
describe("prefix matching", () => {
|
||||
it("should match versioned model names via prefix", () => {
|
||||
// mxbai-embed-large:latest should match mxbai-embed-large
|
||||
expect(vectorDimsForModel("mxbai-embed-large:latest")).toBe(1024);
|
||||
});
|
||||
|
||||
it("should match model with additional version suffix", () => {
|
||||
expect(vectorDimsForModel("nomic-embed-text:v1.5")).toBe(768);
|
||||
});
|
||||
});
|
||||
|
||||
describe("unknown models", () => {
|
||||
it("should return default 1024 for unknown model", () => {
|
||||
expect(vectorDimsForModel("unknown-model")).toBe(1024);
|
||||
});
|
||||
|
||||
it("should return default 1024 for empty string", () => {
|
||||
expect(vectorDimsForModel("")).toBe(1024);
|
||||
});
|
||||
|
||||
it("should return default 1024 for unrecognized prefix", () => {
|
||||
expect(vectorDimsForModel("custom-embed-v2")).toBe(1024);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// resolveExtractionConfig()
|
||||
// ============================================================================
|
||||
|
||||
describe("resolveExtractionConfig", () => {
|
||||
const originalEnv = { ...process.env };
|
||||
|
||||
afterEach(() => {
|
||||
process.env = { ...originalEnv };
|
||||
});
|
||||
|
||||
it("should return disabled config when no API key or explicit baseUrl", () => {
|
||||
delete process.env.OPENROUTER_API_KEY;
|
||||
const config = resolveExtractionConfig();
|
||||
expect(config.enabled).toBe(false);
|
||||
expect(config.apiKey).toBe("");
|
||||
});
|
||||
|
||||
it("should enable when OPENROUTER_API_KEY env var is set", () => {
|
||||
process.env.OPENROUTER_API_KEY = "or-env-key";
|
||||
const config = resolveExtractionConfig();
|
||||
expect(config.enabled).toBe(true);
|
||||
expect(config.apiKey).toBe("or-env-key");
|
||||
});
|
||||
|
||||
it("should enable when plugin config provides apiKey", () => {
|
||||
delete process.env.OPENROUTER_API_KEY;
|
||||
const config = resolveExtractionConfig({
|
||||
apiKey: "or-plugin-key",
|
||||
model: "custom-model",
|
||||
baseUrl: "https://custom.ai/api",
|
||||
});
|
||||
expect(config.enabled).toBe(true);
|
||||
expect(config.apiKey).toBe("or-plugin-key");
|
||||
expect(config.model).toBe("custom-model");
|
||||
expect(config.baseUrl).toBe("https://custom.ai/api");
|
||||
});
|
||||
|
||||
it("should enable when baseUrl is explicitly set (local Ollama, no API key)", () => {
|
||||
delete process.env.OPENROUTER_API_KEY;
|
||||
const config = resolveExtractionConfig({
|
||||
model: "llama3",
|
||||
baseUrl: "http://localhost:11434/v1",
|
||||
});
|
||||
expect(config.enabled).toBe(true);
|
||||
expect(config.apiKey).toBe("");
|
||||
expect(config.baseUrl).toBe("http://localhost:11434/v1");
|
||||
});
|
||||
|
||||
it("should use defaults for model and baseUrl", () => {
|
||||
delete process.env.OPENROUTER_API_KEY;
|
||||
delete process.env.EXTRACTION_MODEL;
|
||||
delete process.env.EXTRACTION_BASE_URL;
|
||||
const config = resolveExtractionConfig();
|
||||
expect(config.model).toBe("anthropic/claude-opus-4-6");
|
||||
expect(config.baseUrl).toBe("https://openrouter.ai/api/v1");
|
||||
});
|
||||
|
||||
it("should use EXTRACTION_MODEL env var", () => {
|
||||
delete process.env.OPENROUTER_API_KEY;
|
||||
process.env.EXTRACTION_MODEL = "meta/llama-3-70b";
|
||||
const config = resolveExtractionConfig();
|
||||
expect(config.model).toBe("meta/llama-3-70b");
|
||||
});
|
||||
|
||||
it("should use EXTRACTION_BASE_URL env var", () => {
|
||||
delete process.env.OPENROUTER_API_KEY;
|
||||
process.env.EXTRACTION_BASE_URL = "https://my-proxy.ai/v1";
|
||||
const config = resolveExtractionConfig();
|
||||
expect(config.baseUrl).toBe("https://my-proxy.ai/v1");
|
||||
});
|
||||
|
||||
it("should always set temperature to 0.0 and maxRetries to 2", () => {
|
||||
const config = resolveExtractionConfig();
|
||||
expect(config.temperature).toBe(0.0);
|
||||
expect(config.maxRetries).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// contextLengthForModel()
|
||||
// ============================================================================
|
||||
|
||||
describe("contextLengthForModel", () => {
|
||||
describe("exact match", () => {
|
||||
it("should return 512 for mxbai-embed-large", () => {
|
||||
expect(contextLengthForModel("mxbai-embed-large")).toBe(512);
|
||||
});
|
||||
|
||||
it("should return 8191 for text-embedding-3-small (OpenAI)", () => {
|
||||
expect(contextLengthForModel("text-embedding-3-small")).toBe(8191);
|
||||
});
|
||||
|
||||
it("should return 8191 for text-embedding-3-large (OpenAI)", () => {
|
||||
expect(contextLengthForModel("text-embedding-3-large")).toBe(8191);
|
||||
});
|
||||
|
||||
it("should return 8192 for nomic-embed-text", () => {
|
||||
expect(contextLengthForModel("nomic-embed-text")).toBe(8192);
|
||||
});
|
||||
|
||||
it("should return 256 for all-minilm", () => {
|
||||
expect(contextLengthForModel("all-minilm")).toBe(256);
|
||||
});
|
||||
});
|
||||
|
||||
describe("prefix match", () => {
|
||||
it("should match mxbai-embed-large-8k:latest via prefix to 8192", () => {
|
||||
expect(contextLengthForModel("mxbai-embed-large-8k:latest")).toBe(8192);
|
||||
});
|
||||
|
||||
it("should match nomic-embed-text:v1.5 via prefix to 8192", () => {
|
||||
expect(contextLengthForModel("nomic-embed-text:v1.5")).toBe(8192);
|
||||
});
|
||||
});
|
||||
|
||||
describe("unknown model fallback", () => {
|
||||
it("should return DEFAULT_EMBEDDING_CONTEXT_LENGTH for unknown model", () => {
|
||||
expect(contextLengthForModel("some-unknown-model")).toBe(DEFAULT_EMBEDDING_CONTEXT_LENGTH);
|
||||
});
|
||||
|
||||
it("should return 512 as the default context length", () => {
|
||||
// Verify the default value itself is 512
|
||||
expect(DEFAULT_EMBEDDING_CONTEXT_LENGTH).toBe(512);
|
||||
expect(contextLengthForModel("some-unknown-model")).toBe(512);
|
||||
});
|
||||
|
||||
it("should return default for empty string", () => {
|
||||
expect(contextLengthForModel("")).toBe(DEFAULT_EMBEDDING_CONTEXT_LENGTH);
|
||||
});
|
||||
});
|
||||
});
|
||||
397
extensions/memory-neo4j/config.ts
Normal file
397
extensions/memory-neo4j/config.ts
Normal file
@@ -0,0 +1,397 @@
|
||||
/**
|
||||
* Configuration schema for memory-neo4j plugin.
|
||||
*
|
||||
* Matches the JSON Schema in openclaw.plugin.json.
|
||||
* Provides runtime parsing with env var resolution and defaults.
|
||||
*/
|
||||
|
||||
import type { MemoryCategory } from "./schema.js";
|
||||
import { MEMORY_CATEGORIES } from "./schema.js";
|
||||
|
||||
export type { MemoryCategory };
|
||||
export { MEMORY_CATEGORIES };
|
||||
|
||||
export type EmbeddingProvider = "openai" | "ollama";
|
||||
|
||||
export type MemoryNeo4jConfig = {
|
||||
neo4j: {
|
||||
uri: string;
|
||||
username: string;
|
||||
password: string;
|
||||
};
|
||||
embedding: {
|
||||
provider: EmbeddingProvider;
|
||||
apiKey?: string;
|
||||
model: string;
|
||||
baseUrl?: string;
|
||||
};
|
||||
extraction?: {
|
||||
apiKey?: string;
|
||||
model: string;
|
||||
baseUrl: string;
|
||||
};
|
||||
autoCapture: boolean;
|
||||
autoCaptureSkipPattern?: RegExp;
|
||||
autoRecall: boolean;
|
||||
autoRecallMinScore: number;
|
||||
/**
|
||||
* RegExp pattern to skip auto-recall for matching session keys.
|
||||
* Useful for voice/realtime sessions where latency is critical.
|
||||
* Example: /voice|realtime/ skips sessions containing "voice" or "realtime".
|
||||
*/
|
||||
autoRecallSkipPattern?: RegExp;
|
||||
coreMemory: {
|
||||
enabled: boolean;
|
||||
/**
|
||||
* Re-inject core memories when context usage reaches this percentage (0-100).
|
||||
* Helps counter "lost in the middle" phenomenon by refreshing core memories
|
||||
* closer to the end of context for recency bias.
|
||||
* Set to null/undefined to disable (default).
|
||||
*/
|
||||
refreshAtContextPercent?: number;
|
||||
};
|
||||
/**
|
||||
* Maximum relationship hops for graph search spreading activation.
|
||||
* Default: 1 (direct + 1-hop neighbors).
|
||||
* Setting to 2+ enables deeper traversal but may slow queries.
|
||||
*/
|
||||
graphSearchDepth: number;
|
||||
/**
|
||||
* Per-category decay curve parameters. Each category can have its own
|
||||
* half-life (days) controlling how fast memories in that category decay.
|
||||
* Categories not listed use the sleep cycle's default (30 days).
|
||||
*/
|
||||
decayCurves: Record<string, { halfLifeDays: number }>;
|
||||
sleepCycle: {
|
||||
auto: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Extraction configuration resolved from environment variables.
|
||||
* Entity extraction auto-enables when OPENROUTER_API_KEY is set.
|
||||
*/
|
||||
export type ExtractionConfig = {
|
||||
enabled: boolean;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
baseUrl: string;
|
||||
temperature: number;
|
||||
maxRetries: number;
|
||||
};
|
||||
|
||||
export const EMBEDDING_DIMENSIONS: Record<string, number> = {
|
||||
// OpenAI models
|
||||
"text-embedding-3-small": 1536,
|
||||
"text-embedding-3-large": 3072,
|
||||
// Ollama models (common ones)
|
||||
"mxbai-embed-large": 1024,
|
||||
"mxbai-embed-large-2k:latest": 1024,
|
||||
"nomic-embed-text": 768,
|
||||
"all-minilm": 384,
|
||||
};
|
||||
|
||||
// Default dimension for unknown models (Ollama models vary)
|
||||
export const DEFAULT_EMBEDDING_DIMS = 1024;
|
||||
|
||||
/**
|
||||
* Lookup a value by exact key or longest matching prefix.
|
||||
* Returns undefined if no match found.
|
||||
*/
|
||||
function lookupByPrefix<T>(table: Record<string, T>, key: string): T | undefined {
|
||||
if (table[key] !== undefined) {
|
||||
return table[key];
|
||||
}
|
||||
let best: { value: T; keyLen: number } | undefined;
|
||||
for (const [known, value] of Object.entries(table)) {
|
||||
if (key.startsWith(known) && (!best || known.length > best.keyLen)) {
|
||||
best = { value, keyLen: known.length };
|
||||
}
|
||||
}
|
||||
return best?.value;
|
||||
}
|
||||
|
||||
export function vectorDimsForModel(model: string): number {
|
||||
// Return default for unknown models — callers should warn when this path is taken,
|
||||
// as the default 1024 dimensions may not match the actual model's output.
|
||||
return lookupByPrefix(EMBEDDING_DIMENSIONS, model) ?? DEFAULT_EMBEDDING_DIMS;
|
||||
}
|
||||
|
||||
/** Max input token lengths for known embedding models. */
|
||||
export const EMBEDDING_CONTEXT_LENGTHS: Record<string, number> = {
|
||||
// OpenAI models
|
||||
"text-embedding-3-small": 8191,
|
||||
"text-embedding-3-large": 8191,
|
||||
// Ollama models
|
||||
"mxbai-embed-large": 512,
|
||||
"mxbai-embed-large-2k": 2048,
|
||||
"mxbai-embed-large-8k": 8192,
|
||||
"nomic-embed-text": 8192,
|
||||
"all-minilm": 256,
|
||||
};
|
||||
|
||||
/** Conservative default for unknown models. */
|
||||
export const DEFAULT_EMBEDDING_CONTEXT_LENGTH = 512;
|
||||
|
||||
export function contextLengthForModel(model: string): number {
|
||||
return lookupByPrefix(EMBEDDING_CONTEXT_LENGTHS, model) ?? DEFAULT_EMBEDDING_CONTEXT_LENGTH;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve ${ENV_VAR} references in string values.
|
||||
*/
|
||||
function resolveEnvVars(value: string): string {
|
||||
return value.replace(/\$\{([^}]+)\}/g, (_, envVar) => {
|
||||
const envValue = process.env[envVar];
|
||||
if (!envValue) {
|
||||
throw new Error(`Environment variable ${envVar} is not set`);
|
||||
}
|
||||
return envValue;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve extraction config from plugin config with env var fallback.
|
||||
* Enabled when an API key is available (cloud) or a baseUrl is explicitly
|
||||
* configured (Ollama / local LLMs that don't need a key).
|
||||
*/
|
||||
export function resolveExtractionConfig(
|
||||
cfgExtraction?: MemoryNeo4jConfig["extraction"],
|
||||
): ExtractionConfig {
|
||||
const apiKey = cfgExtraction?.apiKey ?? process.env.OPENROUTER_API_KEY ?? "";
|
||||
const model = cfgExtraction?.model ?? process.env.EXTRACTION_MODEL ?? "anthropic/claude-opus-4-6";
|
||||
const baseUrl =
|
||||
cfgExtraction?.baseUrl ?? process.env.EXTRACTION_BASE_URL ?? "https://openrouter.ai/api/v1";
|
||||
// Enabled when an API key is set (cloud provider) or baseUrl was explicitly
|
||||
// configured in the plugin config (Ollama / local — no key needed).
|
||||
const enabled = apiKey.length > 0 || cfgExtraction?.baseUrl != null;
|
||||
return {
|
||||
enabled,
|
||||
apiKey,
|
||||
model,
|
||||
baseUrl,
|
||||
temperature: 0.0,
|
||||
maxRetries: 2,
|
||||
};
|
||||
}
|
||||
|
||||
function assertAllowedKeys(value: Record<string, unknown>, allowed: string[], label: string) {
|
||||
const unknown = Object.keys(value).filter((key) => !allowed.includes(key));
|
||||
if (unknown.length > 0) {
|
||||
throw new Error(`${label} has unknown keys: ${unknown.join(", ")}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Parse autoRecallMinScore: must be a number between 0 and 1, default 0.25. */
|
||||
function parseAutoRecallMinScore(value: unknown): number {
|
||||
if (typeof value !== "number") return 0.25;
|
||||
if (value < 0 || value > 1) {
|
||||
throw new Error(`autoRecallMinScore must be between 0 and 1, got: ${value}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Config schema with parse method for runtime validation & transformation.
|
||||
* JSON Schema validation is handled by openclaw.plugin.json; this handles
|
||||
* env var resolution and defaults.
|
||||
*/
|
||||
export const memoryNeo4jConfigSchema = {
|
||||
parse(value: unknown): MemoryNeo4jConfig {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
throw new Error("memory-neo4j config required");
|
||||
}
|
||||
const cfg = value as Record<string, unknown>;
|
||||
assertAllowedKeys(
|
||||
cfg,
|
||||
[
|
||||
"embedding",
|
||||
"neo4j",
|
||||
"autoCapture",
|
||||
"autoCaptureSkipPattern",
|
||||
"autoRecall",
|
||||
"autoRecallMinScore",
|
||||
"autoRecallSkipPattern",
|
||||
"coreMemory",
|
||||
"extraction",
|
||||
"graphSearchDepth",
|
||||
"decayCurves",
|
||||
"sleepCycle",
|
||||
],
|
||||
"memory-neo4j config",
|
||||
);
|
||||
|
||||
// Parse neo4j section
|
||||
const neo4jRaw = cfg.neo4j as Record<string, unknown> | undefined;
|
||||
if (!neo4jRaw || typeof neo4jRaw !== "object") {
|
||||
throw new Error("neo4j config section is required");
|
||||
}
|
||||
assertAllowedKeys(neo4jRaw, ["uri", "user", "username", "password"], "neo4j config");
|
||||
if (typeof neo4jRaw.uri !== "string" || !neo4jRaw.uri) {
|
||||
throw new Error("neo4j.uri is required");
|
||||
}
|
||||
const neo4jUri = resolveEnvVars(neo4jRaw.uri);
|
||||
// Validate URI scheme — must be a valid Neo4j connection protocol
|
||||
const VALID_NEO4J_SCHEMES = [
|
||||
"bolt://",
|
||||
"bolt+s://",
|
||||
"bolt+ssc://",
|
||||
"neo4j://",
|
||||
"neo4j+s://",
|
||||
"neo4j+ssc://",
|
||||
];
|
||||
if (!VALID_NEO4J_SCHEMES.some((scheme) => neo4jUri.startsWith(scheme))) {
|
||||
throw new Error(
|
||||
`neo4j.uri must start with a valid scheme (${VALID_NEO4J_SCHEMES.join(", ")}), got: "${neo4jUri}"`,
|
||||
);
|
||||
}
|
||||
|
||||
const neo4jPassword =
|
||||
typeof neo4jRaw.password === "string" ? resolveEnvVars(neo4jRaw.password) : "";
|
||||
// Support both 'user' and 'username' for neo4j config
|
||||
const neo4jUsername =
|
||||
typeof neo4jRaw.user === "string"
|
||||
? resolveEnvVars(neo4jRaw.user)
|
||||
: typeof neo4jRaw.username === "string"
|
||||
? resolveEnvVars(neo4jRaw.username)
|
||||
: "neo4j";
|
||||
|
||||
// Parse embedding section (optional for ollama without apiKey)
|
||||
const embeddingRaw = cfg.embedding as Record<string, unknown> | undefined;
|
||||
assertAllowedKeys(
|
||||
embeddingRaw ?? {},
|
||||
["provider", "apiKey", "model", "baseUrl"],
|
||||
"embedding config",
|
||||
);
|
||||
|
||||
const provider: EmbeddingProvider = embeddingRaw?.provider === "ollama" ? "ollama" : "openai";
|
||||
|
||||
// apiKey is required for openai, optional for ollama
|
||||
let apiKey: string | undefined;
|
||||
if (typeof embeddingRaw?.apiKey === "string" && embeddingRaw.apiKey) {
|
||||
apiKey = resolveEnvVars(embeddingRaw.apiKey);
|
||||
} else if (provider === "openai") {
|
||||
throw new Error("embedding.apiKey is required for OpenAI provider");
|
||||
}
|
||||
|
||||
const embeddingModel =
|
||||
typeof embeddingRaw?.model === "string"
|
||||
? embeddingRaw.model
|
||||
: provider === "ollama"
|
||||
? "mxbai-embed-large"
|
||||
: "text-embedding-3-small";
|
||||
|
||||
const baseUrl = typeof embeddingRaw?.baseUrl === "string" ? embeddingRaw.baseUrl : undefined;
|
||||
|
||||
// Parse coreMemory section (optional with defaults)
|
||||
const coreMemoryRaw = cfg.coreMemory as Record<string, unknown> | undefined;
|
||||
assertAllowedKeys(
|
||||
coreMemoryRaw ?? {},
|
||||
["enabled", "refreshAtContextPercent"],
|
||||
"coreMemory config",
|
||||
);
|
||||
const coreMemoryEnabled = coreMemoryRaw?.enabled !== false; // enabled by default
|
||||
// refreshAtContextPercent: number between 1-99 to be effective, or undefined to disable.
|
||||
// Values at 0 or below are ignored (disables refresh). Values above 100 are invalid.
|
||||
if (
|
||||
typeof coreMemoryRaw?.refreshAtContextPercent === "number" &&
|
||||
coreMemoryRaw.refreshAtContextPercent > 100
|
||||
) {
|
||||
throw new Error(
|
||||
`coreMemory.refreshAtContextPercent must be between 1 and 100, got: ${coreMemoryRaw.refreshAtContextPercent}`,
|
||||
);
|
||||
}
|
||||
const refreshAtContextPercent =
|
||||
typeof coreMemoryRaw?.refreshAtContextPercent === "number" &&
|
||||
coreMemoryRaw.refreshAtContextPercent > 0 &&
|
||||
coreMemoryRaw.refreshAtContextPercent <= 100
|
||||
? coreMemoryRaw.refreshAtContextPercent
|
||||
: undefined;
|
||||
|
||||
// Parse extraction section (optional — falls back to env vars in resolveExtractionConfig)
|
||||
const extractionRaw = cfg.extraction as Record<string, unknown> | undefined;
|
||||
assertAllowedKeys(extractionRaw ?? {}, ["apiKey", "model", "baseUrl"], "extraction config");
|
||||
let extraction: MemoryNeo4jConfig["extraction"];
|
||||
if (extractionRaw) {
|
||||
const exApiKey =
|
||||
typeof extractionRaw.apiKey === "string" ? resolveEnvVars(extractionRaw.apiKey) : undefined;
|
||||
const exModel = typeof extractionRaw.model === "string" ? extractionRaw.model : undefined;
|
||||
const exBaseUrl =
|
||||
typeof extractionRaw.baseUrl === "string" ? extractionRaw.baseUrl : undefined;
|
||||
// Only include if at least one field was provided
|
||||
if (exApiKey || exModel || exBaseUrl) {
|
||||
extraction = {
|
||||
apiKey: exApiKey,
|
||||
model: exModel ?? (process.env.EXTRACTION_MODEL || "anthropic/claude-opus-4-6"),
|
||||
baseUrl: exBaseUrl ?? (process.env.EXTRACTION_BASE_URL || "https://openrouter.ai/api/v1"),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Parse decayCurves: per-category decay curve overrides
|
||||
const decayCurvesRaw = cfg.decayCurves as Record<string, unknown> | undefined;
|
||||
const decayCurves: Record<string, { halfLifeDays: number }> = {};
|
||||
if (decayCurvesRaw && typeof decayCurvesRaw === "object") {
|
||||
for (const [cat, val] of Object.entries(decayCurvesRaw)) {
|
||||
if (val && typeof val === "object" && "halfLifeDays" in val) {
|
||||
const hl = (val as Record<string, unknown>).halfLifeDays;
|
||||
if (typeof hl === "number" && hl > 0) {
|
||||
decayCurves[cat] = { halfLifeDays: hl };
|
||||
} else {
|
||||
throw new Error(`decayCurves.${cat}.halfLifeDays must be a positive number`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse graphSearchDepth: must be 1-3, default 1
|
||||
const rawDepth = cfg.graphSearchDepth;
|
||||
let graphSearchDepth = 1;
|
||||
if (typeof rawDepth === "number") {
|
||||
if (rawDepth < 1 || rawDepth > 3 || !Number.isInteger(rawDepth)) {
|
||||
throw new Error(`graphSearchDepth must be 1, 2, or 3, got: ${rawDepth}`);
|
||||
}
|
||||
graphSearchDepth = rawDepth;
|
||||
}
|
||||
|
||||
// Parse sleepCycle section (optional with defaults)
|
||||
const sleepCycleRaw = cfg.sleepCycle as Record<string, unknown> | undefined;
|
||||
assertAllowedKeys(sleepCycleRaw ?? {}, ["auto", "autoIntervalMs"], "sleepCycle config");
|
||||
const sleepCycleAuto = sleepCycleRaw?.auto !== false; // enabled by default
|
||||
|
||||
return {
|
||||
neo4j: {
|
||||
uri: neo4jUri,
|
||||
username: neo4jUsername,
|
||||
password: neo4jPassword,
|
||||
},
|
||||
embedding: {
|
||||
provider,
|
||||
apiKey,
|
||||
model: embeddingModel,
|
||||
baseUrl,
|
||||
},
|
||||
extraction,
|
||||
autoCapture: cfg.autoCapture !== false,
|
||||
autoCaptureSkipPattern:
|
||||
typeof cfg.autoCaptureSkipPattern === "string" && cfg.autoCaptureSkipPattern
|
||||
? new RegExp(cfg.autoCaptureSkipPattern)
|
||||
: undefined,
|
||||
autoRecall: cfg.autoRecall !== false,
|
||||
autoRecallMinScore: parseAutoRecallMinScore(cfg.autoRecallMinScore),
|
||||
autoRecallSkipPattern:
|
||||
typeof cfg.autoRecallSkipPattern === "string" && cfg.autoRecallSkipPattern
|
||||
? new RegExp(cfg.autoRecallSkipPattern)
|
||||
: undefined,
|
||||
coreMemory: {
|
||||
enabled: coreMemoryEnabled,
|
||||
refreshAtContextPercent,
|
||||
},
|
||||
graphSearchDepth,
|
||||
decayCurves,
|
||||
sleepCycle: {
|
||||
auto: sleepCycleAuto,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
481
extensions/memory-neo4j/embeddings.test.ts
Normal file
481
extensions/memory-neo4j/embeddings.test.ts
Normal file
@@ -0,0 +1,481 @@
|
||||
/**
|
||||
* Tests for embeddings.ts — Embedding Provider.
|
||||
*
|
||||
* Tests the Embeddings class with mocked OpenAI client and mocked fetch for Ollama.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, afterEach, beforeEach } from "vitest";
|
||||
|
||||
// ============================================================================
|
||||
// Constructor
|
||||
// ============================================================================
|
||||
|
||||
describe("Embeddings constructor", () => {
|
||||
it("should throw when OpenAI provider is used without API key", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
expect(() => new Embeddings(undefined, "text-embedding-3-small", "openai")).toThrow(
|
||||
"API key required for OpenAI embeddings",
|
||||
);
|
||||
});
|
||||
|
||||
it("should not require API key for ollama provider", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama");
|
||||
expect(emb).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Ollama embed
|
||||
// ============================================================================
|
||||
|
||||
describe("Embeddings - Ollama provider", () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
it("should call Ollama API with correct request body", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const mockVector = [0.1, 0.2, 0.3, 0.4];
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({ embeddings: [mockVector] }),
|
||||
});
|
||||
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama");
|
||||
const result = await emb.embed("test text");
|
||||
|
||||
expect(result).toEqual(mockVector);
|
||||
expect(globalThis.fetch).toHaveBeenCalledWith(
|
||||
"http://localhost:11434/api/embed",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: "mxbai-embed-large",
|
||||
input: "test text",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("should use custom baseUrl for Ollama", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const mockVector = [0.5, 0.6];
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({ embeddings: [mockVector] }),
|
||||
});
|
||||
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama", "http://my-host:11434");
|
||||
await emb.embed("test");
|
||||
|
||||
expect(globalThis.fetch).toHaveBeenCalledWith(
|
||||
"http://my-host:11434/api/embed",
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it("should strip trailing slashes from baseUrl", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const mockVector = [0.1, 0.2];
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({ embeddings: [mockVector] }),
|
||||
});
|
||||
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama", "http://my-host:11434/");
|
||||
await emb.embed("test");
|
||||
|
||||
expect(globalThis.fetch).toHaveBeenCalledWith(
|
||||
"http://my-host:11434/api/embed",
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it("should strip multiple trailing slashes from baseUrl", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const mockVector = [0.1, 0.2];
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({ embeddings: [mockVector] }),
|
||||
});
|
||||
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama", "http://my-host:11434///");
|
||||
await emb.embed("test");
|
||||
|
||||
expect(globalThis.fetch).toHaveBeenCalledWith(
|
||||
"http://my-host:11434/api/embed",
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it("should throw when Ollama returns error status", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: false,
|
||||
status: 500,
|
||||
text: () => Promise.resolve("Internal Server Error"),
|
||||
});
|
||||
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama");
|
||||
await expect(emb.embed("test")).rejects.toThrow("Ollama embedding failed: 500");
|
||||
});
|
||||
|
||||
it("should throw when Ollama returns no embeddings", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({ embeddings: [] }),
|
||||
});
|
||||
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama");
|
||||
await expect(emb.embed("test")).rejects.toThrow("No embedding returned from Ollama");
|
||||
});
|
||||
|
||||
it("should throw when Ollama returns null embeddings", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({}),
|
||||
});
|
||||
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama");
|
||||
await expect(emb.embed("test")).rejects.toThrow("No embedding returned from Ollama");
|
||||
});
|
||||
|
||||
it("should propagate fetch errors for Ollama", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
globalThis.fetch = vi.fn().mockRejectedValue(new Error("Network error"));
|
||||
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama");
|
||||
await expect(emb.embed("test")).rejects.toThrow("Network error");
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// OpenAI embed (via mocked client internals)
|
||||
// ============================================================================
|
||||
|
||||
describe("Embeddings - OpenAI provider", () => {
|
||||
it("should create instance with OpenAI provider when API key provided", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
// Just verify construction succeeds with valid params
|
||||
const emb = new Embeddings("sk-test-key", "text-embedding-3-small", "openai");
|
||||
expect(emb).toBeDefined();
|
||||
});
|
||||
|
||||
it("should have embed and embedBatch methods", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const emb = new Embeddings("sk-test-key", "text-embedding-3-small", "openai");
|
||||
expect(typeof emb.embed).toBe("function");
|
||||
expect(typeof emb.embedBatch).toBe("function");
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Batch embedding
|
||||
// ============================================================================
|
||||
|
||||
describe("Embeddings - embedBatch", () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
it("should return empty array for empty input (openai)", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const emb = new Embeddings("sk-test", "text-embedding-3-small", "openai");
|
||||
const results = await emb.embedBatch([]);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
it("should return empty array for empty input (ollama)", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama");
|
||||
const results = await emb.embedBatch([]);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
it("should use sequential calls for Ollama batch (no native batch support)", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
let callCount = 0;
|
||||
globalThis.fetch = vi.fn().mockImplementation(() => {
|
||||
callCount++;
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({ embeddings: [[callCount * 0.1, callCount * 0.2]] }),
|
||||
});
|
||||
});
|
||||
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama");
|
||||
const results = await emb.embedBatch(["text1", "text2", "text3"]);
|
||||
|
||||
// Should make 3 separate calls
|
||||
expect(globalThis.fetch).toHaveBeenCalledTimes(3);
|
||||
expect(results).toHaveLength(3);
|
||||
// Each result should be a vector
|
||||
for (const r of results) {
|
||||
expect(Array.isArray(r)).toBe(true);
|
||||
expect(r.length).toBe(2);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Ollama context-length truncation
|
||||
// ============================================================================
|
||||
|
||||
describe("Embeddings - Ollama context-length truncation", () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({ embeddings: [[0.1, 0.2, 0.3]] }),
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
it("should truncate long input before calling Ollama embed", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama");
|
||||
|
||||
// mxbai-embed-large context length is 512, so maxChars = 512 * 3 = 1536
|
||||
// Create input that exceeds the limit
|
||||
const longText = "word ".repeat(500); // ~2500 chars, well above 1536
|
||||
await emb.embed(longText);
|
||||
|
||||
// Verify the text sent to Ollama was truncated
|
||||
const call = (globalThis.fetch as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||
const body = JSON.parse(call[1].body as string);
|
||||
expect(body.input.length).toBeLessThanOrEqual(512 * 3);
|
||||
});
|
||||
|
||||
it("should truncate at word boundary (not mid-word)", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama");
|
||||
|
||||
// maxChars for mxbai-embed-large = 512 * 3 = 1536
|
||||
// Each "abcdefghij " is 11 chars; 200 repeats = 2200 chars total (exceeds 1536)
|
||||
const longText = "abcdefghij ".repeat(200);
|
||||
await emb.embed(longText);
|
||||
|
||||
const call = (globalThis.fetch as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||
const body = JSON.parse(call[1].body as string);
|
||||
const sentText = body.input as string;
|
||||
|
||||
expect(sentText.length).toBeLessThanOrEqual(512 * 3);
|
||||
// The truncation should land on a word boundary: the sent text should
|
||||
// be a prefix of the original that ends at a complete word (i.e. the
|
||||
// character after the sent text in the original should be a space).
|
||||
// Since the pattern is "abcdefghij " repeated, a word-boundary cut
|
||||
// means sentText ends with "abcdefghij" (no trailing partial word).
|
||||
expect(sentText).toMatch(/abcdefghij$/);
|
||||
// Verify it's a proper prefix of the original
|
||||
expect(longText.startsWith(sentText)).toBe(true);
|
||||
});
|
||||
|
||||
it("should pass short input through unchanged", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama");
|
||||
|
||||
const shortText = "This is a short text that fits within context length.";
|
||||
await emb.embed(shortText);
|
||||
|
||||
const call = (globalThis.fetch as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||
const body = JSON.parse(call[1].body as string);
|
||||
expect(body.input).toBe(shortText);
|
||||
});
|
||||
|
||||
it("should use model-specific context length for truncation", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
// nomic-embed-text has context length 8192, maxChars = 8192 * 3 = 24576
|
||||
const emb = new Embeddings(undefined, "nomic-embed-text", "ollama");
|
||||
|
||||
// Create text that exceeds mxbai limit (1536) but fits nomic limit (24576)
|
||||
const mediumText = "hello ".repeat(400); // ~2400 chars
|
||||
await emb.embed(mediumText);
|
||||
|
||||
const call = (globalThis.fetch as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||
const body = JSON.parse(call[1].body as string);
|
||||
// Should NOT be truncated since 2400 < 24576
|
||||
expect(body.input).toBe(mediumText);
|
||||
});
|
||||
|
||||
it("should truncate each item individually in embedBatch", async () => {
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const emb = new Embeddings(undefined, "mxbai-embed-large", "ollama");
|
||||
|
||||
// maxChars for mxbai-embed-large = 512 * 3 = 1536
|
||||
const longText = "word ".repeat(500); // ~2500 chars, exceeds limit
|
||||
const shortText = "short text"; // well under limit
|
||||
|
||||
await emb.embedBatch([longText, shortText]);
|
||||
|
||||
const calls = (globalThis.fetch as ReturnType<typeof vi.fn>).mock.calls;
|
||||
expect(calls).toHaveLength(2);
|
||||
|
||||
// First call: long text should be truncated
|
||||
const body1 = JSON.parse(calls[0][1].body as string);
|
||||
expect(body1.input.length).toBeLessThanOrEqual(512 * 3);
|
||||
expect(body1.input.length).toBeLessThan(longText.length);
|
||||
|
||||
// Second call: short text should pass through unchanged
|
||||
const body2 = JSON.parse(calls[1][1].body as string);
|
||||
expect(body2.input).toBe(shortText);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// OpenAI embed — functional tests with mocked OpenAI client
|
||||
// ============================================================================
|
||||
|
||||
describe("Embeddings - OpenAI functional", () => {
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("embed() should call OpenAI API with correct model and input", async () => {
|
||||
const mockCreate = vi.fn().mockResolvedValue({
|
||||
data: [{ index: 0, embedding: [0.1, 0.2, 0.3] }],
|
||||
});
|
||||
|
||||
// Mock the openai module
|
||||
vi.doMock("openai", () => ({
|
||||
default: class MockOpenAI {
|
||||
embeddings = { create: mockCreate };
|
||||
},
|
||||
}));
|
||||
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const emb = new Embeddings("sk-test-key", "text-embedding-3-small", "openai");
|
||||
const result = await emb.embed("hello world");
|
||||
|
||||
expect(result).toEqual([0.1, 0.2, 0.3]);
|
||||
expect(mockCreate).toHaveBeenCalledWith({
|
||||
model: "text-embedding-3-small",
|
||||
input: "hello world",
|
||||
});
|
||||
});
|
||||
|
||||
it("embedBatch() should send all texts in a single API call and return correctly ordered results", async () => {
|
||||
const mockCreate = vi.fn().mockResolvedValue({
|
||||
// Return out-of-order to verify sorting by index
|
||||
data: [
|
||||
{ index: 2, embedding: [0.7, 0.8, 0.9] },
|
||||
{ index: 0, embedding: [0.1, 0.2, 0.3] },
|
||||
{ index: 1, embedding: [0.4, 0.5, 0.6] },
|
||||
],
|
||||
});
|
||||
|
||||
vi.doMock("openai", () => ({
|
||||
default: class MockOpenAI {
|
||||
embeddings = { create: mockCreate };
|
||||
},
|
||||
}));
|
||||
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const emb = new Embeddings("sk-test-key", "text-embedding-3-small", "openai");
|
||||
const results = await emb.embedBatch(["first", "second", "third"]);
|
||||
|
||||
// Should have made exactly one API call with all texts
|
||||
expect(mockCreate).toHaveBeenCalledTimes(1);
|
||||
expect(mockCreate).toHaveBeenCalledWith({
|
||||
model: "text-embedding-3-small",
|
||||
input: ["first", "second", "third"],
|
||||
});
|
||||
|
||||
// Results should be sorted by index (0, 1, 2)
|
||||
expect(results).toEqual([
|
||||
[0.1, 0.2, 0.3],
|
||||
[0.4, 0.5, 0.6],
|
||||
[0.7, 0.8, 0.9],
|
||||
]);
|
||||
});
|
||||
|
||||
it("embed() should propagate OpenAI API errors", async () => {
|
||||
const mockCreate = vi.fn().mockRejectedValue(new Error("API rate limit exceeded"));
|
||||
|
||||
vi.doMock("openai", () => ({
|
||||
default: class MockOpenAI {
|
||||
embeddings = { create: mockCreate };
|
||||
},
|
||||
}));
|
||||
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const emb = new Embeddings("sk-test-key", "text-embedding-3-small", "openai");
|
||||
|
||||
await expect(emb.embed("test")).rejects.toThrow("API rate limit exceeded");
|
||||
});
|
||||
|
||||
it("embed() should return cached result on second call for same text", async () => {
|
||||
const mockCreate = vi.fn().mockResolvedValue({
|
||||
data: [{ index: 0, embedding: [0.1, 0.2, 0.3] }],
|
||||
});
|
||||
|
||||
vi.doMock("openai", () => ({
|
||||
default: class MockOpenAI {
|
||||
embeddings = { create: mockCreate };
|
||||
},
|
||||
}));
|
||||
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const emb = new Embeddings("sk-test-key", "text-embedding-3-small", "openai");
|
||||
|
||||
const result1 = await emb.embed("cached text");
|
||||
const result2 = await emb.embed("cached text");
|
||||
|
||||
expect(result1).toEqual([0.1, 0.2, 0.3]);
|
||||
expect(result2).toEqual([0.1, 0.2, 0.3]);
|
||||
// Should only make one API call — second call uses cache
|
||||
expect(mockCreate).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("embedBatch() should use cache for previously embedded texts", async () => {
|
||||
const mockCreate = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
data: [{ index: 0, embedding: [0.1, 0.2, 0.3] }],
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
data: [{ index: 0, embedding: [0.7, 0.8, 0.9] }],
|
||||
});
|
||||
|
||||
vi.doMock("openai", () => ({
|
||||
default: class MockOpenAI {
|
||||
embeddings = { create: mockCreate };
|
||||
},
|
||||
}));
|
||||
|
||||
const { Embeddings } = await import("./embeddings.js");
|
||||
const emb = new Embeddings("sk-test-key", "text-embedding-3-small", "openai");
|
||||
|
||||
// First: embed "alpha" to populate cache
|
||||
await emb.embed("alpha");
|
||||
expect(mockCreate).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Now batch with "alpha" (cached) and "beta" (uncached)
|
||||
const results = await emb.embedBatch(["alpha", "beta"]);
|
||||
// Should only call API once more for "beta"
|
||||
expect(mockCreate).toHaveBeenCalledTimes(2);
|
||||
expect(mockCreate).toHaveBeenLastCalledWith({
|
||||
model: "text-embedding-3-small",
|
||||
input: ["beta"],
|
||||
});
|
||||
expect(results).toEqual([
|
||||
[0.1, 0.2, 0.3], // cached
|
||||
[0.7, 0.8, 0.9], // freshly computed
|
||||
]);
|
||||
});
|
||||
});
|
||||
322
extensions/memory-neo4j/embeddings.ts
Normal file
322
extensions/memory-neo4j/embeddings.ts
Normal file
@@ -0,0 +1,322 @@
|
||||
/**
|
||||
* Embedding generation for memory-neo4j.
|
||||
*
|
||||
* Supports both OpenAI and Ollama providers.
|
||||
* Includes an LRU cache to avoid redundant API calls within a session.
|
||||
*/
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import OpenAI from "openai";
|
||||
import type { EmbeddingProvider } from "./config.js";
|
||||
import type { Logger } from "./schema.js";
|
||||
import { contextLengthForModel } from "./config.js";
|
||||
|
||||
/**
|
||||
* Simple LRU cache for embedding vectors.
|
||||
* Keyed by SHA-256 hash of the input text to avoid storing large strings.
|
||||
*/
|
||||
class EmbeddingCache {
|
||||
private readonly map = new Map<string, number[]>();
|
||||
private readonly maxSize: number;
|
||||
|
||||
constructor(maxSize: number = 200) {
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
|
||||
private static hashText(text: string): string {
|
||||
return createHash("sha256").update(text).digest("hex");
|
||||
}
|
||||
|
||||
get(text: string): number[] | undefined {
|
||||
const key = EmbeddingCache.hashText(text);
|
||||
const value = this.map.get(key);
|
||||
if (value !== undefined) {
|
||||
// Move to end (most recently used) by re-inserting
|
||||
this.map.delete(key);
|
||||
this.map.set(key, value);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
set(text: string, embedding: number[]): void {
|
||||
const key = EmbeddingCache.hashText(text);
|
||||
// If key exists, delete first to refresh position
|
||||
if (this.map.has(key)) {
|
||||
this.map.delete(key);
|
||||
} else if (this.map.size >= this.maxSize) {
|
||||
// Evict oldest (first) entry
|
||||
const oldest = this.map.keys().next().value;
|
||||
if (oldest !== undefined) {
|
||||
this.map.delete(oldest);
|
||||
}
|
||||
}
|
||||
this.map.set(key, embedding);
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.map.size;
|
||||
}
|
||||
}
|
||||
|
||||
/** Default concurrency for Ollama embedding requests */
|
||||
const OLLAMA_EMBED_CONCURRENCY = 4;
|
||||
|
||||
export class Embeddings {
|
||||
private client: OpenAI | null = null;
|
||||
private readonly provider: EmbeddingProvider;
|
||||
private readonly baseUrl: string;
|
||||
private readonly logger: Logger | undefined;
|
||||
private readonly contextLength: number;
|
||||
private readonly cache = new EmbeddingCache(200);
|
||||
|
||||
constructor(
|
||||
private readonly apiKey: string | undefined,
|
||||
private readonly model: string = "text-embedding-3-small",
|
||||
provider: EmbeddingProvider = "openai",
|
||||
baseUrl?: string,
|
||||
logger?: Logger,
|
||||
) {
|
||||
this.provider = provider;
|
||||
this.baseUrl = (baseUrl ?? (provider === "ollama" ? "http://localhost:11434" : "")).replace(
|
||||
/\/+$/,
|
||||
"",
|
||||
);
|
||||
this.logger = logger;
|
||||
this.contextLength = contextLengthForModel(model);
|
||||
|
||||
if (provider === "openai") {
|
||||
if (!apiKey) {
|
||||
throw new Error("API key required for OpenAI embeddings");
|
||||
}
|
||||
this.client = new OpenAI({ apiKey });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate text to fit within the model's context length.
|
||||
* Uses a conservative ~3 chars/token estimate to leave headroom —
|
||||
* code, URLs, and punctuation-heavy text tokenize at 1–2 chars/token,
|
||||
* so the classic ~4 estimate is too generous for mixed content.
|
||||
* Truncates at a word boundary when possible.
|
||||
*/
|
||||
private truncateToContext(text: string): string {
|
||||
const maxChars = this.contextLength * 3;
|
||||
if (text.length <= maxChars) {
|
||||
return text;
|
||||
}
|
||||
|
||||
// Try to truncate at a word boundary
|
||||
let truncated = text.slice(0, maxChars);
|
||||
const lastSpace = truncated.lastIndexOf(" ");
|
||||
if (lastSpace > maxChars * 0.8) {
|
||||
truncated = truncated.slice(0, lastSpace);
|
||||
}
|
||||
|
||||
this.logger?.debug?.(
|
||||
`memory-neo4j: truncated embedding input from ${text.length} to ${truncated.length} chars (model context: ${this.contextLength} tokens)`,
|
||||
);
|
||||
return truncated;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate an embedding vector for a single text.
|
||||
* Results are cached to avoid redundant API calls.
|
||||
*/
|
||||
async embed(text: string): Promise<number[]> {
|
||||
const input = this.truncateToContext(text);
|
||||
|
||||
// Check cache first
|
||||
const cached = this.cache.get(input);
|
||||
if (cached) {
|
||||
this.logger?.debug?.("memory-neo4j: embedding cache hit");
|
||||
return cached;
|
||||
}
|
||||
|
||||
const embedding =
|
||||
this.provider === "ollama" ? await this.embedOllama(input) : await this.embedOpenAI(input);
|
||||
|
||||
this.cache.set(input, embedding);
|
||||
return embedding;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embeddings for multiple texts.
|
||||
* Returns array of embeddings in the same order as input.
|
||||
*
|
||||
* For Ollama: processes in chunks of OLLAMA_EMBED_CONCURRENCY to avoid
|
||||
* overwhelming the local server. Individual failures don't break the
|
||||
* entire batch — failed embeddings are replaced with empty arrays.
|
||||
*/
|
||||
async embedBatch(texts: string[]): Promise<number[][]> {
|
||||
if (texts.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const truncated = texts.map((t) => this.truncateToContext(t));
|
||||
|
||||
// Check cache for each text; only compute uncached ones
|
||||
const results: (number[] | null)[] = truncated.map((t) => this.cache.get(t) ?? null);
|
||||
const uncachedIndices: number[] = [];
|
||||
const uncachedTexts: string[] = [];
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
if (results[i] === null) {
|
||||
uncachedIndices.push(i);
|
||||
uncachedTexts.push(truncated[i]);
|
||||
}
|
||||
}
|
||||
|
||||
if (uncachedTexts.length === 0) {
|
||||
this.logger?.debug?.(`memory-neo4j: embedBatch fully cached (${texts.length} texts)`);
|
||||
return results as number[][];
|
||||
}
|
||||
|
||||
let computed: number[][];
|
||||
|
||||
if (this.provider === "ollama") {
|
||||
computed = await this.embedBatchOllama(uncachedTexts);
|
||||
} else {
|
||||
computed = await this.embedBatchOpenAI(uncachedTexts);
|
||||
}
|
||||
|
||||
// Merge computed results back and populate cache
|
||||
for (let i = 0; i < uncachedIndices.length; i++) {
|
||||
const embedding = computed[i];
|
||||
results[uncachedIndices[i]] = embedding;
|
||||
if (embedding.length > 0) {
|
||||
this.cache.set(uncachedTexts[i], embedding);
|
||||
}
|
||||
}
|
||||
|
||||
return results as number[][];
|
||||
}
|
||||
|
||||
/**
|
||||
* Ollama batch embedding with concurrency limiting.
|
||||
* Processes in chunks to avoid overwhelming the server.
|
||||
*/
|
||||
private async embedBatchOllama(texts: string[]): Promise<number[][]> {
|
||||
const embeddings: number[][] = [];
|
||||
let failures = 0;
|
||||
|
||||
// Process in chunks of OLLAMA_EMBED_CONCURRENCY
|
||||
for (let i = 0; i < texts.length; i += OLLAMA_EMBED_CONCURRENCY) {
|
||||
const chunk = texts.slice(i, i + OLLAMA_EMBED_CONCURRENCY);
|
||||
const chunkResults = await Promise.allSettled(chunk.map((t) => this.embedOllama(t)));
|
||||
|
||||
for (let j = 0; j < chunkResults.length; j++) {
|
||||
const result = chunkResults[j];
|
||||
if (result.status === "fulfilled") {
|
||||
embeddings.push(result.value);
|
||||
} else {
|
||||
failures++;
|
||||
this.logger?.warn?.(
|
||||
`memory-neo4j: Ollama embedding failed for text ${i + j}: ${String(result.reason)}`,
|
||||
);
|
||||
// Use empty array as placeholder so indices stay aligned
|
||||
embeddings.push([]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (failures > 0) {
|
||||
this.logger?.warn?.(
|
||||
`memory-neo4j: ${failures}/${texts.length} Ollama embeddings failed in batch`,
|
||||
);
|
||||
}
|
||||
|
||||
return embeddings;
|
||||
}
|
||||
|
||||
private async embedOpenAI(text: string): Promise<number[]> {
|
||||
if (!this.client) {
|
||||
throw new Error("OpenAI client not initialized");
|
||||
}
|
||||
const response = await this.client.embeddings.create({
|
||||
model: this.model,
|
||||
input: text,
|
||||
});
|
||||
return response.data[0].embedding;
|
||||
}
|
||||
|
||||
private async embedBatchOpenAI(texts: string[]): Promise<number[][]> {
|
||||
if (!this.client) {
|
||||
throw new Error("OpenAI client not initialized");
|
||||
}
|
||||
const response = await this.client.embeddings.create({
|
||||
model: this.model,
|
||||
input: texts,
|
||||
});
|
||||
// Sort by index to ensure correct order
|
||||
return [...response.data].sort((a, b) => a.index - b.index).map((d) => d.embedding);
|
||||
}
|
||||
|
||||
// Timeout for Ollama embedding fetch calls to prevent hanging indefinitely
|
||||
private static readonly EMBED_TIMEOUT_MS = 30_000;
|
||||
// Retry configuration for transient Ollama errors (model loading, GPU pressure)
|
||||
private static readonly OLLAMA_MAX_RETRIES = 2;
|
||||
private static readonly OLLAMA_RETRY_BASE_DELAY_MS = 1000;
|
||||
|
||||
private async embedOllama(text: string): Promise<number[]> {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt <= Embeddings.OLLAMA_MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
return await this.fetchOllamaEmbedding(text);
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
if (attempt < Embeddings.OLLAMA_MAX_RETRIES) {
|
||||
const delay = Embeddings.OLLAMA_RETRY_BASE_DELAY_MS * Math.pow(2, attempt);
|
||||
this.logger?.warn?.(
|
||||
`memory-neo4j: Ollama embedding failed (attempt ${attempt + 1}/${Embeddings.OLLAMA_MAX_RETRIES + 1}), retrying in ${delay}ms: ${String(err)}`,
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
private async fetchOllamaEmbedding(text: string): Promise<number[]> {
|
||||
const url = `${this.baseUrl}/api/embed`;
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: this.model,
|
||||
input: text,
|
||||
}),
|
||||
signal: AbortSignal.timeout(Embeddings.EMBED_TIMEOUT_MS),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new Error(`Ollama embedding failed: ${response.status} ${error}`);
|
||||
}
|
||||
|
||||
const data = (await response.json()) as { embeddings?: number[][] };
|
||||
if (!data.embeddings?.[0]) {
|
||||
throw new Error("No embedding returned from Ollama");
|
||||
}
|
||||
return data.embeddings[0];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute cosine similarity between two embedding vectors.
|
||||
* Returns a value between -1 and 1 (1 = identical, 0 = orthogonal).
|
||||
* Returns 0 if either vector is empty or they differ in length.
|
||||
*/
|
||||
export function cosineSimilarity(a: number[], b: number[]): number {
|
||||
if (a.length === 0 || a.length !== b.length) {
|
||||
return 0;
|
||||
}
|
||||
let dot = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dot += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
return denom === 0 ? 0 : dot / denom;
|
||||
}
|
||||
2713
extensions/memory-neo4j/extractor.test.ts
Normal file
2713
extensions/memory-neo4j/extractor.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
715
extensions/memory-neo4j/extractor.ts
Normal file
715
extensions/memory-neo4j/extractor.ts
Normal file
@@ -0,0 +1,715 @@
|
||||
/**
|
||||
* LLM-based entity extraction and memory operations for memory-neo4j.
|
||||
*
|
||||
* Extraction uses a configurable OpenAI-compatible LLM (OpenRouter, Ollama, etc.) to:
|
||||
* - Extract entities, relationships, and tags from stored memories
|
||||
* - Classify memories into categories (preference, fact, decision, etc.)
|
||||
* - Rate memory importance on a 1-10 scale
|
||||
* - Detect semantic duplicates via LLM comparison
|
||||
* - Resolve conflicting memories
|
||||
*
|
||||
* Runs as background fire-and-forget operations with graceful degradation.
|
||||
*/
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { ExtractionConfig } from "./config.js";
|
||||
import type { Embeddings } from "./embeddings.js";
|
||||
import type { Neo4jMemoryClient } from "./neo4j-client.js";
|
||||
import type { EntityType, ExtractionResult, Logger, MemoryCategory } from "./schema.js";
|
||||
import { callOpenRouter, callOpenRouterStream, isTransientError } from "./llm-client.js";
|
||||
import { ALLOWED_RELATIONSHIP_TYPES, ENTITY_TYPES, MEMORY_CATEGORIES } from "./schema.js";
|
||||
|
||||
// ============================================================================
|
||||
// Extraction Prompt
|
||||
// ============================================================================
|
||||
|
||||
// System instruction (no user data) — user message contains the memory text
|
||||
const ENTITY_EXTRACTION_SYSTEM = `You are an entity extraction system for a personal memory store.
|
||||
Extract entities and relationships from the memory text provided by the user, and classify the memory.
|
||||
|
||||
Return JSON:
|
||||
{
|
||||
"category": "preference|fact|decision|entity|other",
|
||||
"entities": [
|
||||
{"name": "alice", "type": "person", "aliases": ["manager"], "description": "brief description"}
|
||||
],
|
||||
"relationships": [
|
||||
{"source": "alice", "target": "acme corp", "type": "WORKS_AT", "confidence": 0.95}
|
||||
],
|
||||
"tags": [
|
||||
{"name": "neo4j", "category": "technology"}
|
||||
]
|
||||
}
|
||||
|
||||
Rules:
|
||||
- Normalize entity names to lowercase
|
||||
- Entity types: person, organization, location, event, concept
|
||||
- Relationship types: WORKS_AT, LIVES_AT, KNOWS, MARRIED_TO, PREFERS, DECIDED, RELATED_TO
|
||||
- Confidence: 0.0-1.0
|
||||
- Only extract SPECIFIC named entities: real people, companies, products, tools, places, events
|
||||
- Do NOT extract generic technology terms (python, javascript, docker, linux, api, sql, html, css, json, etc.)
|
||||
- Do NOT extract generic concepts (meeting, project, training, email, code, data, server, file, script, etc.)
|
||||
- Do NOT extract programming abstractions (function, class, module, async, sync, process, etc.)
|
||||
- Good entities: "Tarun", "Abundent Academy", "Tioman Island", "LiveKit", "Neo4j", "Fish Speech S1 Mini"
|
||||
- Bad entities: "python", "ai", "automation", "email", "docker", "machine learning", "api"
|
||||
- When in doubt, do NOT extract — fewer high-quality entities beat many generic ones
|
||||
- Keep entity descriptions brief (1 sentence max)
|
||||
- Category: "preference" for opinions/preferences, "fact" for factual info, "decision" for choices made, "entity" for entity-focused, "other" for miscellaneous
|
||||
- ALWAYS generate at least 2 tags. Every memory has a topic — there are no exceptions.
|
||||
- Tags describe the TOPIC or DOMAIN of the memory, not the entities themselves.
|
||||
- Do NOT use entity names as tags (e.g., don't tag "tarun" if Tarun is already an entity).
|
||||
- Good tags: "travel planning", "family", "voice synthesis", "linkedin automation", "expense tracking", "cron scheduling", "api integration"
|
||||
- Tag categories: "topic", "domain", "workflow", "technology", "personal", "business"
|
||||
- Return empty entity/relationship arrays if nothing specific to extract, but NEVER return empty tags.`;
|
||||
|
||||
// ============================================================================
|
||||
// Retroactive Tagging Prompt
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Lightweight prompt for retroactive tagging of memories that were extracted
|
||||
* without tags. Only asks for tags — no entities or relationships.
|
||||
*/
|
||||
const RETROACTIVE_TAGGING_SYSTEM = `You are a topic tagging system for a personal memory store.
|
||||
Generate 2-4 topic tags that describe what this memory is about.
|
||||
|
||||
Return JSON:
|
||||
{
|
||||
"tags": [
|
||||
{"name": "tag name", "category": "topic|domain|workflow|technology|personal|business"}
|
||||
]
|
||||
}
|
||||
|
||||
Rules:
|
||||
- Tags describe the TOPIC or DOMAIN of the memory, not specific people or tools mentioned.
|
||||
- Good tags: "travel planning", "family", "voice synthesis", "linkedin automation", "expense tracking", "cron scheduling", "api integration", "system configuration", "memory management"
|
||||
- Bad tags: names of people, companies, or specific tools (those are entities, not topics)
|
||||
- Tag categories: "topic" (general subject), "domain" (field/area), "workflow" (process/procedure), "technology" (tech area), "personal" (personal life), "business" (work/business)
|
||||
- ALWAYS return at least 2 tags. Every memory has a topic.
|
||||
- Normalize tag names to lowercase with spaces (no hyphens or underscores).`;
|
||||
|
||||
// ============================================================================
|
||||
// Entity Extraction
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Max retries for transient extraction failures before marking permanently failed.
|
||||
*
|
||||
* Retry budget accounting — two layers of retry:
|
||||
* Layer 1: callOpenRouter/callOpenRouterStream internal retries (config.maxRetries, default 2 = 3 attempts)
|
||||
* Layer 2: Sleep cycle retries (MAX_EXTRACTION_RETRIES = 3 sleep cycles)
|
||||
* Total worst-case: 3 × 3 = 9 LLM attempts per memory
|
||||
*/
|
||||
const MAX_EXTRACTION_RETRIES = 3;
|
||||
|
||||
/**
|
||||
* Extract entities and relationships from a memory text using LLM.
|
||||
*
|
||||
* Uses streaming for responsive abort signal handling and better latency.
|
||||
*
|
||||
* Returns { result, transientFailure }:
|
||||
* - result is the ExtractionResult or null if extraction returned nothing useful
|
||||
* - transientFailure is true if the failure was due to a network/timeout issue
|
||||
* (caller should retry later) vs a permanent failure (bad JSON, etc.)
|
||||
*/
|
||||
export async function extractEntities(
|
||||
text: string,
|
||||
config: ExtractionConfig,
|
||||
abortSignal?: AbortSignal,
|
||||
): Promise<{ result: ExtractionResult | null; transientFailure: boolean }> {
|
||||
if (!config.enabled) {
|
||||
return { result: null, transientFailure: false };
|
||||
}
|
||||
|
||||
// System/user separation prevents memory text from being interpreted as instructions
|
||||
const messages = [
|
||||
{ role: "system", content: ENTITY_EXTRACTION_SYSTEM },
|
||||
{ role: "user", content: text },
|
||||
];
|
||||
|
||||
let content: string | null;
|
||||
try {
|
||||
// Use streaming for extraction — allows responsive abort and better latency
|
||||
content = await callOpenRouterStream(config, messages, abortSignal);
|
||||
} catch (err) {
|
||||
// Network/timeout errors are transient — caller should retry
|
||||
return { result: null, transientFailure: isTransientError(err) };
|
||||
}
|
||||
|
||||
if (!content) {
|
||||
return { result: null, transientFailure: false };
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(content) as Record<string, unknown>;
|
||||
return { result: validateExtractionResult(parsed), transientFailure: false };
|
||||
} catch {
|
||||
// JSON parse failure is permanent — LLM returned malformed output
|
||||
return { result: null, transientFailure: false };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract only tags from a memory text using a lightweight LLM prompt.
|
||||
* Used for retroactive tagging of memories that were extracted without tags.
|
||||
*
|
||||
* Returns an array of tags, or null on failure.
|
||||
*/
|
||||
export async function extractTagsOnly(
|
||||
text: string,
|
||||
config: ExtractionConfig,
|
||||
abortSignal?: AbortSignal,
|
||||
): Promise<Array<{ name: string; category: string }> | null> {
|
||||
if (!config.enabled) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const messages = [
|
||||
{ role: "system", content: RETROACTIVE_TAGGING_SYSTEM },
|
||||
{ role: "user", content: text },
|
||||
];
|
||||
|
||||
let content: string | null;
|
||||
try {
|
||||
content = await callOpenRouterStream(config, messages, abortSignal);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!content) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(content) as { tags?: unknown };
|
||||
const rawTags = Array.isArray(parsed.tags) ? parsed.tags : [];
|
||||
return rawTags
|
||||
.filter(
|
||||
(t: unknown): t is Record<string, unknown> =>
|
||||
t !== null &&
|
||||
typeof t === "object" &&
|
||||
typeof (t as Record<string, unknown>).name === "string",
|
||||
)
|
||||
.map((t) => ({
|
||||
name: normalizeTagName(String(t.name)),
|
||||
category: typeof t.category === "string" ? t.category : "topic",
|
||||
}))
|
||||
.filter((t) => t.name.length > 0);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a tag name: lowercase, collapse hyphens/underscores to spaces,
|
||||
* collapse multiple spaces, trim. Ensures "machine-learning", "machine_learning",
|
||||
* and "machine learning" all resolve to the same tag node.
|
||||
*/
|
||||
function normalizeTagName(name: string): string {
|
||||
return name.trim().toLowerCase().replace(/[-_]+/g, " ").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic terms that should never be extracted as entities.
|
||||
* These are common technology/concept words that the LLM tends to
|
||||
* extract despite prompt instructions. Post-filter is more reliable
|
||||
* than prompt engineering alone.
|
||||
*/
|
||||
const GENERIC_ENTITY_BLOCKLIST = new Set([
|
||||
// Programming languages & frameworks
|
||||
"python",
|
||||
"javascript",
|
||||
"typescript",
|
||||
"java",
|
||||
"go",
|
||||
"rust",
|
||||
"ruby",
|
||||
"php",
|
||||
"c",
|
||||
"c++",
|
||||
"c#",
|
||||
"swift",
|
||||
"kotlin",
|
||||
"bash",
|
||||
"shell",
|
||||
"html",
|
||||
"css",
|
||||
"sql",
|
||||
"nosql",
|
||||
"json",
|
||||
"xml",
|
||||
"yaml",
|
||||
"react",
|
||||
"vue",
|
||||
"angular",
|
||||
"svelte",
|
||||
"next.js",
|
||||
"express",
|
||||
"fastapi",
|
||||
"django",
|
||||
"flask",
|
||||
// Generic tech concepts
|
||||
"ai",
|
||||
"artificial intelligence",
|
||||
"machine learning",
|
||||
"deep learning",
|
||||
"neural network",
|
||||
"automation",
|
||||
"api",
|
||||
"rest api",
|
||||
"graphql",
|
||||
"webhook",
|
||||
"websocket",
|
||||
"database",
|
||||
"server",
|
||||
"client",
|
||||
"cloud",
|
||||
"microservice",
|
||||
"monolith",
|
||||
"frontend",
|
||||
"backend",
|
||||
"fullstack",
|
||||
"devops",
|
||||
"ci/cd",
|
||||
"deployment",
|
||||
// Generic tools/infra
|
||||
"docker",
|
||||
"kubernetes",
|
||||
"linux",
|
||||
"windows",
|
||||
"macos",
|
||||
"nginx",
|
||||
"apache",
|
||||
"git",
|
||||
"npm",
|
||||
"pnpm",
|
||||
"yarn",
|
||||
"pip",
|
||||
"node",
|
||||
"nodejs",
|
||||
"node.js",
|
||||
// Generic work concepts
|
||||
"meeting",
|
||||
"project",
|
||||
"training",
|
||||
"email",
|
||||
"calendar",
|
||||
"task",
|
||||
"ticket",
|
||||
"code",
|
||||
"data",
|
||||
"file",
|
||||
"folder",
|
||||
"directory",
|
||||
"script",
|
||||
"module",
|
||||
"debug",
|
||||
"deploy",
|
||||
"build",
|
||||
"release",
|
||||
"update",
|
||||
"upgrade",
|
||||
"user",
|
||||
"admin",
|
||||
"system",
|
||||
"service",
|
||||
"process",
|
||||
"job",
|
||||
"worker",
|
||||
// Programming abstractions
|
||||
"function",
|
||||
"class",
|
||||
"method",
|
||||
"variable",
|
||||
"object",
|
||||
"array",
|
||||
"string",
|
||||
"async",
|
||||
"sync",
|
||||
"promise",
|
||||
"callback",
|
||||
"event",
|
||||
"hook",
|
||||
"middleware",
|
||||
"component",
|
||||
"plugin",
|
||||
"extension",
|
||||
"library",
|
||||
"package",
|
||||
"dependency",
|
||||
// Generic descriptors
|
||||
"app",
|
||||
"application",
|
||||
"web",
|
||||
"mobile",
|
||||
"desktop",
|
||||
"browser",
|
||||
"config",
|
||||
"configuration",
|
||||
"settings",
|
||||
"environment",
|
||||
"production",
|
||||
"staging",
|
||||
"error",
|
||||
"bug",
|
||||
"issue",
|
||||
"fix",
|
||||
"patch",
|
||||
"feature",
|
||||
"improvement",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Validate and sanitize LLM extraction output.
|
||||
*/
|
||||
function validateExtractionResult(raw: Record<string, unknown>): ExtractionResult {
|
||||
const entities = Array.isArray(raw.entities) ? raw.entities : [];
|
||||
const relationships = Array.isArray(raw.relationships) ? raw.relationships : [];
|
||||
const tags = Array.isArray(raw.tags) ? raw.tags : [];
|
||||
|
||||
const validEntityTypes = new Set<string>(ENTITY_TYPES);
|
||||
const validCategories = new Set<string>(MEMORY_CATEGORIES);
|
||||
const rawCategory = typeof raw.category === "string" ? raw.category : undefined;
|
||||
const category =
|
||||
rawCategory && validCategories.has(rawCategory) ? (rawCategory as MemoryCategory) : undefined;
|
||||
|
||||
return {
|
||||
category,
|
||||
entities: entities
|
||||
.filter(
|
||||
(e: unknown): e is Record<string, unknown> =>
|
||||
e !== null &&
|
||||
typeof e === "object" &&
|
||||
typeof (e as Record<string, unknown>).name === "string" &&
|
||||
typeof (e as Record<string, unknown>).type === "string",
|
||||
)
|
||||
.map((e) => ({
|
||||
name: String(e.name).trim().toLowerCase(),
|
||||
type: validEntityTypes.has(String(e.type)) ? (String(e.type) as EntityType) : "concept",
|
||||
aliases: Array.isArray(e.aliases)
|
||||
? (e.aliases as unknown[])
|
||||
.filter((a): a is string => typeof a === "string")
|
||||
.map((a) => a.trim().toLowerCase())
|
||||
: undefined,
|
||||
description: typeof e.description === "string" ? e.description : undefined,
|
||||
}))
|
||||
.filter((e) => e.name.length > 0 && !GENERIC_ENTITY_BLOCKLIST.has(e.name)),
|
||||
|
||||
relationships: relationships
|
||||
.filter(
|
||||
(r: unknown): r is Record<string, unknown> =>
|
||||
r !== null &&
|
||||
typeof r === "object" &&
|
||||
typeof (r as Record<string, unknown>).source === "string" &&
|
||||
typeof (r as Record<string, unknown>).target === "string" &&
|
||||
typeof (r as Record<string, unknown>).type === "string" &&
|
||||
ALLOWED_RELATIONSHIP_TYPES.has(String((r as Record<string, unknown>).type)),
|
||||
)
|
||||
.map((r) => ({
|
||||
source: String(r.source).trim().toLowerCase(),
|
||||
target: String(r.target).trim().toLowerCase(),
|
||||
type: String(r.type),
|
||||
confidence: typeof r.confidence === "number" ? Math.min(1, Math.max(0, r.confidence)) : 0.7,
|
||||
})),
|
||||
|
||||
tags: tags
|
||||
.filter(
|
||||
(t: unknown): t is Record<string, unknown> =>
|
||||
t !== null &&
|
||||
typeof t === "object" &&
|
||||
typeof (t as Record<string, unknown>).name === "string",
|
||||
)
|
||||
.map((t) => ({
|
||||
name: normalizeTagName(String(t.name)),
|
||||
category: typeof t.category === "string" ? t.category : "topic",
|
||||
}))
|
||||
.filter((t) => t.name.length > 0),
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Conflict Resolution
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Use an LLM to determine whether two memories genuinely conflict.
|
||||
* Returns which memory to keep, or "both" if they don't actually conflict.
|
||||
* Returns "skip" on any failure (network, parse, disabled config).
|
||||
*/
|
||||
export async function resolveConflict(
|
||||
memA: string,
|
||||
memB: string,
|
||||
config: ExtractionConfig,
|
||||
abortSignal?: AbortSignal,
|
||||
): Promise<"a" | "b" | "both" | "skip"> {
|
||||
if (!config.enabled) return "skip";
|
||||
|
||||
try {
|
||||
const content = await callOpenRouter(
|
||||
config,
|
||||
[
|
||||
{
|
||||
role: "system",
|
||||
content: `Two memories may conflict with each other. Determine which should be kept.
|
||||
|
||||
If they genuinely contradict each other, keep the one that is more current, specific, or accurate.
|
||||
If they don't actually conflict (they cover different aspects or are both valid), keep both.
|
||||
|
||||
Return JSON: {"keep": "a"|"b"|"both", "reason": "brief explanation"}`,
|
||||
},
|
||||
{ role: "user", content: `Memory A: "${memA}"\nMemory B: "${memB}"` },
|
||||
],
|
||||
abortSignal,
|
||||
);
|
||||
if (!content) return "skip";
|
||||
|
||||
const parsed = JSON.parse(content) as { keep?: string };
|
||||
const keep = parsed.keep;
|
||||
if (keep === "a" || keep === "b" || keep === "both") return keep;
|
||||
return "skip";
|
||||
} catch {
|
||||
return "skip";
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Background Extraction Pipeline
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Run entity extraction in the background for a stored memory.
|
||||
* Fire-and-forget: errors are logged but never propagated.
|
||||
*
|
||||
* Flow:
|
||||
* 1. Call LLM to extract entities and relationships
|
||||
* 2. MERGE Entity nodes (idempotent)
|
||||
* 3. Create MENTIONS relationships from Memory → Entity
|
||||
* 4. Create inter-Entity relationships (WORKS_AT, KNOWS, etc.)
|
||||
* 5. Tag the memory
|
||||
* 6. Update extractionStatus to "complete", "pending" (transient retry), or "failed"
|
||||
*
|
||||
* Transient failures (network/timeout) leave status as "pending" with an incremented
|
||||
* retry counter. After MAX_EXTRACTION_RETRIES transient failures, the memory is
|
||||
* permanently marked "failed". Permanent failures (malformed JSON) are immediately "failed".
|
||||
*/
|
||||
export async function runBackgroundExtraction(
|
||||
memoryId: string,
|
||||
text: string,
|
||||
db: Neo4jMemoryClient,
|
||||
embeddings: Embeddings,
|
||||
config: ExtractionConfig,
|
||||
logger: Logger,
|
||||
currentRetries: number = 0,
|
||||
abortSignal?: AbortSignal,
|
||||
): Promise<{ success: boolean; memoryId: string }> {
|
||||
if (!config.enabled) {
|
||||
await db.updateExtractionStatus(memoryId, "skipped").catch(() => {});
|
||||
return { success: true, memoryId };
|
||||
}
|
||||
|
||||
try {
|
||||
const { result, transientFailure } = await extractEntities(text, config, abortSignal);
|
||||
|
||||
if (!result) {
|
||||
if (transientFailure) {
|
||||
// Transient failure (network/timeout) — leave as pending for retry
|
||||
const retries = currentRetries + 1;
|
||||
if (retries >= MAX_EXTRACTION_RETRIES) {
|
||||
logger.warn(
|
||||
`memory-neo4j: extraction permanently failed for ${memoryId.slice(0, 8)} after ${retries} transient retries`,
|
||||
);
|
||||
await db.updateExtractionStatus(memoryId, "failed", { incrementRetries: true });
|
||||
} else {
|
||||
logger.info(
|
||||
`memory-neo4j: extraction transient failure for ${memoryId.slice(0, 8)}, will retry (${retries}/${MAX_EXTRACTION_RETRIES})`,
|
||||
);
|
||||
// Keep status as "pending" but increment retry counter
|
||||
await db.updateExtractionStatus(memoryId, "pending", { incrementRetries: true });
|
||||
}
|
||||
} else {
|
||||
// Permanent failure (JSON parse, empty response, etc.)
|
||||
await db.updateExtractionStatus(memoryId, "failed");
|
||||
}
|
||||
return { success: false, memoryId };
|
||||
}
|
||||
|
||||
// Empty extraction is valid — not all memories have extractable entities
|
||||
if (
|
||||
result.entities.length === 0 &&
|
||||
result.relationships.length === 0 &&
|
||||
result.tags.length === 0
|
||||
) {
|
||||
await db.updateExtractionStatus(memoryId, "complete");
|
||||
return { success: true, memoryId };
|
||||
}
|
||||
|
||||
// Batch all entity operations into a single transaction:
|
||||
// entity merges, mentions, relationships, tags, category, and extraction status
|
||||
await db.batchEntityOperations(
|
||||
memoryId,
|
||||
result.entities.map((e) => ({
|
||||
id: randomUUID(),
|
||||
name: e.name,
|
||||
type: e.type,
|
||||
aliases: e.aliases,
|
||||
description: e.description,
|
||||
})),
|
||||
result.relationships,
|
||||
result.tags,
|
||||
result.category,
|
||||
);
|
||||
|
||||
logger.info(
|
||||
`memory-neo4j: extraction complete for ${memoryId.slice(0, 8)} — ` +
|
||||
`${result.entities.length} entities, ${result.relationships.length} rels, ${result.tags.length} tags` +
|
||||
(result.category ? `, category=${result.category}` : ""),
|
||||
);
|
||||
return { success: true, memoryId };
|
||||
} catch (err) {
|
||||
// Unexpected error during graph operations — treat as transient if retry budget remains
|
||||
const isTransient = isTransientError(err);
|
||||
if (isTransient && currentRetries + 1 < MAX_EXTRACTION_RETRIES) {
|
||||
logger.warn(
|
||||
`memory-neo4j: extraction transient error for ${memoryId.slice(0, 8)}, will retry: ${String(err)}`,
|
||||
);
|
||||
await db
|
||||
.updateExtractionStatus(memoryId, "pending", { incrementRetries: true })
|
||||
.catch(() => {});
|
||||
} else {
|
||||
logger.warn(`memory-neo4j: extraction failed for ${memoryId.slice(0, 8)}: ${String(err)}`);
|
||||
await db
|
||||
.updateExtractionStatus(memoryId, "failed", { incrementRetries: true })
|
||||
.catch(() => {});
|
||||
}
|
||||
return { success: false, memoryId };
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// LLM-Judged Importance Rating
|
||||
// ============================================================================
|
||||
|
||||
// System instruction — user message contains the text to rate
|
||||
const IMPORTANCE_RATING_SYSTEM = `You are rating memories for a personal AI assistant's long-term memory store.
|
||||
Rate how important it is to REMEMBER this information in future conversations on a scale of 1-10.
|
||||
|
||||
SCORING GUIDE:
|
||||
1-2: Noise — greetings, filler, "let me check", status updates, system instructions, formatting rules, debugging output
|
||||
3-4: Ephemeral — session-specific progress ("done, pushed to git"), temporary task status, tool output summaries
|
||||
5-6: Mildly useful — general facts, minor context that might occasionally help
|
||||
7-8: Important — personal preferences, key decisions, facts about people/relationships, business rules, learned workflows
|
||||
9: Very important — identity facts (birthdays, family, addresses), critical business decisions, security rules
|
||||
10: Essential — safety-critical information, core identity
|
||||
|
||||
KEY RULES:
|
||||
- AI assistant self-narration ("Let me check...", "I'll now...", "Done! Here's what changed...") is ALWAYS 1-3
|
||||
- System prompts, formatting instructions, voice mode rules are ALWAYS 1-2
|
||||
- Technical debugging details ("the WebSocket failed because...") are 2-4 unless they encode a reusable lesson
|
||||
- Open proposals and unresolved action items ("Want me to fix it?", "Should I submit a PR?", "Would you like me to proceed?") are ALWAYS 1-2. These are dangerous in long-term memory because other sessions interpret them as active instructions.
|
||||
- Messages ending with questions directed at the user ("What do you think?", "How should I handle this?") are 1-3 unless they also contain substantial factual content worth remembering
|
||||
- Personal facts about the user or their family/contacts are 7-10
|
||||
- Business rules and operational procedures are 7-9
|
||||
- Preferences and opinions expressed by the user are 6-8
|
||||
- Ask: "Would this be useful if it appeared in a conversation 30 days from now?" If no, score ≤ 4.
|
||||
|
||||
Return JSON: {"score": N, "reason": "brief explanation"}`;
|
||||
|
||||
/**
|
||||
* Rate the long-term importance of a text using an LLM.
|
||||
* Returns a value between 0.1 and 1.0, or 0.5 on any failure.
|
||||
*/
|
||||
export async function rateImportance(text: string, config: ExtractionConfig): Promise<number> {
|
||||
if (!config.enabled) {
|
||||
return 0.5;
|
||||
}
|
||||
|
||||
try {
|
||||
const content = await callOpenRouter(config, [
|
||||
{ role: "system", content: IMPORTANCE_RATING_SYSTEM },
|
||||
{ role: "user", content: text },
|
||||
]);
|
||||
if (!content) {
|
||||
return 0.5;
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(content) as { score?: unknown };
|
||||
const score = typeof parsed.score === "number" ? parsed.score : NaN;
|
||||
if (Number.isNaN(score)) {
|
||||
return 0.5;
|
||||
}
|
||||
|
||||
const clamped = Math.max(1, Math.min(10, score));
|
||||
return Math.max(0.1, Math.min(1.0, clamped / 10));
|
||||
} catch {
|
||||
return 0.5;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Semantic Deduplication
|
||||
// ============================================================================
|
||||
|
||||
// System instruction — user message contains the two texts to compare
|
||||
const SEMANTIC_DEDUP_SYSTEM = `You are a memory deduplication system. Determine whether the new text conveys the SAME factual information as the existing memory.
|
||||
|
||||
Rules:
|
||||
- Return "duplicate" if the new text is conveying the same core fact(s), even if worded differently
|
||||
- Return "duplicate" if the new text is a subset of information already in the existing memory
|
||||
- Return "unique" if the new text contains genuinely new information not in the existing memory
|
||||
- Ignore differences in formatting, pronouns, or phrasing — focus on the underlying facts
|
||||
|
||||
Return JSON: {"verdict": "duplicate"|"unique", "reason": "brief explanation"}`;
|
||||
|
||||
/**
|
||||
* Minimum cosine similarity to proceed with the LLM comparison.
|
||||
* Below this threshold, texts are too dissimilar to be semantic duplicates,
|
||||
* saving an expensive LLM call. Exported for testing.
|
||||
*/
|
||||
export const SEMANTIC_DEDUP_VECTOR_THRESHOLD = 0.8;
|
||||
|
||||
/**
|
||||
* Check whether new text is semantically a duplicate of an existing memory.
|
||||
*
|
||||
* When a pre-computed vector similarity score is provided (from findSimilar
|
||||
* or findDuplicateClusters), the LLM call is skipped entirely for pairs
|
||||
* below SEMANTIC_DEDUP_VECTOR_THRESHOLD — a fast pre-screen that avoids
|
||||
* the most expensive part of the pipeline.
|
||||
*
|
||||
* Returns true if the new text is a duplicate (should be skipped).
|
||||
* Returns false on any failure (allow storage).
|
||||
*/
|
||||
export async function isSemanticDuplicate(
|
||||
newText: string,
|
||||
existingText: string,
|
||||
config: ExtractionConfig,
|
||||
vectorSimilarity?: number,
|
||||
abortSignal?: AbortSignal,
|
||||
): Promise<boolean> {
|
||||
if (!config.enabled) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Vector pre-screen: skip LLM call when similarity is below threshold
|
||||
if (vectorSimilarity !== undefined && vectorSimilarity < SEMANTIC_DEDUP_VECTOR_THRESHOLD) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const content = await callOpenRouter(
|
||||
config,
|
||||
[
|
||||
{ role: "system", content: SEMANTIC_DEDUP_SYSTEM },
|
||||
{ role: "user", content: `Existing memory: "${existingText}"\nNew text: "${newText}"` },
|
||||
],
|
||||
abortSignal,
|
||||
);
|
||||
if (!content) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(content) as { verdict?: string };
|
||||
return parsed.verdict === "duplicate";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
754
extensions/memory-neo4j/index.test.ts
Normal file
754
extensions/memory-neo4j/index.test.ts
Normal file
@@ -0,0 +1,754 @@
|
||||
/**
|
||||
* Tests for the memory-neo4j plugin entry point.
|
||||
*
|
||||
* Covers:
|
||||
* 1. Attention gates (user and assistant) — re-exported from attention-gate.ts
|
||||
* 2. Message extraction — extractUserMessages, extractAssistantMessages from message-utils.ts
|
||||
* 3. Strip wrappers — stripMessageWrappers, stripAssistantWrappers from message-utils.ts
|
||||
*
|
||||
* Does NOT test the plugin registration or CLI commands (those require the
|
||||
* full OpenClaw SDK runtime). Focuses on pure functions and the behavioral
|
||||
* contracts of the auto-capture pipeline helpers.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { passesAttentionGate, passesAssistantAttentionGate } from "./attention-gate.js";
|
||||
import {
|
||||
extractUserMessages,
|
||||
extractAssistantMessages,
|
||||
stripMessageWrappers,
|
||||
stripAssistantWrappers,
|
||||
} from "./message-utils.js";
|
||||
|
||||
// ============================================================================
|
||||
// Test Helpers
|
||||
// ============================================================================
|
||||
|
||||
/** Generate a string of a specific length using a repeating word pattern. */
|
||||
function makeText(wordCount: number, word = "lorem"): string {
|
||||
return Array.from({ length: wordCount }, () => word).join(" ");
|
||||
}
|
||||
|
||||
/** Generate a string of a specific character length. */
|
||||
function makeChars(charCount: number, char = "x"): string {
|
||||
return char.repeat(charCount);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// passesAttentionGate() — User Attention Gate
|
||||
// ============================================================================
|
||||
|
||||
describe("passesAttentionGate", () => {
|
||||
// -----------------------------------------------------------------------
|
||||
// Length bounds
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe("length bounds", () => {
|
||||
it("should reject messages shorter than 30 characters", () => {
|
||||
expect(passesAttentionGate("too short")).toBe(false);
|
||||
expect(passesAttentionGate("a".repeat(29))).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject messages longer than 2000 characters", () => {
|
||||
// 2001 chars — exceeds MAX_CAPTURE_CHARS
|
||||
const longText = makeText(300, "longword");
|
||||
expect(longText.length).toBeGreaterThan(2000);
|
||||
expect(passesAttentionGate(longText)).toBe(false);
|
||||
});
|
||||
|
||||
it("should accept messages at exactly 30 characters with sufficient words", () => {
|
||||
// Need 30+ chars and 8+ words
|
||||
const text = "ab cd ef gh ij kl mn op qr st u";
|
||||
expect(text.length).toBeGreaterThanOrEqual(30);
|
||||
expect(text.split(/\s+/).length).toBeGreaterThanOrEqual(8);
|
||||
expect(passesAttentionGate(text)).toBe(true);
|
||||
});
|
||||
|
||||
it("should accept messages at exactly 2000 characters with sufficient words", () => {
|
||||
// Build exactly 2000 chars: repeated "testing " (8 chars each) = 250 words
|
||||
// 250 * 8 = 2000, but join adds spaces between (not after last), so 250 * 7 + 249 = 1999
|
||||
// Use a padded approach: fill with "testing " then pad to exactly 2000
|
||||
const base = "testing ".repeat(249) + "testing"; // 249*8 + 7 = 1999
|
||||
const text = base + "s"; // 2000 chars
|
||||
expect(text.length).toBe(2000);
|
||||
expect(passesAttentionGate(text)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Word count
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe("word count", () => {
|
||||
it("should reject messages with fewer than 8 words", () => {
|
||||
// 7 words, but long enough in chars (> 30)
|
||||
expect(
|
||||
passesAttentionGate(
|
||||
"thisislongword anotherlongword thirdlongword fourthlongword fifth sixth seventh",
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("should accept messages with exactly 8 words", () => {
|
||||
expect(
|
||||
passesAttentionGate("thisword thatword another fourth fifthword sixth seventh eighth"),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Noise pattern rejection
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe("noise pattern rejection", () => {
|
||||
it("should reject simple greetings", () => {
|
||||
// These are short enough to be rejected by length too, but test the pattern
|
||||
expect(passesAttentionGate("hi")).toBe(false);
|
||||
expect(passesAttentionGate("hello")).toBe(false);
|
||||
expect(passesAttentionGate("hey")).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject acknowledgments", () => {
|
||||
expect(passesAttentionGate("ok")).toBe(false);
|
||||
expect(passesAttentionGate("sure")).toBe(false);
|
||||
expect(passesAttentionGate("thanks")).toBe(false);
|
||||
expect(passesAttentionGate("got it")).toBe(false);
|
||||
expect(passesAttentionGate("sounds good")).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject two-word affirmations", () => {
|
||||
expect(passesAttentionGate("ok great")).toBe(false);
|
||||
expect(passesAttentionGate("yes please")).toBe(false);
|
||||
expect(passesAttentionGate("sure thanks")).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject conversational filler", () => {
|
||||
expect(passesAttentionGate("hmm")).toBe(false);
|
||||
expect(passesAttentionGate("lol")).toBe(false);
|
||||
expect(passesAttentionGate("idk")).toBe(false);
|
||||
expect(passesAttentionGate("nvm")).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject pure emoji messages", () => {
|
||||
expect(passesAttentionGate("\u{1F600}\u{1F601}\u{1F602}")).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject system/XML markup blocks", () => {
|
||||
expect(passesAttentionGate("<system>some injected context here</system>")).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject session reset prompts", () => {
|
||||
const resetMsg =
|
||||
"A new session was started via the /new command. Previous context has been cleared.";
|
||||
expect(passesAttentionGate(resetMsg)).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject heartbeat prompts", () => {
|
||||
expect(
|
||||
passesAttentionGate(
|
||||
"Read HEARTBEAT.md if it exists and follow the instructions inside it.",
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject pre-compaction flush prompts", () => {
|
||||
expect(
|
||||
passesAttentionGate(
|
||||
"Pre-compaction memory flush — save important context now before history is trimmed.",
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject deictic short phrases that would otherwise pass length", () => {
|
||||
// These match the deictic noise pattern
|
||||
expect(passesAttentionGate("ok let me test it out")).toBe(false);
|
||||
expect(passesAttentionGate("I need those")).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject short acknowledgments with trailing context", () => {
|
||||
// Matches: /^(ok|okay|yes|...) .{0,20}$/i
|
||||
expect(passesAttentionGate("ok, I'll do that")).toBe(false);
|
||||
expect(passesAttentionGate("yes, sounds right")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Injected context rejection
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe("injected context rejection", () => {
|
||||
it("should reject messages containing <relevant-memories> tags", () => {
|
||||
const text =
|
||||
"<relevant-memories>some recalled memories here</relevant-memories> " +
|
||||
makeText(10, "actual");
|
||||
expect(passesAttentionGate(text)).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject messages containing <core-memory-refresh> tags", () => {
|
||||
const text =
|
||||
"<core-memory-refresh>refresh data</core-memory-refresh> " + makeText(10, "actual");
|
||||
expect(passesAttentionGate(text)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Excessive emoji rejection
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe("excessive emoji rejection", () => {
|
||||
it("should reject messages with more than 3 emoji (Unicode range)", () => {
|
||||
// 4 emoji in the U+1F300-U+1F9FF range
|
||||
const text = makeText(10, "word") + " \u{1F600}\u{1F601}\u{1F602}\u{1F603}";
|
||||
expect(passesAttentionGate(text)).toBe(false);
|
||||
});
|
||||
|
||||
it("should accept messages with 3 or fewer emoji", () => {
|
||||
const text = makeText(10, "testing") + " \u{1F600}\u{1F601}\u{1F602}";
|
||||
expect(passesAttentionGate(text)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Substantive messages that should pass
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe("substantive messages", () => {
|
||||
it("should accept a clear factual statement", () => {
|
||||
expect(passesAttentionGate("I prefer dark mode for all my code editors and terminals")).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("should accept a preference statement", () => {
|
||||
expect(
|
||||
passesAttentionGate(
|
||||
"My favorite programming language is TypeScript because of its type system",
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("should accept a decision statement", () => {
|
||||
expect(
|
||||
passesAttentionGate(
|
||||
"We decided to use Neo4j for the knowledge graph instead of PostgreSQL",
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("should accept a multi-sentence message", () => {
|
||||
expect(
|
||||
passesAttentionGate(
|
||||
"The deployment pipeline uses GitHub Actions. It builds and tests on every push to main.",
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("should handle leading/trailing whitespace via trimming", () => {
|
||||
expect(
|
||||
passesAttentionGate(" I prefer using vitest for testing my TypeScript projects "),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// passesAssistantAttentionGate() — Assistant Attention Gate
|
||||
// ============================================================================
|
||||
|
||||
describe("passesAssistantAttentionGate", () => {
|
||||
// -----------------------------------------------------------------------
|
||||
// Length bounds (stricter than user)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe("length bounds", () => {
|
||||
it("should reject messages shorter than 30 characters", () => {
|
||||
expect(passesAssistantAttentionGate("short msg")).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject messages longer than 1000 characters", () => {
|
||||
const longText = makeText(200, "wordword");
|
||||
expect(longText.length).toBeGreaterThan(1000);
|
||||
expect(passesAssistantAttentionGate(longText)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Word count (higher threshold — 10 words minimum)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe("word count", () => {
|
||||
it("should reject messages with fewer than 10 words", () => {
|
||||
// 9 words, each 5 chars + space = more than 30 chars total
|
||||
const nineWords = "alpha bravo charm delta eerie found ghost horse india";
|
||||
expect(nineWords.split(/\s+/).length).toBe(9);
|
||||
expect(nineWords.length).toBeGreaterThan(30);
|
||||
expect(passesAssistantAttentionGate(nineWords)).toBe(false);
|
||||
});
|
||||
|
||||
it("should accept messages with exactly 10 words", () => {
|
||||
const tenWords = "alpha bravo charm delta eerie found ghost horse india julep";
|
||||
expect(tenWords.split(/\s+/).length).toBe(10);
|
||||
expect(tenWords.length).toBeGreaterThan(30);
|
||||
expect(passesAssistantAttentionGate(tenWords)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Code-heavy message rejection (> 50% fenced code)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe("code-heavy rejection", () => {
|
||||
it("should reject messages that are more than 50% fenced code blocks", () => {
|
||||
// ~60 chars of prose + ~200 chars of code block => code > 50%
|
||||
const text =
|
||||
"Here is some explanation for the code below that follows.\n" +
|
||||
"```typescript\n" +
|
||||
"function example() {\n" +
|
||||
" const x = 1;\n" +
|
||||
" const y = 2;\n" +
|
||||
" return x + y;\n" +
|
||||
"}\n" +
|
||||
"function another() {\n" +
|
||||
" const a = 3;\n" +
|
||||
" return a * 2;\n" +
|
||||
"}\n" +
|
||||
"```";
|
||||
expect(passesAssistantAttentionGate(text)).toBe(false);
|
||||
});
|
||||
|
||||
it("should accept messages with less than 50% code", () => {
|
||||
const text =
|
||||
"The configuration requires setting up the environment variables correctly. " +
|
||||
"You need to set NEO4J_URI, NEO4J_USER, and NEO4J_PASSWORD. " +
|
||||
"Make sure the password is at least 8 characters long for security. " +
|
||||
"```\nNEO4J_URI=bolt://localhost:7687\n```";
|
||||
expect(passesAssistantAttentionGate(text)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Tool output rejection
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe("tool output rejection", () => {
|
||||
it("should reject messages containing <tool_result> tags", () => {
|
||||
const text =
|
||||
"Here is the result of the search query across all the relevant documents " +
|
||||
"<tool_result>some result data here</tool_result>";
|
||||
expect(passesAssistantAttentionGate(text)).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject messages containing <tool_use> tags", () => {
|
||||
const text =
|
||||
"I will use this tool to help answer your question about the system setup " +
|
||||
"<tool_use>tool invocation here</tool_use>";
|
||||
expect(passesAssistantAttentionGate(text)).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject messages containing <function_call> tags", () => {
|
||||
const text =
|
||||
"Calling the function to retrieve the relevant data from the database now " +
|
||||
"<function_call>fn call here</function_call>";
|
||||
expect(passesAssistantAttentionGate(text)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Injected context rejection
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe("injected context rejection", () => {
|
||||
it("should reject messages with <relevant-memories> tags", () => {
|
||||
const text =
|
||||
"<relevant-memories>cached recall data</relevant-memories> " + makeText(15, "answer");
|
||||
expect(passesAssistantAttentionGate(text)).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject messages with <core-memory-refresh> tags", () => {
|
||||
const text =
|
||||
"<core-memory-refresh>identity refresh</core-memory-refresh> " + makeText(15, "answer");
|
||||
expect(passesAssistantAttentionGate(text)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Noise patterns and emoji (shared with user gate)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe("noise patterns", () => {
|
||||
it("should reject greeting noise", () => {
|
||||
expect(passesAssistantAttentionGate("hello")).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject excessive emoji", () => {
|
||||
const text = makeText(15, "answer") + " \u{1F600}\u{1F601}\u{1F602}\u{1F603}";
|
||||
expect(passesAssistantAttentionGate(text)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Substantive assistant messages that should pass
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe("substantive assistant messages", () => {
|
||||
it("should accept a clear explanatory response", () => {
|
||||
expect(
|
||||
passesAssistantAttentionGate(
|
||||
"The Neo4j database uses a property graph model where nodes represent entities and edges represent relationships between them.",
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("should accept a recommendation response", () => {
|
||||
expect(
|
||||
passesAssistantAttentionGate(
|
||||
"Based on your requirements, I recommend using vitest for unit testing because it has native TypeScript support and fast execution times.",
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// extractUserMessages()
|
||||
// ============================================================================
|
||||
|
||||
describe("extractUserMessages", () => {
|
||||
it("should extract text from string content format", () => {
|
||||
const messages = [{ role: "user", content: "This is a substantive user message for testing" }];
|
||||
const result = extractUserMessages(messages);
|
||||
expect(result).toEqual(["This is a substantive user message for testing"]);
|
||||
});
|
||||
|
||||
it("should extract text from content block array format", () => {
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "This is a substantive user message from a block array" }],
|
||||
},
|
||||
];
|
||||
const result = extractUserMessages(messages);
|
||||
expect(result).toEqual(["This is a substantive user message from a block array"]);
|
||||
});
|
||||
|
||||
it("should extract multiple text blocks from a single message", () => {
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "First text block with enough characters" },
|
||||
{ type: "image", url: "http://example.com/img.png" },
|
||||
{ type: "text", text: "Second text block with enough characters" },
|
||||
],
|
||||
},
|
||||
];
|
||||
const result = extractUserMessages(messages);
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0]).toBe("First text block with enough characters");
|
||||
expect(result[1]).toBe("Second text block with enough characters");
|
||||
});
|
||||
|
||||
it("should ignore non-user messages", () => {
|
||||
const messages = [
|
||||
{ role: "assistant", content: "I am the assistant response message here" },
|
||||
{ role: "system", content: "This is the system prompt configuration text" },
|
||||
{ role: "user", content: "This is the actual user message text here" },
|
||||
];
|
||||
const result = extractUserMessages(messages);
|
||||
expect(result).toEqual(["This is the actual user message text here"]);
|
||||
});
|
||||
|
||||
it("should filter out messages shorter than 10 characters after stripping", () => {
|
||||
const messages = [
|
||||
{ role: "user", content: "short" },
|
||||
{ role: "user", content: "This is a long enough message to pass the filter" },
|
||||
];
|
||||
const result = extractUserMessages(messages);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0]).toBe("This is a long enough message to pass the filter");
|
||||
});
|
||||
|
||||
it("should strip Telegram wrappers before returning", () => {
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content:
|
||||
"[Telegram @user123 in group] The actual user message is right here\n[message_id: 456]",
|
||||
},
|
||||
];
|
||||
const result = extractUserMessages(messages);
|
||||
expect(result).toEqual(["The actual user message is right here"]);
|
||||
});
|
||||
|
||||
it("should strip Slack wrappers before returning", () => {
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content:
|
||||
"[Slack workspace #channel @user] The actual user message text goes here\n[slack message id: abc123]",
|
||||
},
|
||||
];
|
||||
const result = extractUserMessages(messages);
|
||||
expect(result).toEqual(["The actual user message text goes here"]);
|
||||
});
|
||||
|
||||
it("should strip injected <relevant-memories> context", () => {
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content:
|
||||
"<relevant-memories>recalled: user likes dark mode</relevant-memories> What editor do you recommend for me?",
|
||||
},
|
||||
];
|
||||
const result = extractUserMessages(messages);
|
||||
expect(result).toEqual(["What editor do you recommend for me?"]);
|
||||
});
|
||||
|
||||
it("should handle null and non-object entries gracefully", () => {
|
||||
const messages = [
|
||||
null,
|
||||
undefined,
|
||||
42,
|
||||
"string",
|
||||
{ role: "user", content: "This is a valid message with enough text" },
|
||||
];
|
||||
const result = extractUserMessages(messages as unknown[]);
|
||||
expect(result).toEqual(["This is a valid message with enough text"]);
|
||||
});
|
||||
|
||||
it("should handle empty messages array", () => {
|
||||
expect(extractUserMessages([])).toEqual([]);
|
||||
});
|
||||
|
||||
it("should ignore content blocks that are not type 'text'", () => {
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image", url: "http://example.com/photo.jpg" },
|
||||
{ type: "audio", data: "base64data..." },
|
||||
],
|
||||
},
|
||||
];
|
||||
const result = extractUserMessages(messages);
|
||||
expect(result).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// extractAssistantMessages()
|
||||
// ============================================================================
|
||||
|
||||
describe("extractAssistantMessages", () => {
|
||||
it("should extract text from string content format", () => {
|
||||
const messages = [
|
||||
{ role: "assistant", content: "Here is a substantive assistant response text" },
|
||||
];
|
||||
const result = extractAssistantMessages(messages);
|
||||
expect(result).toEqual(["Here is a substantive assistant response text"]);
|
||||
});
|
||||
|
||||
it("should extract text from content block array format", () => {
|
||||
const messages = [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "The assistant provides an answer to your question here" }],
|
||||
},
|
||||
];
|
||||
const result = extractAssistantMessages(messages);
|
||||
expect(result).toEqual(["The assistant provides an answer to your question here"]);
|
||||
});
|
||||
|
||||
it("should ignore non-assistant messages", () => {
|
||||
const messages = [
|
||||
{ role: "user", content: "This is a user message that should be ignored" },
|
||||
{ role: "assistant", content: "This is the assistant response message here" },
|
||||
];
|
||||
const result = extractAssistantMessages(messages);
|
||||
expect(result).toEqual(["This is the assistant response message here"]);
|
||||
});
|
||||
|
||||
it("should filter out messages shorter than 10 characters after stripping", () => {
|
||||
const messages = [
|
||||
{ role: "assistant", content: "short" },
|
||||
{ role: "assistant", content: "This is a long enough assistant response message" },
|
||||
];
|
||||
const result = extractAssistantMessages(messages);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0]).toBe("This is a long enough assistant response message");
|
||||
});
|
||||
|
||||
it("should strip tool-use blocks from assistant messages", () => {
|
||||
const messages = [
|
||||
{
|
||||
role: "assistant",
|
||||
content:
|
||||
"<tool_use>search function call parameters</tool_use>Here is the answer to your question about configuration",
|
||||
},
|
||||
];
|
||||
const result = extractAssistantMessages(messages);
|
||||
expect(result).toEqual(["Here is the answer to your question about configuration"]);
|
||||
});
|
||||
|
||||
it("should strip tool_result blocks from assistant messages", () => {
|
||||
const messages = [
|
||||
{
|
||||
role: "assistant",
|
||||
content:
|
||||
"The query returned: <tool_result>raw database output here</tool_result> which means the config is correct and working.",
|
||||
},
|
||||
];
|
||||
const result = extractAssistantMessages(messages);
|
||||
expect(result).toEqual(["The query returned: which means the config is correct and working."]);
|
||||
});
|
||||
|
||||
it("should strip thinking blocks from assistant messages", () => {
|
||||
const messages = [
|
||||
{
|
||||
role: "assistant",
|
||||
content:
|
||||
"<thinking>I need to figure out the best approach here</thinking>The best approach is to use a hybrid search combining vector and BM25 signals.",
|
||||
},
|
||||
];
|
||||
const result = extractAssistantMessages(messages);
|
||||
expect(result).toEqual([
|
||||
"The best approach is to use a hybrid search combining vector and BM25 signals.",
|
||||
]);
|
||||
});
|
||||
|
||||
it("should strip code_output blocks from assistant messages", () => {
|
||||
const messages = [
|
||||
{
|
||||
role: "assistant",
|
||||
content:
|
||||
"I ran the code: <code_output>stdout: success</code_output> and it completed without any errors at all.",
|
||||
},
|
||||
];
|
||||
const result = extractAssistantMessages(messages);
|
||||
expect(result).toEqual(["I ran the code: and it completed without any errors at all."]);
|
||||
});
|
||||
|
||||
it("should handle null and non-object entries gracefully", () => {
|
||||
const messages = [
|
||||
null,
|
||||
undefined,
|
||||
{ role: "assistant", content: "This is a valid assistant response text" },
|
||||
];
|
||||
const result = extractAssistantMessages(messages as unknown[]);
|
||||
expect(result).toEqual(["This is a valid assistant response text"]);
|
||||
});
|
||||
|
||||
it("should handle empty messages array", () => {
|
||||
expect(extractAssistantMessages([])).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// stripMessageWrappers()
|
||||
// ============================================================================
|
||||
|
||||
describe("stripMessageWrappers", () => {
|
||||
it("should strip <relevant-memories> tags and content", () => {
|
||||
const input =
|
||||
"<relevant-memories>user likes dark mode</relevant-memories> What editor should I use?";
|
||||
expect(stripMessageWrappers(input)).toBe("What editor should I use?");
|
||||
});
|
||||
|
||||
it("should strip <core-memory-refresh> tags and content", () => {
|
||||
const input =
|
||||
"<core-memory-refresh>identity: Tarun</core-memory-refresh> How do I configure this?";
|
||||
expect(stripMessageWrappers(input)).toBe("How do I configure this?");
|
||||
});
|
||||
|
||||
it("should strip <system> tags and content", () => {
|
||||
const input = "<system>You are a helpful assistant.</system> What is the weather?";
|
||||
expect(stripMessageWrappers(input)).toBe("What is the weather?");
|
||||
});
|
||||
|
||||
it("should strip <file> attachment tags", () => {
|
||||
const input = '<file name="doc.pdf">base64content</file> Summarize this document for me.';
|
||||
expect(stripMessageWrappers(input)).toBe("Summarize this document for me.");
|
||||
});
|
||||
|
||||
it("should strip Telegram wrapper and message_id", () => {
|
||||
const input = "[Telegram @john in private] Please remember my preference\n[message_id: 12345]";
|
||||
expect(stripMessageWrappers(input)).toBe("Please remember my preference");
|
||||
});
|
||||
|
||||
it("should strip Slack wrapper and slack message id", () => {
|
||||
const input =
|
||||
"[Slack acme-corp #general @alice] Please deploy the latest build\n[slack message id: ts-123]";
|
||||
expect(stripMessageWrappers(input)).toBe("Please deploy the latest build");
|
||||
});
|
||||
|
||||
it("should strip media attachment preamble", () => {
|
||||
const input =
|
||||
"[media attached: image/jpeg]\nTo send an image reply with...\n[Telegram @user in private] What is this picture?";
|
||||
expect(stripMessageWrappers(input)).toBe("What is this picture?");
|
||||
});
|
||||
|
||||
it("should strip System exec output blocks before Telegram wrapper", () => {
|
||||
const input =
|
||||
"System: [2024-01-01] exec completed\n[Telegram @user in private] What happened with the deploy?";
|
||||
expect(stripMessageWrappers(input)).toBe("What happened with the deploy?");
|
||||
});
|
||||
|
||||
it("should handle multiple wrappers in one message", () => {
|
||||
const input =
|
||||
"<relevant-memories>recalled facts</relevant-memories> <system>You are helpful.</system> [Telegram @user in group] What is up?";
|
||||
const result = stripMessageWrappers(input);
|
||||
expect(result).toBe("What is up?");
|
||||
});
|
||||
|
||||
it("should return trimmed text when no wrappers are present", () => {
|
||||
expect(stripMessageWrappers(" Just a plain message ")).toBe("Just a plain message");
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// stripAssistantWrappers()
|
||||
// ============================================================================
|
||||
|
||||
describe("stripAssistantWrappers", () => {
|
||||
it("should strip <tool_use> blocks", () => {
|
||||
const input = "<tool_use>call search</tool_use>The answer is 42.";
|
||||
expect(stripAssistantWrappers(input)).toBe("The answer is 42.");
|
||||
});
|
||||
|
||||
it("should strip <tool_result> blocks", () => {
|
||||
const input = "Result: <tool_result>raw output</tool_result> processed successfully.";
|
||||
// The regex consumes trailing whitespace after the closing tag
|
||||
expect(stripAssistantWrappers(input)).toBe("Result: processed successfully.");
|
||||
});
|
||||
|
||||
it("should strip <function_call> blocks", () => {
|
||||
const input = "<function_call>fn(args)</function_call>Done with the operation.";
|
||||
expect(stripAssistantWrappers(input)).toBe("Done with the operation.");
|
||||
});
|
||||
|
||||
it("should strip <thinking> blocks", () => {
|
||||
const input = "<thinking>Let me consider...</thinking>I recommend using vitest.";
|
||||
expect(stripAssistantWrappers(input)).toBe("I recommend using vitest.");
|
||||
});
|
||||
|
||||
it("should strip <antThinking> blocks", () => {
|
||||
const input = "<antThinking>analyzing the request</antThinking>Here is the analysis.";
|
||||
expect(stripAssistantWrappers(input)).toBe("Here is the analysis.");
|
||||
});
|
||||
|
||||
it("should strip <code_output> blocks", () => {
|
||||
const input = "Output: <code_output>success</code_output> everything worked.";
|
||||
// The regex consumes trailing whitespace after the closing tag
|
||||
expect(stripAssistantWrappers(input)).toBe("Output: everything worked.");
|
||||
});
|
||||
|
||||
it("should strip multiple wrapper types in one message", () => {
|
||||
const input =
|
||||
"<thinking>hmm</thinking><tool_use>search</tool_use>The final answer is here.<tool_result>data</tool_result>";
|
||||
expect(stripAssistantWrappers(input)).toBe("The final answer is here.");
|
||||
});
|
||||
|
||||
it("should return trimmed text when no wrappers are present", () => {
|
||||
expect(stripAssistantWrappers(" Plain assistant text ")).toBe("Plain assistant text");
|
||||
});
|
||||
});
|
||||
1044
extensions/memory-neo4j/index.ts
Normal file
1044
extensions/memory-neo4j/index.ts
Normal file
File diff suppressed because it is too large
Load Diff
194
extensions/memory-neo4j/llm-client.ts
Normal file
194
extensions/memory-neo4j/llm-client.ts
Normal file
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* OpenRouter/OpenAI-compatible LLM API client for memory-neo4j.
|
||||
*
|
||||
* Handles non-streaming and streaming chat completion requests with
|
||||
* retry logic, timeout handling, and abort signal support.
|
||||
*/
|
||||
|
||||
import type { ExtractionConfig } from "./config.js";
|
||||
|
||||
// Timeout for LLM and embedding fetch calls to prevent hanging indefinitely
|
||||
export const FETCH_TIMEOUT_MS = 30_000;
|
||||
|
||||
/**
|
||||
* Build a combined abort signal from the caller's signal and a per-request timeout.
|
||||
*/
|
||||
function buildSignal(abortSignal?: AbortSignal): AbortSignal {
|
||||
return abortSignal
|
||||
? AbortSignal.any([abortSignal, AbortSignal.timeout(FETCH_TIMEOUT_MS)])
|
||||
: AbortSignal.timeout(FETCH_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared request/retry logic for OpenRouter API calls.
|
||||
* Handles signal composition, request building, error handling, and exponential backoff.
|
||||
* The `parseFn` callback processes the Response differently for streaming vs non-streaming.
|
||||
*/
|
||||
async function openRouterRequest(
|
||||
config: ExtractionConfig,
|
||||
messages: Array<{ role: string; content: string }>,
|
||||
abortSignal: AbortSignal | undefined,
|
||||
stream: boolean,
|
||||
parseFn: (response: Response, abortSignal?: AbortSignal) => Promise<string | null>,
|
||||
): Promise<string | null> {
|
||||
for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
|
||||
try {
|
||||
const signal = buildSignal(abortSignal);
|
||||
|
||||
const response = await fetch(`${config.baseUrl}/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${config.apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: config.model,
|
||||
messages,
|
||||
temperature: config.temperature,
|
||||
response_format: { type: "json_object" },
|
||||
...(stream ? { stream: true } : {}),
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => "");
|
||||
throw new Error(`OpenRouter API error ${response.status}: ${body}`);
|
||||
}
|
||||
|
||||
return await parseFn(response, abortSignal);
|
||||
} catch (err) {
|
||||
if (attempt >= config.maxRetries) {
|
||||
throw err;
|
||||
}
|
||||
// Exponential backoff
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 * 2 ** attempt));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a non-streaming JSON response.
|
||||
*/
|
||||
function parseNonStreaming(response: Response): Promise<string | null> {
|
||||
return response.json().then((data: unknown) => {
|
||||
const typed = data as {
|
||||
choices?: Array<{ message?: { content?: string } }>;
|
||||
};
|
||||
return typed.choices?.[0]?.message?.content ?? null;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a streaming SSE response, accumulating chunks into a single string.
|
||||
*/
|
||||
async function parseStreaming(
|
||||
response: Response,
|
||||
abortSignal?: AbortSignal,
|
||||
): Promise<string | null> {
|
||||
if (!response.body) {
|
||||
throw new Error("No response body for streaming request");
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let accumulated = "";
|
||||
let buffer = "";
|
||||
|
||||
for (;;) {
|
||||
// Check abort between chunks for responsive cancellation
|
||||
if (abortSignal?.aborted) {
|
||||
reader.cancel().catch(() => {});
|
||||
return null;
|
||||
}
|
||||
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
|
||||
// Parse SSE lines
|
||||
const lines = buffer.split("\n");
|
||||
buffer = lines.pop() ?? "";
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed.startsWith("data: ")) continue;
|
||||
const data = trimmed.slice(6);
|
||||
if (data === "[DONE]") continue;
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(data) as {
|
||||
choices?: Array<{ delta?: { content?: string } }>;
|
||||
};
|
||||
const chunk = parsed.choices?.[0]?.delta?.content;
|
||||
if (chunk) {
|
||||
accumulated += chunk;
|
||||
}
|
||||
} catch {
|
||||
// Skip malformed SSE chunks
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return accumulated || null;
|
||||
}
|
||||
|
||||
export async function callOpenRouter(
|
||||
config: ExtractionConfig,
|
||||
prompt: string | Array<{ role: string; content: string }>,
|
||||
abortSignal?: AbortSignal,
|
||||
): Promise<string | null> {
|
||||
const messages = typeof prompt === "string" ? [{ role: "user", content: prompt }] : prompt;
|
||||
return openRouterRequest(config, messages, abortSignal, false, parseNonStreaming);
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming variant of callOpenRouter. Uses the streaming API to receive chunks
|
||||
* incrementally, allowing earlier cancellation via abort signal and better
|
||||
* latency characteristics for long responses.
|
||||
*
|
||||
* Accumulates all chunks into a single response string since extraction
|
||||
* uses JSON mode (which requires the complete object to parse).
|
||||
*/
|
||||
export async function callOpenRouterStream(
|
||||
config: ExtractionConfig,
|
||||
prompt: string | Array<{ role: string; content: string }>,
|
||||
abortSignal?: AbortSignal,
|
||||
): Promise<string | null> {
|
||||
const messages = typeof prompt === "string" ? [{ role: "user", content: prompt }] : prompt;
|
||||
return openRouterRequest(config, messages, abortSignal, true, parseStreaming);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an error is transient (network/timeout) vs permanent (JSON parse, etc.)
|
||||
*/
|
||||
export function isTransientError(err: unknown): boolean {
|
||||
if (!err || typeof err !== "object") {
|
||||
return false;
|
||||
}
|
||||
const name =
|
||||
typeof (err as { name?: unknown }).name === "string" ? (err as { name: string }).name : "";
|
||||
const message =
|
||||
typeof (err as { message?: unknown }).message === "string"
|
||||
? (err as { message: string }).message
|
||||
: "";
|
||||
const msg = message.toLowerCase();
|
||||
return (
|
||||
name === "AbortError" ||
|
||||
name === "TimeoutError" ||
|
||||
msg.includes("timeout") ||
|
||||
msg.includes("econnrefused") ||
|
||||
msg.includes("econnreset") ||
|
||||
msg.includes("etimedout") ||
|
||||
msg.includes("enotfound") ||
|
||||
msg.includes("network") ||
|
||||
msg.includes("fetch failed") ||
|
||||
msg.includes("socket hang up") ||
|
||||
msg.includes("api error 429") ||
|
||||
msg.includes("api error 502") ||
|
||||
msg.includes("api error 503") ||
|
||||
msg.includes("api error 504")
|
||||
);
|
||||
}
|
||||
135
extensions/memory-neo4j/message-utils.ts
Normal file
135
extensions/memory-neo4j/message-utils.ts
Normal file
@@ -0,0 +1,135 @@
|
||||
/**
|
||||
* Message extraction utilities for the memory pipeline.
|
||||
*
|
||||
* Extracts and cleans user/assistant messages from the raw event.messages
|
||||
* array, stripping channel wrappers, injected context, tool output, and
|
||||
* other noise so downstream consumers (attention gate, memory store) see
|
||||
* only the substantive text.
|
||||
*/
|
||||
|
||||
// ============================================================================
|
||||
// Core Extraction
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Extract text blocks from messages with a given role, apply a strip function,
|
||||
* and filter out short results. Handles both string content and content block arrays.
|
||||
*/
|
||||
function extractMessagesByRole(
|
||||
messages: unknown[],
|
||||
role: string,
|
||||
stripFn: (text: string) => string,
|
||||
): string[] {
|
||||
const texts: string[] = [];
|
||||
|
||||
for (const msg of messages) {
|
||||
if (!msg || typeof msg !== "object") {
|
||||
continue;
|
||||
}
|
||||
const msgObj = msg as Record<string, unknown>;
|
||||
|
||||
if (msgObj.role !== role) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const content = msgObj.content;
|
||||
if (typeof content === "string") {
|
||||
texts.push(content);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Array.isArray(content)) {
|
||||
for (const block of content) {
|
||||
if (
|
||||
block &&
|
||||
typeof block === "object" &&
|
||||
"type" in block &&
|
||||
(block as Record<string, unknown>).type === "text" &&
|
||||
"text" in block &&
|
||||
typeof (block as Record<string, unknown>).text === "string"
|
||||
) {
|
||||
texts.push((block as Record<string, unknown>).text as string);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return texts.map(stripFn).filter((t) => t.length >= 10);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// User Message Extraction
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Extract user message texts from the event.messages array.
|
||||
*/
|
||||
export function extractUserMessages(messages: unknown[]): string[] {
|
||||
return extractMessagesByRole(messages, "user", stripMessageWrappers);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip injected context, channel metadata wrappers, and system prefixes
|
||||
* so the attention gate sees only the raw user text.
|
||||
* Exported for use by the cleanup command.
|
||||
*/
|
||||
export function stripMessageWrappers(text: string): string {
|
||||
let s = text;
|
||||
// Injected context from memory system
|
||||
s = s.replace(/<relevant-memories>[\s\S]*?<\/relevant-memories>\s*/g, "");
|
||||
s = s.replace(/<core-memory-refresh>[\s\S]*?<\/core-memory-refresh>\s*/g, "");
|
||||
s = s.replace(/<system>[\s\S]*?<\/system>\s*/g, "");
|
||||
// File attachments (PDFs, images, etc. forwarded inline by channels)
|
||||
s = s.replace(/<file\b[^>]*>[\s\S]*?<\/file>\s*/g, "");
|
||||
// Media attachment preamble (appears before Telegram wrapper)
|
||||
s = s.replace(/^\[media attached:[^\]]*\]\s*(?:To send an image[^\n]*\n?)*/i, "");
|
||||
// System exec output blocks (may appear before Telegram wrapper)
|
||||
s = s.replace(/^(?:System:\s*\[[^\]]*\][^\n]*\n?)+/gi, "");
|
||||
// Voice chat timestamp prefix: [Tue 2026-02-10 19:41 GMT+8]
|
||||
s = s.replace(
|
||||
/^\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{1,2}:\d{2}\s+GMT[+-]\d+\]\s*/i,
|
||||
"",
|
||||
);
|
||||
// Conversation info metadata block (gateway routing context with JSON code fence)
|
||||
s = s.replace(/Conversation info\s*\(untrusted metadata\):\s*```[\s\S]*?```\s*/g, "");
|
||||
// Queued message batch header and separators
|
||||
s = s.replace(/^\[Queued messages while agent was busy\]\s*/i, "");
|
||||
s = s.replace(/---\s*Queued #\d+\s*/g, "");
|
||||
// Telegram wrapper — may now be at start after previous strips
|
||||
s = s.replace(/^\s*\[Telegram\s[^\]]+\]\s*/i, "");
|
||||
// "[message_id: ...]" suffix (Telegram and other channel IDs)
|
||||
s = s.replace(/\n?\[message_id:\s*[^\]]+\]\s*$/i, "");
|
||||
// Slack wrapper — "[Slack <workspace> #channel @user] MESSAGE [slack message id: ...]"
|
||||
s = s.replace(/^\s*\[Slack\s[^\]]+\]\s*/i, "");
|
||||
s = s.replace(/\n?\[slack message id:\s*[^\]]*\]\s*$/i, "");
|
||||
return s.trim();
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Assistant Message Extraction
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Strip tool-use, thinking, and code-output blocks from assistant messages
|
||||
* so the attention gate sees only the substantive assistant text.
|
||||
*/
|
||||
export function stripAssistantWrappers(text: string): string {
|
||||
let s = text;
|
||||
// Tool-use / tool-result / function_call blocks
|
||||
s = s.replace(/<tool_use>[\s\S]*?<\/tool_use>\s*/g, "");
|
||||
s = s.replace(/<tool_result>[\s\S]*?<\/tool_result>\s*/g, "");
|
||||
s = s.replace(/<function_call>[\s\S]*?<\/function_call>\s*/g, "");
|
||||
// Thinking tags
|
||||
s = s.replace(/<thinking>[\s\S]*?<\/thinking>\s*/g, "");
|
||||
s = s.replace(/<antThinking>[\s\S]*?<\/antThinking>\s*/g, "");
|
||||
// Code execution output
|
||||
s = s.replace(/<code_output>[\s\S]*?<\/code_output>\s*/g, "");
|
||||
return s.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract assistant message texts from the event.messages array.
|
||||
*/
|
||||
export function extractAssistantMessages(messages: unknown[]): string[] {
|
||||
return extractMessagesByRole(messages, "assistant", stripAssistantWrappers);
|
||||
}
|
||||
332
extensions/memory-neo4j/mid-session-refresh.test.ts
Normal file
332
extensions/memory-neo4j/mid-session-refresh.test.ts
Normal file
@@ -0,0 +1,332 @@
|
||||
/**
|
||||
* Tests for mid-session core memory refresh feature.
|
||||
*
|
||||
* Verifies that core memories are re-injected when context usage exceeds threshold.
|
||||
* Tests config parsing, threshold calculation, shouldRefresh logic, and edge cases.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
|
||||
// ============================================================================
|
||||
// Config parsing for refreshAtContextPercent
|
||||
// ============================================================================
|
||||
|
||||
describe("mid-session core memory refresh", () => {
|
||||
describe("config parsing", () => {
|
||||
it("should accept valid refreshAtContextPercent values", async () => {
|
||||
const { memoryNeo4jConfigSchema } = await import("./config.js");
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", user: "neo4j", password: "test" },
|
||||
embedding: { provider: "ollama" },
|
||||
coreMemory: { refreshAtContextPercent: 50 },
|
||||
});
|
||||
expect(config.coreMemory.refreshAtContextPercent).toBe(50);
|
||||
});
|
||||
|
||||
it("should accept refreshAtContextPercent of 1 (minimum)", async () => {
|
||||
const { memoryNeo4jConfigSchema } = await import("./config.js");
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", user: "neo4j", password: "test" },
|
||||
embedding: { provider: "ollama" },
|
||||
coreMemory: { refreshAtContextPercent: 1 },
|
||||
});
|
||||
expect(config.coreMemory.refreshAtContextPercent).toBe(1);
|
||||
});
|
||||
|
||||
it("should accept refreshAtContextPercent of 100 (maximum)", async () => {
|
||||
const { memoryNeo4jConfigSchema } = await import("./config.js");
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", user: "neo4j", password: "test" },
|
||||
embedding: { provider: "ollama" },
|
||||
coreMemory: { refreshAtContextPercent: 100 },
|
||||
});
|
||||
expect(config.coreMemory.refreshAtContextPercent).toBe(100);
|
||||
});
|
||||
|
||||
it("should treat refreshAtContextPercent of 0 as disabled (undefined)", async () => {
|
||||
const { memoryNeo4jConfigSchema } = await import("./config.js");
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", user: "neo4j", password: "test" },
|
||||
embedding: { provider: "ollama" },
|
||||
coreMemory: { refreshAtContextPercent: 0 },
|
||||
});
|
||||
expect(config.coreMemory.refreshAtContextPercent).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should treat negative refreshAtContextPercent as disabled (undefined)", async () => {
|
||||
const { memoryNeo4jConfigSchema } = await import("./config.js");
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", user: "neo4j", password: "test" },
|
||||
embedding: { provider: "ollama" },
|
||||
coreMemory: { refreshAtContextPercent: -10 },
|
||||
});
|
||||
expect(config.coreMemory.refreshAtContextPercent).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should throw for refreshAtContextPercent over 100", async () => {
|
||||
const { memoryNeo4jConfigSchema } = await import("./config.js");
|
||||
expect(() =>
|
||||
memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", user: "neo4j", password: "test" },
|
||||
embedding: { provider: "ollama" },
|
||||
coreMemory: { refreshAtContextPercent: 150 },
|
||||
}),
|
||||
).toThrow("coreMemory.refreshAtContextPercent must be between 1 and 100");
|
||||
});
|
||||
|
||||
it("should default to undefined when coreMemory section is omitted", async () => {
|
||||
const { memoryNeo4jConfigSchema } = await import("./config.js");
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", user: "neo4j", password: "test" },
|
||||
embedding: { provider: "ollama" },
|
||||
});
|
||||
expect(config.coreMemory.refreshAtContextPercent).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should default to undefined when refreshAtContextPercent is omitted", async () => {
|
||||
const { memoryNeo4jConfigSchema } = await import("./config.js");
|
||||
const config = memoryNeo4jConfigSchema.parse({
|
||||
neo4j: { uri: "bolt://localhost:7687", user: "neo4j", password: "test" },
|
||||
embedding: { provider: "ollama" },
|
||||
coreMemory: { enabled: true },
|
||||
});
|
||||
expect(config.coreMemory.refreshAtContextPercent).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// shouldRefresh logic (tests the decision flow from index.ts)
|
||||
// ============================================================================
|
||||
|
||||
describe("shouldRefresh decision logic", () => {
|
||||
// These tests mirror the logic from index.ts lines 893-916:
|
||||
// 1. Skip if contextWindowTokens or estimatedUsedTokens not available
|
||||
// 2. Calculate usagePercent = (estimatedUsedTokens / contextWindowTokens) * 100
|
||||
// 3. Skip if usagePercent < refreshThreshold
|
||||
// 4. Skip if tokens since last refresh < MIN_TOKENS_SINCE_REFRESH (10_000)
|
||||
// 5. Otherwise, refresh
|
||||
|
||||
const MIN_TOKENS_SINCE_REFRESH = 10_000;
|
||||
|
||||
function shouldRefresh(params: {
|
||||
contextWindowTokens: number | undefined;
|
||||
estimatedUsedTokens: number | undefined;
|
||||
refreshThreshold: number;
|
||||
lastRefreshTokens: number;
|
||||
}): boolean {
|
||||
const { contextWindowTokens, estimatedUsedTokens, refreshThreshold, lastRefreshTokens } =
|
||||
params;
|
||||
|
||||
// Skip if context info not available
|
||||
if (!contextWindowTokens || !estimatedUsedTokens) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const usagePercent = (estimatedUsedTokens / contextWindowTokens) * 100;
|
||||
|
||||
// Only refresh if we've crossed the threshold
|
||||
if (usagePercent < refreshThreshold) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check if we've already refreshed recently
|
||||
const tokensSinceRefresh = estimatedUsedTokens - lastRefreshTokens;
|
||||
if (tokensSinceRefresh < MIN_TOKENS_SINCE_REFRESH) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
it("should trigger refresh when usage exceeds threshold and enough tokens accumulated", () => {
|
||||
expect(
|
||||
shouldRefresh({
|
||||
contextWindowTokens: 200_000,
|
||||
estimatedUsedTokens: 120_000, // 60%
|
||||
refreshThreshold: 50,
|
||||
lastRefreshTokens: 0, // Never refreshed
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("should not trigger when usage is below threshold", () => {
|
||||
expect(
|
||||
shouldRefresh({
|
||||
contextWindowTokens: 200_000,
|
||||
estimatedUsedTokens: 80_000, // 40%
|
||||
refreshThreshold: 50,
|
||||
lastRefreshTokens: 0,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("should not trigger when not enough tokens since last refresh", () => {
|
||||
expect(
|
||||
shouldRefresh({
|
||||
contextWindowTokens: 200_000,
|
||||
estimatedUsedTokens: 105_000, // 52.5%
|
||||
refreshThreshold: 50,
|
||||
lastRefreshTokens: 100_000, // Only 5k tokens since last refresh
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("should trigger when enough tokens accumulated since last refresh", () => {
|
||||
expect(
|
||||
shouldRefresh({
|
||||
contextWindowTokens: 200_000,
|
||||
estimatedUsedTokens: 115_000, // 57.5%
|
||||
refreshThreshold: 50,
|
||||
lastRefreshTokens: 100_000, // 15k tokens since last refresh
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("should not trigger when contextWindowTokens is undefined", () => {
|
||||
expect(
|
||||
shouldRefresh({
|
||||
contextWindowTokens: undefined,
|
||||
estimatedUsedTokens: 120_000,
|
||||
refreshThreshold: 50,
|
||||
lastRefreshTokens: 0,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("should not trigger when estimatedUsedTokens is undefined", () => {
|
||||
expect(
|
||||
shouldRefresh({
|
||||
contextWindowTokens: 200_000,
|
||||
estimatedUsedTokens: undefined,
|
||||
refreshThreshold: 50,
|
||||
lastRefreshTokens: 0,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("should handle 0% usage (empty context)", () => {
|
||||
expect(
|
||||
shouldRefresh({
|
||||
contextWindowTokens: 200_000,
|
||||
estimatedUsedTokens: 0,
|
||||
refreshThreshold: 50,
|
||||
lastRefreshTokens: 0,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("should handle 100% usage", () => {
|
||||
expect(
|
||||
shouldRefresh({
|
||||
contextWindowTokens: 200_000,
|
||||
estimatedUsedTokens: 200_000, // 100%
|
||||
refreshThreshold: 50,
|
||||
lastRefreshTokens: 0,
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("should handle exact threshold boundary (50% == 50% threshold)", () => {
|
||||
// usagePercent == refreshThreshold: usagePercent < refreshThreshold is false, so it proceeds
|
||||
expect(
|
||||
shouldRefresh({
|
||||
contextWindowTokens: 200_000,
|
||||
estimatedUsedTokens: 100_000, // exactly 50%
|
||||
refreshThreshold: 50,
|
||||
lastRefreshTokens: 0,
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("should handle threshold of 1 (refresh almost immediately)", () => {
|
||||
expect(
|
||||
shouldRefresh({
|
||||
contextWindowTokens: 200_000,
|
||||
estimatedUsedTokens: 15_000, // 7.5%
|
||||
refreshThreshold: 1,
|
||||
lastRefreshTokens: 0,
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("should handle threshold of 100 (refresh only at full context)", () => {
|
||||
expect(
|
||||
shouldRefresh({
|
||||
contextWindowTokens: 200_000,
|
||||
estimatedUsedTokens: 190_000, // 95%
|
||||
refreshThreshold: 100,
|
||||
lastRefreshTokens: 0,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("should allow first refresh even when lastRefreshTokens is 0", () => {
|
||||
expect(
|
||||
shouldRefresh({
|
||||
contextWindowTokens: 200_000,
|
||||
estimatedUsedTokens: 110_000,
|
||||
refreshThreshold: 50,
|
||||
lastRefreshTokens: 0,
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("should support multiple refresh cycles with cumulative token growth", () => {
|
||||
// First refresh at 110k tokens
|
||||
const firstResult = shouldRefresh({
|
||||
contextWindowTokens: 200_000,
|
||||
estimatedUsedTokens: 110_000,
|
||||
refreshThreshold: 50,
|
||||
lastRefreshTokens: 0,
|
||||
});
|
||||
expect(firstResult).toBe(true);
|
||||
|
||||
// Second attempt too soon (only 5k since first)
|
||||
const secondResult = shouldRefresh({
|
||||
contextWindowTokens: 200_000,
|
||||
estimatedUsedTokens: 115_000,
|
||||
refreshThreshold: 50,
|
||||
lastRefreshTokens: 110_000,
|
||||
});
|
||||
expect(secondResult).toBe(false);
|
||||
|
||||
// Third attempt after enough growth (15k since first refresh)
|
||||
const thirdResult = shouldRefresh({
|
||||
contextWindowTokens: 200_000,
|
||||
estimatedUsedTokens: 125_000,
|
||||
refreshThreshold: 50,
|
||||
lastRefreshTokens: 110_000,
|
||||
});
|
||||
expect(thirdResult).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Output format
|
||||
// ============================================================================
|
||||
|
||||
describe("refresh output format", () => {
|
||||
it("should format core memories as XML-wrapped bullet list", () => {
|
||||
const coreMemories = [
|
||||
{ text: "User prefers TypeScript over JavaScript" },
|
||||
{ text: "User works at Acme Corp" },
|
||||
];
|
||||
const content = coreMemories.map((m) => `- ${m.text}`).join("\n");
|
||||
const output = `<core-memory-refresh>\nReminder of persistent context (you may have seen this earlier, re-stating for recency):\n${content}\n</core-memory-refresh>`;
|
||||
|
||||
expect(output).toContain("<core-memory-refresh>");
|
||||
expect(output).toContain("</core-memory-refresh>");
|
||||
expect(output).toContain("- User prefers TypeScript over JavaScript");
|
||||
expect(output).toContain("- User works at Acme Corp");
|
||||
});
|
||||
|
||||
it("should handle single core memory", () => {
|
||||
const coreMemories = [{ text: "Only memory" }];
|
||||
const content = coreMemories.map((m) => `- ${m.text}`).join("\n");
|
||||
const output = `<core-memory-refresh>\nReminder of persistent context (you may have seen this earlier, re-stating for recency):\n${content}\n</core-memory-refresh>`;
|
||||
|
||||
expect(output).toContain("- Only memory");
|
||||
expect(output.match(/^- /gm)?.length).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
327
extensions/memory-neo4j/neo4j-client.entity-dedup.test.ts
Normal file
327
extensions/memory-neo4j/neo4j-client.entity-dedup.test.ts
Normal file
@@ -0,0 +1,327 @@
|
||||
/**
|
||||
* Tests for entity deduplication in neo4j-client.ts.
|
||||
*
|
||||
* Tests findDuplicateEntityPairs() and mergeEntityPair() using mocked Neo4j driver.
|
||||
* Verifies substring-matching logic, mention-count based decisions, and merge behavior.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { Neo4jMemoryClient } from "./neo4j-client.js";
|
||||
|
||||
// ============================================================================
|
||||
// Test Helpers
|
||||
// ============================================================================
|
||||
|
||||
function createMockSession() {
|
||||
return {
|
||||
run: vi.fn().mockResolvedValue({ records: [] }),
|
||||
close: vi.fn().mockResolvedValue(undefined),
|
||||
executeWrite: vi.fn(
|
||||
async (work: (tx: { run: ReturnType<typeof vi.fn> }) => Promise<unknown>) => {
|
||||
const mockTx = { run: vi.fn().mockResolvedValue({ records: [] }) };
|
||||
return work(mockTx);
|
||||
},
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function createMockDriver() {
|
||||
return {
|
||||
session: vi.fn().mockReturnValue(createMockSession()),
|
||||
close: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
function createMockLogger() {
|
||||
return {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function mockRecord(data: Record<string, unknown>) {
|
||||
return {
|
||||
get: (key: string) => data[key],
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Entity Deduplication Tests
|
||||
// ============================================================================
|
||||
|
||||
describe("Entity Deduplication", () => {
|
||||
let client: Neo4jMemoryClient;
|
||||
let mockDriver: ReturnType<typeof createMockDriver>;
|
||||
let mockSession: ReturnType<typeof createMockSession>;
|
||||
let mockLogger: ReturnType<typeof createMockLogger>;
|
||||
|
||||
beforeEach(() => {
|
||||
mockLogger = createMockLogger();
|
||||
mockDriver = createMockDriver();
|
||||
mockSession = createMockSession();
|
||||
mockDriver.session.mockReturnValue(mockSession);
|
||||
|
||||
client = new Neo4jMemoryClient("bolt://localhost:7687", "neo4j", "password", 1024, mockLogger);
|
||||
(client as any).driver = mockDriver;
|
||||
(client as any).indexesReady = true;
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// findDuplicateEntityPairs()
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe("findDuplicateEntityPairs", () => {
|
||||
it("finds substring matches: 'tarun' + 'tarun sukhani' (same type)", async () => {
|
||||
mockSession.run.mockResolvedValueOnce({
|
||||
records: [
|
||||
mockRecord({
|
||||
id1: "e1",
|
||||
name1: "tarun",
|
||||
mc1: 5,
|
||||
id2: "e2",
|
||||
name2: "tarun sukhani",
|
||||
mc2: 3,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const pairs = await client.findDuplicateEntityPairs();
|
||||
|
||||
expect(pairs).toHaveLength(1);
|
||||
// "tarun" has more mentions (5 > 3), so it should be kept
|
||||
expect(pairs[0].keepId).toBe("e1");
|
||||
expect(pairs[0].keepName).toBe("tarun");
|
||||
expect(pairs[0].removeId).toBe("e2");
|
||||
expect(pairs[0].removeName).toBe("tarun sukhani");
|
||||
});
|
||||
|
||||
it("keeps entity with more mentions regardless of name length", async () => {
|
||||
mockSession.run.mockResolvedValueOnce({
|
||||
records: [
|
||||
mockRecord({
|
||||
id1: "e1",
|
||||
name1: "fish speech",
|
||||
mc1: 2,
|
||||
id2: "e2",
|
||||
name2: "fish speech s1 mini",
|
||||
mc2: 10,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const pairs = await client.findDuplicateEntityPairs();
|
||||
|
||||
expect(pairs).toHaveLength(1);
|
||||
// "fish speech s1 mini" has more mentions (10 > 2), so it should be kept
|
||||
expect(pairs[0].keepId).toBe("e2");
|
||||
expect(pairs[0].keepName).toBe("fish speech s1 mini");
|
||||
expect(pairs[0].removeId).toBe("e1");
|
||||
expect(pairs[0].removeName).toBe("fish speech");
|
||||
});
|
||||
|
||||
it("keeps shorter name when mentions are equal", async () => {
|
||||
mockSession.run.mockResolvedValueOnce({
|
||||
records: [
|
||||
mockRecord({
|
||||
id1: "e1",
|
||||
name1: "aaditya",
|
||||
mc1: 5,
|
||||
id2: "e2",
|
||||
name2: "aaditya sukhani",
|
||||
mc2: 5,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const pairs = await client.findDuplicateEntityPairs();
|
||||
|
||||
expect(pairs).toHaveLength(1);
|
||||
// Equal mentions, so keep the shorter name ("aaditya")
|
||||
expect(pairs[0].keepId).toBe("e1");
|
||||
expect(pairs[0].keepName).toBe("aaditya");
|
||||
expect(pairs[0].removeId).toBe("e2");
|
||||
expect(pairs[0].removeName).toBe("aaditya sukhani");
|
||||
});
|
||||
|
||||
it("returns empty array when no duplicates exist", async () => {
|
||||
mockSession.run.mockResolvedValueOnce({ records: [] });
|
||||
|
||||
const pairs = await client.findDuplicateEntityPairs();
|
||||
|
||||
expect(pairs).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("handles multiple duplicate pairs", async () => {
|
||||
mockSession.run.mockResolvedValueOnce({
|
||||
records: [
|
||||
mockRecord({
|
||||
id1: "e1",
|
||||
name1: "tarun",
|
||||
mc1: 5,
|
||||
id2: "e2",
|
||||
name2: "tarun sukhani",
|
||||
mc2: 3,
|
||||
}),
|
||||
mockRecord({
|
||||
id1: "e3",
|
||||
name1: "fish speech",
|
||||
mc1: 2,
|
||||
id2: "e4",
|
||||
name2: "fish speech s1 mini",
|
||||
mc2: 8,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const pairs = await client.findDuplicateEntityPairs();
|
||||
|
||||
expect(pairs).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("handles NULL mention counts (treats as 0)", async () => {
|
||||
mockSession.run.mockResolvedValueOnce({
|
||||
records: [
|
||||
mockRecord({
|
||||
id1: "e1",
|
||||
name1: "neo4j",
|
||||
mc1: null,
|
||||
id2: "e2",
|
||||
name2: "neo4j database",
|
||||
mc2: null,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const pairs = await client.findDuplicateEntityPairs();
|
||||
|
||||
expect(pairs).toHaveLength(1);
|
||||
// Both NULL (treated as 0), so keep the shorter name
|
||||
expect(pairs[0].keepId).toBe("e1");
|
||||
expect(pairs[0].keepName).toBe("neo4j");
|
||||
});
|
||||
|
||||
it("passes the Cypher query with substring matching and type constraint", async () => {
|
||||
mockSession.run.mockResolvedValueOnce({ records: [] });
|
||||
|
||||
await client.findDuplicateEntityPairs();
|
||||
|
||||
const query = mockSession.run.mock.calls[0][0] as string;
|
||||
// Verify the query checks same type
|
||||
expect(query).toContain("e1.type = e2.type");
|
||||
// Verify the query checks CONTAINS in both directions
|
||||
expect(query).toContain("e1.name CONTAINS e2.name");
|
||||
expect(query).toContain("e2.name CONTAINS e1.name");
|
||||
// Verify minimum name length filter
|
||||
expect(query).toContain("size(e1.name) > 2");
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// mergeEntityPair()
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe("mergeEntityPair", () => {
|
||||
it("transfers MENTIONS and deletes source entity", async () => {
|
||||
// mergeEntityPair uses executeWrite, so we need to set up the mock transaction
|
||||
const mockTx = {
|
||||
run: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
// Transfer MENTIONS
|
||||
records: [mockRecord({ transferred: 3 })],
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
// Update mentionCount
|
||||
records: [],
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
// Delete removed entity
|
||||
records: [],
|
||||
}),
|
||||
};
|
||||
|
||||
mockSession.executeWrite.mockImplementationOnce(async (work: any) => work(mockTx));
|
||||
|
||||
const result = await client.mergeEntityPair("keep-id", "remove-id");
|
||||
|
||||
expect(result).toBe(true);
|
||||
// Should have been called 3 times: transfer, update count, delete
|
||||
expect(mockTx.run).toHaveBeenCalledTimes(3);
|
||||
|
||||
// Verify transfer query
|
||||
const transferQuery = mockTx.run.mock.calls[0][0] as string;
|
||||
expect(transferQuery).toContain("MERGE (m)-[:MENTIONS]->(keep)");
|
||||
expect(transferQuery).toContain("DELETE r");
|
||||
|
||||
// Verify update mentionCount
|
||||
const updateQuery = mockTx.run.mock.calls[1][0] as string;
|
||||
expect(updateQuery).toContain("mentionCount");
|
||||
|
||||
// Verify delete query
|
||||
const deleteQuery = mockTx.run.mock.calls[2][0] as string;
|
||||
expect(deleteQuery).toContain("DETACH DELETE e");
|
||||
});
|
||||
|
||||
it("skips mentionCount update when no relationships to transfer", async () => {
|
||||
const mockTx = {
|
||||
run: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
// Transfer MENTIONS — 0 transferred
|
||||
records: [mockRecord({ transferred: 0 })],
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
// Delete removed entity (mentionCount update is skipped)
|
||||
records: [],
|
||||
}),
|
||||
};
|
||||
|
||||
mockSession.executeWrite.mockImplementationOnce(async (work: any) => work(mockTx));
|
||||
|
||||
const result = await client.mergeEntityPair("keep-id", "remove-id");
|
||||
|
||||
expect(result).toBe(true);
|
||||
// Only 2 calls: transfer (0 results) and delete (skip update)
|
||||
expect(mockTx.run).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("returns false on error", async () => {
|
||||
mockSession.executeWrite.mockRejectedValueOnce(new Error("Neo4j connection lost"));
|
||||
|
||||
const result = await client.mergeEntityPair("keep-id", "remove-id");
|
||||
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// reconcileEntityMentionCounts()
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe("reconcileEntityMentionCounts", () => {
|
||||
it("updates entities with NULL mentionCount", async () => {
|
||||
mockSession.run.mockResolvedValueOnce({
|
||||
records: [mockRecord({ updated: 42 })],
|
||||
});
|
||||
|
||||
const updated = await client.reconcileEntityMentionCounts();
|
||||
|
||||
expect(updated).toBe(42);
|
||||
const query = mockSession.run.mock.calls[0][0] as string;
|
||||
expect(query).toContain("mentionCount IS NULL");
|
||||
expect(query).toContain("SET e.mentionCount = actual");
|
||||
});
|
||||
|
||||
it("returns 0 when all entities have mentionCount set", async () => {
|
||||
mockSession.run.mockResolvedValueOnce({
|
||||
records: [mockRecord({ updated: 0 })],
|
||||
});
|
||||
|
||||
const updated = await client.reconcileEntityMentionCounts();
|
||||
|
||||
expect(updated).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
1241
extensions/memory-neo4j/neo4j-client.test.ts
Normal file
1241
extensions/memory-neo4j/neo4j-client.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
2232
extensions/memory-neo4j/neo4j-client.ts
Normal file
2232
extensions/memory-neo4j/neo4j-client.ts
Normal file
File diff suppressed because it is too large
Load Diff
206
extensions/memory-neo4j/openclaw.plugin.json
Normal file
206
extensions/memory-neo4j/openclaw.plugin.json
Normal file
@@ -0,0 +1,206 @@
|
||||
{
|
||||
"id": "memory-neo4j",
|
||||
"kind": "memory",
|
||||
"uiHints": {
|
||||
"embedding.provider": {
|
||||
"label": "Embedding Provider",
|
||||
"placeholder": "openai",
|
||||
"help": "Provider for embeddings: 'openai' or 'ollama'"
|
||||
},
|
||||
"embedding.apiKey": {
|
||||
"label": "API Key",
|
||||
"sensitive": true,
|
||||
"placeholder": "sk-proj-...",
|
||||
"help": "API key for OpenAI embeddings (not needed for Ollama)"
|
||||
},
|
||||
"embedding.model": {
|
||||
"label": "Embedding Model",
|
||||
"placeholder": "text-embedding-3-small",
|
||||
"help": "Embedding model to use (e.g., text-embedding-3-small for OpenAI, mxbai-embed-large for Ollama)"
|
||||
},
|
||||
"embedding.baseUrl": {
|
||||
"label": "Base URL",
|
||||
"placeholder": "http://localhost:11434",
|
||||
"help": "Base URL for Ollama API (optional)"
|
||||
},
|
||||
"neo4j.uri": {
|
||||
"label": "Neo4j URI",
|
||||
"placeholder": "bolt://localhost:7687",
|
||||
"help": "Bolt connection URI for your Neo4j instance"
|
||||
},
|
||||
"neo4j.user": {
|
||||
"label": "Neo4j Username",
|
||||
"placeholder": "neo4j"
|
||||
},
|
||||
"neo4j.password": {
|
||||
"label": "Neo4j Password",
|
||||
"sensitive": true
|
||||
},
|
||||
"autoCapture": {
|
||||
"label": "Auto-Capture",
|
||||
"help": "Automatically capture important information from conversations"
|
||||
},
|
||||
"autoRecall": {
|
||||
"label": "Auto-Recall",
|
||||
"help": "Automatically inject relevant memories into context"
|
||||
},
|
||||
"autoRecallMinScore": {
|
||||
"label": "Auto-Recall Min Score",
|
||||
"help": "Minimum similarity score (0-1) for auto-recall results (default: 0.25)"
|
||||
},
|
||||
"coreMemory.enabled": {
|
||||
"label": "Core Memory",
|
||||
"help": "Enable core memory bootstrap (top memories auto-loaded into context)"
|
||||
},
|
||||
"coreMemory.refreshAtContextPercent": {
|
||||
"label": "Core Memory Refresh %",
|
||||
"help": "Re-inject core memories when context usage reaches this percentage (1-100, optional)"
|
||||
},
|
||||
"extraction.apiKey": {
|
||||
"label": "Extraction API Key",
|
||||
"sensitive": true,
|
||||
"placeholder": "sk-or-v1-...",
|
||||
"help": "API key for extraction LLM (not needed for Ollama/local models)"
|
||||
},
|
||||
"extraction.model": {
|
||||
"label": "Extraction Model",
|
||||
"placeholder": "google/gemini-2.0-flash-001",
|
||||
"help": "Model for entity extraction (e.g., google/gemini-2.0-flash-001 for OpenRouter, llama3.1:8b for Ollama)"
|
||||
},
|
||||
"extraction.baseUrl": {
|
||||
"label": "Extraction Base URL",
|
||||
"placeholder": "https://openrouter.ai/api/v1",
|
||||
"help": "Base URL for extraction API (e.g., https://openrouter.ai/api/v1 or http://localhost:11434/v1 for Ollama)"
|
||||
},
|
||||
"graphSearchDepth": {
|
||||
"label": "Graph Search Depth",
|
||||
"help": "Maximum relationship hops for graph search spreading activation (1-3, default: 1)"
|
||||
},
|
||||
"decayCurves": {
|
||||
"label": "Decay Curves",
|
||||
"help": "Per-category decay curve overrides. Example: {\"fact\": {\"halfLifeDays\": 60}, \"other\": {\"halfLifeDays\": 14}}"
|
||||
},
|
||||
"sleepCycle.auto": {
|
||||
"label": "Auto Sleep Cycle",
|
||||
"help": "Automatically run memory consolidation (dedup, extraction, decay) daily at 3:00 AM local time (default: on)"
|
||||
}
|
||||
},
|
||||
"configSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"embedding": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"provider": {
|
||||
"type": "string",
|
||||
"enum": ["openai", "ollama"]
|
||||
},
|
||||
"apiKey": {
|
||||
"type": "string"
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"baseUrl": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"neo4j": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"uri": {
|
||||
"type": "string"
|
||||
},
|
||||
"user": {
|
||||
"type": "string"
|
||||
},
|
||||
"username": {
|
||||
"type": "string"
|
||||
},
|
||||
"password": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["uri"]
|
||||
},
|
||||
"autoCapture": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"autoRecall": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"autoRecallMinScore": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1
|
||||
},
|
||||
"coreMemory": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"refreshAtContextPercent": {
|
||||
"type": "number",
|
||||
"minimum": 1,
|
||||
"maximum": 100
|
||||
}
|
||||
}
|
||||
},
|
||||
"extraction": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"apiKey": {
|
||||
"type": "string"
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"baseUrl": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"graphSearchDepth": {
|
||||
"type": "number",
|
||||
"minimum": 1,
|
||||
"maximum": 3
|
||||
},
|
||||
"decayCurves": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"halfLifeDays": {
|
||||
"type": "number",
|
||||
"minimum": 1
|
||||
}
|
||||
},
|
||||
"required": ["halfLifeDays"]
|
||||
}
|
||||
},
|
||||
"autoRecallSkipPattern": {
|
||||
"type": "string",
|
||||
"description": "RegExp pattern to skip auto-recall for matching session keys (e.g. voice|realtime)"
|
||||
},
|
||||
"autoCaptureSkipPattern": {
|
||||
"type": "string",
|
||||
"description": "RegExp pattern to skip auto-capture for matching session keys (e.g. voice|realtime)"
|
||||
},
|
||||
"sleepCycle": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"auto": { "type": "boolean" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["neo4j"]
|
||||
}
|
||||
}
|
||||
19
extensions/memory-neo4j/package.json
Normal file
19
extensions/memory-neo4j/package.json
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "@openclaw/memory-neo4j",
|
||||
"version": "2026.2.2",
|
||||
"description": "OpenClaw Neo4j-backed long-term memory plugin with three-signal hybrid search, entity extraction, and knowledge graph",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@sinclair/typebox": "0.34.48",
|
||||
"neo4j-driver": "^5.27.0",
|
||||
"openai": "^6.17.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"openclaw": "workspace:*"
|
||||
},
|
||||
"openclaw": {
|
||||
"extensions": [
|
||||
"./index.ts"
|
||||
]
|
||||
}
|
||||
}
|
||||
224
extensions/memory-neo4j/schema.test.ts
Normal file
224
extensions/memory-neo4j/schema.test.ts
Normal file
@@ -0,0 +1,224 @@
|
||||
/**
|
||||
* Tests for schema.ts — Schema Validation & Helpers.
|
||||
*
|
||||
* Tests the exported pure functions: escapeLucene(), validateRelationshipType(),
|
||||
* and the exported constants and types.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import type { MemorySource } from "./schema.js";
|
||||
import {
|
||||
escapeLucene,
|
||||
validateRelationshipType,
|
||||
ALLOWED_RELATIONSHIP_TYPES,
|
||||
MEMORY_CATEGORIES,
|
||||
ENTITY_TYPES,
|
||||
} from "./schema.js";
|
||||
|
||||
// ============================================================================
|
||||
// escapeLucene()
|
||||
// ============================================================================
|
||||
|
||||
describe("escapeLucene", () => {
|
||||
it("should return normal text unchanged", () => {
|
||||
expect(escapeLucene("hello world")).toBe("hello world");
|
||||
});
|
||||
|
||||
it("should return empty string unchanged", () => {
|
||||
expect(escapeLucene("")).toBe("");
|
||||
});
|
||||
|
||||
it("should escape plus sign", () => {
|
||||
expect(escapeLucene("a+b")).toBe("a\\+b");
|
||||
});
|
||||
|
||||
it("should escape minus sign", () => {
|
||||
expect(escapeLucene("a-b")).toBe("a\\-b");
|
||||
});
|
||||
|
||||
it("should escape ampersand", () => {
|
||||
expect(escapeLucene("a&b")).toBe("a\\&b");
|
||||
});
|
||||
|
||||
it("should escape pipe", () => {
|
||||
expect(escapeLucene("a|b")).toBe("a\\|b");
|
||||
});
|
||||
|
||||
it("should escape exclamation mark", () => {
|
||||
expect(escapeLucene("hello!")).toBe("hello\\!");
|
||||
});
|
||||
|
||||
it("should escape parentheses", () => {
|
||||
expect(escapeLucene("(group)")).toBe("\\(group\\)");
|
||||
});
|
||||
|
||||
it("should escape curly braces", () => {
|
||||
expect(escapeLucene("{range}")).toBe("\\{range\\}");
|
||||
});
|
||||
|
||||
it("should escape square brackets", () => {
|
||||
expect(escapeLucene("[range]")).toBe("\\[range\\]");
|
||||
});
|
||||
|
||||
it("should escape caret", () => {
|
||||
expect(escapeLucene("boost^2")).toBe("boost\\^2");
|
||||
});
|
||||
|
||||
it("should escape double quotes", () => {
|
||||
expect(escapeLucene('"exact"')).toBe('\\"exact\\"');
|
||||
});
|
||||
|
||||
it("should escape tilde", () => {
|
||||
expect(escapeLucene("fuzzy~")).toBe("fuzzy\\~");
|
||||
});
|
||||
|
||||
it("should escape asterisk", () => {
|
||||
expect(escapeLucene("wild*")).toBe("wild\\*");
|
||||
});
|
||||
|
||||
it("should escape question mark", () => {
|
||||
expect(escapeLucene("single?")).toBe("single\\?");
|
||||
});
|
||||
|
||||
it("should escape colon", () => {
|
||||
expect(escapeLucene("field:value")).toBe("field\\:value");
|
||||
});
|
||||
|
||||
it("should escape backslash", () => {
|
||||
expect(escapeLucene("path\\file")).toBe("path\\\\file");
|
||||
});
|
||||
|
||||
it("should escape forward slash", () => {
|
||||
expect(escapeLucene("a/b")).toBe("a\\/b");
|
||||
});
|
||||
|
||||
it("should escape multiple special characters in one string", () => {
|
||||
expect(escapeLucene("(a+b) && c*")).toBe("\\(a\\+b\\) \\&\\& c\\*");
|
||||
});
|
||||
|
||||
it("should handle mixed normal and special characters", () => {
|
||||
expect(escapeLucene("hello world! [test]")).toBe("hello world\\! \\[test\\]");
|
||||
});
|
||||
|
||||
it("should handle strings with only special characters", () => {
|
||||
expect(escapeLucene("+-")).toBe("\\+\\-");
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// validateRelationshipType()
|
||||
// ============================================================================
|
||||
|
||||
describe("validateRelationshipType", () => {
|
||||
describe("valid relationship types", () => {
|
||||
it("should accept WORKS_AT", () => {
|
||||
expect(validateRelationshipType("WORKS_AT")).toBe(true);
|
||||
});
|
||||
|
||||
it("should accept LIVES_AT", () => {
|
||||
expect(validateRelationshipType("LIVES_AT")).toBe(true);
|
||||
});
|
||||
|
||||
it("should accept KNOWS", () => {
|
||||
expect(validateRelationshipType("KNOWS")).toBe(true);
|
||||
});
|
||||
|
||||
it("should accept MARRIED_TO", () => {
|
||||
expect(validateRelationshipType("MARRIED_TO")).toBe(true);
|
||||
});
|
||||
|
||||
it("should accept PREFERS", () => {
|
||||
expect(validateRelationshipType("PREFERS")).toBe(true);
|
||||
});
|
||||
|
||||
it("should accept DECIDED", () => {
|
||||
expect(validateRelationshipType("DECIDED")).toBe(true);
|
||||
});
|
||||
|
||||
it("should accept RELATED_TO", () => {
|
||||
expect(validateRelationshipType("RELATED_TO")).toBe(true);
|
||||
});
|
||||
|
||||
it("should accept all ALLOWED_RELATIONSHIP_TYPES", () => {
|
||||
for (const type of ALLOWED_RELATIONSHIP_TYPES) {
|
||||
expect(validateRelationshipType(type)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("invalid relationship types", () => {
|
||||
it("should reject unknown relationship type", () => {
|
||||
expect(validateRelationshipType("HATES")).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject empty string", () => {
|
||||
expect(validateRelationshipType("")).toBe(false);
|
||||
});
|
||||
|
||||
it("should be case sensitive — lowercase is rejected", () => {
|
||||
expect(validateRelationshipType("works_at")).toBe(false);
|
||||
});
|
||||
|
||||
it("should be case sensitive — mixed case is rejected", () => {
|
||||
expect(validateRelationshipType("Works_At")).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject types with extra whitespace", () => {
|
||||
expect(validateRelationshipType(" WORKS_AT ")).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject potential Cypher injection", () => {
|
||||
expect(validateRelationshipType("WORKS_AT]->(n) DELETE n//")).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Exported Constants
|
||||
// ============================================================================
|
||||
|
||||
describe("exported constants", () => {
|
||||
it("MEMORY_CATEGORIES should contain expected categories", () => {
|
||||
expect(MEMORY_CATEGORIES).toContain("preference");
|
||||
expect(MEMORY_CATEGORIES).toContain("fact");
|
||||
expect(MEMORY_CATEGORIES).toContain("decision");
|
||||
expect(MEMORY_CATEGORIES).toContain("entity");
|
||||
expect(MEMORY_CATEGORIES).toContain("other");
|
||||
});
|
||||
|
||||
it("ENTITY_TYPES should contain expected types", () => {
|
||||
expect(ENTITY_TYPES).toContain("person");
|
||||
expect(ENTITY_TYPES).toContain("organization");
|
||||
expect(ENTITY_TYPES).toContain("location");
|
||||
expect(ENTITY_TYPES).toContain("event");
|
||||
expect(ENTITY_TYPES).toContain("concept");
|
||||
});
|
||||
|
||||
it("ALLOWED_RELATIONSHIP_TYPES should be a Set", () => {
|
||||
expect(ALLOWED_RELATIONSHIP_TYPES).toBeInstanceOf(Set);
|
||||
expect(ALLOWED_RELATIONSHIP_TYPES.size).toBe(7);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// MemorySource Type
|
||||
// ============================================================================
|
||||
|
||||
describe("MemorySource type", () => {
|
||||
it("should accept 'auto-capture-assistant' as a valid MemorySource value", () => {
|
||||
// Type-level check: this assignment should compile without error
|
||||
const source: MemorySource = "auto-capture-assistant";
|
||||
expect(source).toBe("auto-capture-assistant");
|
||||
});
|
||||
|
||||
it("should accept all MemorySource values", () => {
|
||||
const sources: MemorySource[] = [
|
||||
"user",
|
||||
"auto-capture",
|
||||
"auto-capture-assistant",
|
||||
"memory-watcher",
|
||||
"import",
|
||||
];
|
||||
expect(sources).toHaveLength(5);
|
||||
});
|
||||
});
|
||||
206
extensions/memory-neo4j/schema.ts
Normal file
206
extensions/memory-neo4j/schema.ts
Normal file
@@ -0,0 +1,206 @@
|
||||
/**
|
||||
* Graph schema types, Cypher query templates, and constants for memory-neo4j.
|
||||
*/
|
||||
|
||||
// ============================================================================
|
||||
// Shared Types
|
||||
// ============================================================================
|
||||
|
||||
export type Logger = {
|
||||
info: (msg: string) => void;
|
||||
warn: (msg: string) => void;
|
||||
error: (msg: string) => void;
|
||||
debug?: (msg: string) => void;
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Node Types
|
||||
// ============================================================================
|
||||
|
||||
export type MemoryCategory = "core" | "preference" | "fact" | "decision" | "entity" | "other";
|
||||
export type EntityType = "person" | "organization" | "location" | "event" | "concept";
|
||||
export type ExtractionStatus = "pending" | "complete" | "failed" | "skipped";
|
||||
export type MemorySource =
|
||||
| "user"
|
||||
| "auto-capture"
|
||||
| "auto-capture-assistant"
|
||||
| "memory-watcher"
|
||||
| "import";
|
||||
|
||||
export type MemoryNode = {
|
||||
id: string;
|
||||
text: string;
|
||||
embedding: number[];
|
||||
importance: number;
|
||||
category: MemoryCategory;
|
||||
source: MemorySource;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
extractionStatus: ExtractionStatus;
|
||||
extractionRetries: number;
|
||||
agentId: string;
|
||||
sessionKey?: string;
|
||||
retrievalCount: number;
|
||||
lastRetrievedAt?: string;
|
||||
taskId?: string; // Optional link to TASKS.md task (e.g., "TASK-001")
|
||||
};
|
||||
|
||||
export type EntityNode = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: EntityType;
|
||||
aliases: string[];
|
||||
description?: string;
|
||||
firstSeen: string;
|
||||
lastSeen: string;
|
||||
mentionCount: number;
|
||||
};
|
||||
|
||||
export type TagNode = {
|
||||
id: string;
|
||||
name: string;
|
||||
category: string;
|
||||
createdAt: string;
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Extraction Types
|
||||
// ============================================================================
|
||||
|
||||
export type ExtractedEntity = {
|
||||
name: string;
|
||||
type: EntityType;
|
||||
aliases?: string[];
|
||||
description?: string;
|
||||
};
|
||||
|
||||
export type ExtractedRelationship = {
|
||||
source: string;
|
||||
target: string;
|
||||
type: string;
|
||||
confidence: number;
|
||||
};
|
||||
|
||||
export type ExtractedTag = {
|
||||
name: string;
|
||||
category: string;
|
||||
};
|
||||
|
||||
export type ExtractionResult = {
|
||||
category?: MemoryCategory;
|
||||
entities: ExtractedEntity[];
|
||||
relationships: ExtractedRelationship[];
|
||||
tags: ExtractedTag[];
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Search Types
|
||||
// ============================================================================
|
||||
|
||||
export type SearchSignalResult = {
|
||||
id: string;
|
||||
text: string;
|
||||
category: string;
|
||||
importance: number;
|
||||
createdAt: string;
|
||||
score: number;
|
||||
taskId?: string; // Optional link to TASKS.md task (e.g., "TASK-001")
|
||||
};
|
||||
|
||||
export type SignalAttribution = {
|
||||
rank: number; // 1-indexed, 0 = absent from this signal
|
||||
score: number; // raw signal score, 0 = absent
|
||||
};
|
||||
|
||||
export type HybridSearchResult = {
|
||||
id: string;
|
||||
text: string;
|
||||
category: string;
|
||||
importance: number;
|
||||
createdAt: string;
|
||||
score: number;
|
||||
taskId?: string; // Optional link to TASKS.md task (e.g., "TASK-001")
|
||||
signals?: {
|
||||
vector: SignalAttribution;
|
||||
bm25: SignalAttribution;
|
||||
graph: SignalAttribution;
|
||||
};
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Input Types
|
||||
// ============================================================================
|
||||
|
||||
export type StoreMemoryInput = {
|
||||
id: string;
|
||||
text: string;
|
||||
embedding: number[];
|
||||
importance: number;
|
||||
category: MemoryCategory;
|
||||
source: MemorySource;
|
||||
extractionStatus: ExtractionStatus;
|
||||
agentId: string;
|
||||
sessionKey?: string;
|
||||
taskId?: string; // Optional link to TASKS.md task (e.g., "TASK-001")
|
||||
};
|
||||
|
||||
export type MergeEntityInput = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: EntityType;
|
||||
aliases?: string[];
|
||||
description?: string;
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Constants
|
||||
// ============================================================================
|
||||
|
||||
export const MEMORY_CATEGORIES = [
|
||||
"core",
|
||||
"preference",
|
||||
"fact",
|
||||
"decision",
|
||||
"entity",
|
||||
"other",
|
||||
] as const;
|
||||
|
||||
export const ENTITY_TYPES = ["person", "organization", "location", "event", "concept"] as const;
|
||||
|
||||
export const ALLOWED_RELATIONSHIP_TYPES = new Set([
|
||||
"WORKS_AT",
|
||||
"LIVES_AT",
|
||||
"KNOWS",
|
||||
"MARRIED_TO",
|
||||
"PREFERS",
|
||||
"DECIDED",
|
||||
"RELATED_TO",
|
||||
]);
|
||||
|
||||
// ============================================================================
|
||||
// Lucene Helpers
|
||||
// ============================================================================
|
||||
|
||||
const LUCENE_SPECIAL_CHARS = /[+\-&|!(){}[\]^"~*?:\\/]/g;
|
||||
|
||||
/**
|
||||
* Escape special characters for Lucene fulltext search queries.
|
||||
*/
|
||||
export function escapeLucene(query: string): string {
|
||||
return query.replace(LUCENE_SPECIAL_CHARS, "\\$&");
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that a relationship type is in the allowed set.
|
||||
* Prevents Cypher injection via dynamic relationship type.
|
||||
*/
|
||||
export function validateRelationshipType(type: string): boolean {
|
||||
return ALLOWED_RELATIONSHIP_TYPES.has(type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a canonical key for a pair of IDs (sorted for order-independence).
|
||||
*/
|
||||
export function makePairKey(a: string, b: string): string {
|
||||
return a < b ? `${a}:${b}` : `${b}:${a}`;
|
||||
}
|
||||
554
extensions/memory-neo4j/search.test.ts
Normal file
554
extensions/memory-neo4j/search.test.ts
Normal file
@@ -0,0 +1,554 @@
|
||||
/**
|
||||
* Tests for search.ts — Hybrid Search & RRF Fusion.
|
||||
*
|
||||
* Tests the exported pure logic: classifyQuery(), getAdaptiveWeights(), and fuseWithConfidenceRRF().
|
||||
* hybridSearch() is tested with mocked Neo4j client and Embeddings.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import type { Embeddings } from "./embeddings.js";
|
||||
import type { Neo4jMemoryClient } from "./neo4j-client.js";
|
||||
import type { SearchSignalResult } from "./schema.js";
|
||||
import {
|
||||
classifyQuery,
|
||||
getAdaptiveWeights,
|
||||
fuseWithConfidenceRRF,
|
||||
hybridSearch,
|
||||
} from "./search.js";
|
||||
|
||||
// ============================================================================
|
||||
// classifyQuery()
|
||||
// ============================================================================
|
||||
|
||||
describe("classifyQuery", () => {
|
||||
describe("short queries (1-2 words)", () => {
|
||||
it("should classify a single word as 'short'", () => {
|
||||
expect(classifyQuery("dogs")).toBe("short");
|
||||
});
|
||||
|
||||
it("should classify two words as 'short'", () => {
|
||||
expect(classifyQuery("best coffee")).toBe("short");
|
||||
});
|
||||
|
||||
it("should handle whitespace-padded short queries", () => {
|
||||
expect(classifyQuery(" hello ")).toBe("short");
|
||||
});
|
||||
});
|
||||
|
||||
describe("entity queries (proper nouns)", () => {
|
||||
it("should classify a single capitalized word as 'entity' (proper noun detection)", () => {
|
||||
expect(classifyQuery("TypeScript")).toBe("entity");
|
||||
});
|
||||
it("should classify query with proper noun as 'entity'", () => {
|
||||
expect(classifyQuery("tell me about Tarun")).toBe("entity");
|
||||
});
|
||||
|
||||
it("should classify query with organization name as 'entity'", () => {
|
||||
expect(classifyQuery("what about Google")).toBe("entity");
|
||||
});
|
||||
|
||||
it("should classify question patterns targeting entities", () => {
|
||||
expect(classifyQuery("who is the CEO")).toBe("entity");
|
||||
});
|
||||
|
||||
it("should classify 'where is' patterns as entity", () => {
|
||||
expect(classifyQuery("where is the office")).toBe("entity");
|
||||
});
|
||||
|
||||
it("should classify 'what does' patterns as entity", () => {
|
||||
expect(classifyQuery("what does she do")).toBe("entity");
|
||||
});
|
||||
|
||||
it("should not treat common words (The, Is, etc.) as entity indicators", () => {
|
||||
// "The" and "Is" are excluded from capitalized word detection
|
||||
// 3 words, no proper nouns detected, no question pattern -> default
|
||||
expect(classifyQuery("this is fine")).toBe("default");
|
||||
});
|
||||
});
|
||||
|
||||
describe("long queries (5+ words)", () => {
|
||||
it("should classify a 5-word query as 'long'", () => {
|
||||
expect(classifyQuery("what is the best framework")).toBe("long");
|
||||
});
|
||||
|
||||
it("should classify a longer sentence as 'long'", () => {
|
||||
expect(classifyQuery("tell me about the history of programming languages")).toBe("long");
|
||||
});
|
||||
|
||||
it("should classify a verbose question as 'long'", () => {
|
||||
expect(classifyQuery("how do i configure the database connection")).toBe("long");
|
||||
});
|
||||
});
|
||||
|
||||
describe("default queries (3-4 words, no entities)", () => {
|
||||
it("should classify a 3-word lowercase query as 'default'", () => {
|
||||
expect(classifyQuery("my favorite color")).toBe("default");
|
||||
});
|
||||
|
||||
it("should classify a 4-word lowercase query as 'default'", () => {
|
||||
expect(classifyQuery("best practices for testing")).toBe("default");
|
||||
});
|
||||
});
|
||||
|
||||
describe("edge cases", () => {
|
||||
it("should handle empty string", () => {
|
||||
// Empty string splits to [""], length 1 -> "short"
|
||||
expect(classifyQuery("")).toBe("short");
|
||||
});
|
||||
|
||||
it("should handle only whitespace", () => {
|
||||
// " ".trim() = "", splits to [""], length 1 -> "short"
|
||||
expect(classifyQuery(" ")).toBe("short");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// getAdaptiveWeights()
|
||||
// ============================================================================
|
||||
|
||||
describe("getAdaptiveWeights", () => {
|
||||
describe("with graph enabled", () => {
|
||||
it("should boost BM25 for short queries", () => {
|
||||
const [vector, bm25, graph] = getAdaptiveWeights("short", true);
|
||||
expect(bm25).toBeGreaterThan(vector);
|
||||
expect(vector).toBe(0.8);
|
||||
expect(bm25).toBe(1.2);
|
||||
expect(graph).toBe(1.0);
|
||||
});
|
||||
|
||||
it("should boost graph for entity queries", () => {
|
||||
const [vector, bm25, graph] = getAdaptiveWeights("entity", true);
|
||||
expect(graph).toBeGreaterThan(vector);
|
||||
expect(graph).toBeGreaterThan(bm25);
|
||||
expect(vector).toBe(0.8);
|
||||
expect(bm25).toBe(1.0);
|
||||
expect(graph).toBe(1.3);
|
||||
});
|
||||
|
||||
it("should boost vector for long queries", () => {
|
||||
const [vector, bm25, graph] = getAdaptiveWeights("long", true);
|
||||
expect(vector).toBeGreaterThan(bm25);
|
||||
expect(vector).toBeGreaterThan(graph);
|
||||
expect(vector).toBe(1.2);
|
||||
expect(bm25).toBe(0.7);
|
||||
expect(graph).toBeCloseTo(0.8);
|
||||
});
|
||||
|
||||
it("should return balanced weights for default queries", () => {
|
||||
const [vector, bm25, graph] = getAdaptiveWeights("default", true);
|
||||
expect(vector).toBe(1.0);
|
||||
expect(bm25).toBe(1.0);
|
||||
expect(graph).toBe(1.0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("with graph disabled", () => {
|
||||
it("should zero-out graph weight for short queries", () => {
|
||||
const [vector, bm25, graph] = getAdaptiveWeights("short", false);
|
||||
expect(graph).toBe(0);
|
||||
expect(vector).toBe(0.8);
|
||||
expect(bm25).toBe(1.2);
|
||||
});
|
||||
|
||||
it("should zero-out graph weight for entity queries", () => {
|
||||
const [, , graph] = getAdaptiveWeights("entity", false);
|
||||
expect(graph).toBe(0);
|
||||
});
|
||||
|
||||
it("should zero-out graph weight for long queries", () => {
|
||||
const [, , graph] = getAdaptiveWeights("long", false);
|
||||
expect(graph).toBe(0);
|
||||
});
|
||||
|
||||
it("should zero-out graph weight for default queries", () => {
|
||||
const [, , graph] = getAdaptiveWeights("default", false);
|
||||
expect(graph).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// hybridSearch() — integration test with mocked dependencies
|
||||
// ============================================================================
|
||||
|
||||
describe("hybridSearch", () => {
|
||||
// Properly typed mocks matching the interfaces hybridSearch depends on.
|
||||
// Using Pick<> to extract only the methods hybridSearch actually calls,
|
||||
// so TypeScript will catch interface changes (e.g. renamed or removed methods).
|
||||
type MockedDb = {
|
||||
[K in keyof Pick<
|
||||
Neo4jMemoryClient,
|
||||
"vectorSearch" | "bm25Search" | "graphSearch" | "recordRetrievals"
|
||||
>]: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
type MockedEmbeddings = {
|
||||
[K in keyof Pick<Embeddings, "embed" | "embedBatch">]: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
|
||||
const mockDb: MockedDb = {
|
||||
vectorSearch: vi.fn(),
|
||||
bm25Search: vi.fn(),
|
||||
graphSearch: vi.fn(),
|
||||
recordRetrievals: vi.fn(),
|
||||
};
|
||||
|
||||
const mockEmbeddings: MockedEmbeddings = {
|
||||
embed: vi.fn(),
|
||||
embedBatch: vi.fn(),
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
mockEmbeddings.embed.mockResolvedValue([0.1, 0.2, 0.3]);
|
||||
mockDb.recordRetrievals.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
function makeSignalResult(overrides: Partial<SearchSignalResult> = {}): SearchSignalResult {
|
||||
return {
|
||||
id: "mem-1",
|
||||
text: "Test memory",
|
||||
category: "fact",
|
||||
importance: 0.7,
|
||||
createdAt: "2025-01-01T00:00:00Z",
|
||||
score: 0.9,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
it("should return empty array when no signals return results", async () => {
|
||||
mockDb.vectorSearch.mockResolvedValue([]);
|
||||
mockDb.bm25Search.mockResolvedValue([]);
|
||||
|
||||
const results = await hybridSearch(
|
||||
mockDb as unknown as Neo4jMemoryClient,
|
||||
mockEmbeddings as unknown as Embeddings,
|
||||
"test query",
|
||||
5,
|
||||
"agent-1",
|
||||
false,
|
||||
);
|
||||
|
||||
expect(results).toEqual([]);
|
||||
expect(mockDb.recordRetrievals).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should fuse results from vector and BM25 signals", async () => {
|
||||
const vectorResult = makeSignalResult({ id: "mem-1", score: 0.95, text: "Vector match" });
|
||||
const bm25Result = makeSignalResult({ id: "mem-2", score: 0.8, text: "BM25 match" });
|
||||
|
||||
mockDb.vectorSearch.mockResolvedValue([vectorResult]);
|
||||
mockDb.bm25Search.mockResolvedValue([bm25Result]);
|
||||
|
||||
const results = await hybridSearch(
|
||||
mockDb as unknown as Neo4jMemoryClient,
|
||||
mockEmbeddings as unknown as Embeddings,
|
||||
"test query",
|
||||
5,
|
||||
"agent-1",
|
||||
false,
|
||||
);
|
||||
|
||||
expect(results.length).toBe(2);
|
||||
// Results should have scores normalized to 0-1
|
||||
expect(results[0].score).toBeLessThanOrEqual(1);
|
||||
expect(results[0].score).toBeGreaterThanOrEqual(0);
|
||||
// First result should have the highest score (normalized to 1)
|
||||
expect(results[0].score).toBe(1);
|
||||
});
|
||||
|
||||
it("should deduplicate across signals (same memory in multiple signals)", async () => {
|
||||
const sharedResult = makeSignalResult({ id: "mem-shared", score: 0.9 });
|
||||
|
||||
mockDb.vectorSearch.mockResolvedValue([sharedResult]);
|
||||
mockDb.bm25Search.mockResolvedValue([{ ...sharedResult, score: 0.85 }]);
|
||||
|
||||
const results = await hybridSearch(
|
||||
mockDb as unknown as Neo4jMemoryClient,
|
||||
mockEmbeddings as unknown as Embeddings,
|
||||
"test query",
|
||||
5,
|
||||
"agent-1",
|
||||
false,
|
||||
);
|
||||
|
||||
// Should only have one result (deduplicated by ID)
|
||||
expect(results.length).toBe(1);
|
||||
expect(results[0].id).toBe("mem-shared");
|
||||
// Score should be higher than either individual signal (boosted by appearing in both)
|
||||
expect(results[0].score).toBe(1); // It's the only result, so normalized to 1
|
||||
});
|
||||
|
||||
it("should include graph signal when graphEnabled is true", async () => {
|
||||
mockDb.vectorSearch.mockResolvedValue([]);
|
||||
mockDb.bm25Search.mockResolvedValue([]);
|
||||
mockDb.graphSearch.mockResolvedValue([
|
||||
makeSignalResult({ id: "mem-graph", score: 0.7, text: "Graph result" }),
|
||||
]);
|
||||
|
||||
const results = await hybridSearch(
|
||||
mockDb as unknown as Neo4jMemoryClient,
|
||||
mockEmbeddings as unknown as Embeddings,
|
||||
"tell me about Tarun",
|
||||
5,
|
||||
"agent-1",
|
||||
true,
|
||||
);
|
||||
|
||||
expect(mockDb.graphSearch).toHaveBeenCalled();
|
||||
expect(results.length).toBe(1);
|
||||
expect(results[0].id).toBe("mem-graph");
|
||||
});
|
||||
|
||||
it("should not call graphSearch when graphEnabled is false", async () => {
|
||||
mockDb.vectorSearch.mockResolvedValue([]);
|
||||
mockDb.bm25Search.mockResolvedValue([]);
|
||||
|
||||
await hybridSearch(
|
||||
mockDb as unknown as Neo4jMemoryClient,
|
||||
mockEmbeddings as unknown as Embeddings,
|
||||
"test query",
|
||||
5,
|
||||
"agent-1",
|
||||
false,
|
||||
);
|
||||
|
||||
expect(mockDb.graphSearch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should limit results to the requested count", async () => {
|
||||
const manyResults = Array.from({ length: 10 }, (_, i) =>
|
||||
makeSignalResult({ id: `mem-${i}`, score: 0.9 - i * 0.05 }),
|
||||
);
|
||||
|
||||
mockDb.vectorSearch.mockResolvedValue(manyResults);
|
||||
mockDb.bm25Search.mockResolvedValue([]);
|
||||
|
||||
const results = await hybridSearch(
|
||||
mockDb as unknown as Neo4jMemoryClient,
|
||||
mockEmbeddings as unknown as Embeddings,
|
||||
"test query",
|
||||
3,
|
||||
"agent-1",
|
||||
false,
|
||||
);
|
||||
|
||||
expect(results.length).toBe(3);
|
||||
});
|
||||
|
||||
it("should record retrieval events for returned results", async () => {
|
||||
mockDb.vectorSearch.mockResolvedValue([
|
||||
makeSignalResult({ id: "mem-1" }),
|
||||
makeSignalResult({ id: "mem-2" }),
|
||||
]);
|
||||
mockDb.bm25Search.mockResolvedValue([]);
|
||||
|
||||
await hybridSearch(
|
||||
mockDb as unknown as Neo4jMemoryClient,
|
||||
mockEmbeddings as unknown as Embeddings,
|
||||
"test query",
|
||||
5,
|
||||
"agent-1",
|
||||
false,
|
||||
);
|
||||
|
||||
expect(mockDb.recordRetrievals).toHaveBeenCalledWith(["mem-1", "mem-2"]);
|
||||
});
|
||||
|
||||
it("should silently handle recordRetrievals failure", async () => {
|
||||
mockDb.vectorSearch.mockResolvedValue([makeSignalResult({ id: "mem-1" })]);
|
||||
mockDb.bm25Search.mockResolvedValue([]);
|
||||
mockDb.recordRetrievals.mockRejectedValue(new Error("DB connection lost"));
|
||||
|
||||
// Should not throw
|
||||
const results = await hybridSearch(
|
||||
mockDb as unknown as Neo4jMemoryClient,
|
||||
mockEmbeddings as unknown as Embeddings,
|
||||
"test query",
|
||||
5,
|
||||
"agent-1",
|
||||
false,
|
||||
);
|
||||
|
||||
expect(results.length).toBe(1);
|
||||
});
|
||||
|
||||
it("should normalize scores to 0-1 range", async () => {
|
||||
mockDb.vectorSearch.mockResolvedValue([
|
||||
makeSignalResult({ id: "mem-1", score: 0.95 }),
|
||||
makeSignalResult({ id: "mem-2", score: 0.5 }),
|
||||
]);
|
||||
mockDb.bm25Search.mockResolvedValue([]);
|
||||
|
||||
const results = await hybridSearch(
|
||||
mockDb as unknown as Neo4jMemoryClient,
|
||||
mockEmbeddings as unknown as Embeddings,
|
||||
"test query",
|
||||
5,
|
||||
"agent-1",
|
||||
false,
|
||||
);
|
||||
|
||||
for (const r of results) {
|
||||
expect(r.score).toBeGreaterThanOrEqual(0);
|
||||
expect(r.score).toBeLessThanOrEqual(1);
|
||||
}
|
||||
});
|
||||
|
||||
it("should use candidateMultiplier option", async () => {
|
||||
mockDb.vectorSearch.mockResolvedValue([]);
|
||||
mockDb.bm25Search.mockResolvedValue([]);
|
||||
|
||||
await hybridSearch(
|
||||
mockDb as unknown as Neo4jMemoryClient,
|
||||
mockEmbeddings as unknown as Embeddings,
|
||||
"test query",
|
||||
5,
|
||||
"agent-1",
|
||||
false,
|
||||
{ candidateMultiplier: 8 },
|
||||
);
|
||||
|
||||
// limit=5, multiplier=8 => candidateLimit = 40
|
||||
expect(mockDb.vectorSearch).toHaveBeenCalledWith(expect.any(Array), 40, 0.1, "agent-1");
|
||||
expect(mockDb.bm25Search).toHaveBeenCalledWith("test query", 40, "agent-1");
|
||||
});
|
||||
|
||||
it("should pass default agentId when not specified", async () => {
|
||||
mockDb.vectorSearch.mockResolvedValue([]);
|
||||
mockDb.bm25Search.mockResolvedValue([]);
|
||||
|
||||
await hybridSearch(
|
||||
mockDb as unknown as Neo4jMemoryClient,
|
||||
mockEmbeddings as unknown as Embeddings,
|
||||
"test query",
|
||||
);
|
||||
|
||||
expect(mockDb.vectorSearch).toHaveBeenCalledWith(
|
||||
expect.any(Array),
|
||||
expect.any(Number),
|
||||
0.1,
|
||||
"default",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// fuseWithConfidenceRRF()
|
||||
// ============================================================================
|
||||
|
||||
describe("fuseWithConfidenceRRF", () => {
|
||||
function makeSignal(id: string, score: number, text = `Memory ${id}`): SearchSignalResult {
|
||||
return {
|
||||
id,
|
||||
text,
|
||||
category: "fact",
|
||||
importance: 0.7,
|
||||
createdAt: "2025-01-01T00:00:00Z",
|
||||
score,
|
||||
};
|
||||
}
|
||||
|
||||
it("should return empty array when all signals are empty", () => {
|
||||
const result = fuseWithConfidenceRRF([[], [], []], 60, [1.0, 1.0, 1.0]);
|
||||
expect(result).toEqual([]);
|
||||
});
|
||||
|
||||
it("should handle a single signal with results", () => {
|
||||
const signal = [makeSignal("a", 0.9), makeSignal("b", 0.5)];
|
||||
const result = fuseWithConfidenceRRF([signal, [], []], 60, [1.0, 1.0, 1.0]);
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0].id).toBe("a");
|
||||
expect(result[1].id).toBe("b");
|
||||
// First result should have higher RRF score than second
|
||||
expect(result[0].rrfScore).toBeGreaterThan(result[1].rrfScore);
|
||||
});
|
||||
|
||||
it("should boost candidates appearing in multiple signals", () => {
|
||||
const vectorSignal = [makeSignal("shared", 0.9), makeSignal("vec-only", 0.8)];
|
||||
const bm25Signal = [makeSignal("shared", 0.85)];
|
||||
|
||||
const result = fuseWithConfidenceRRF([vectorSignal, bm25Signal, []], 60, [1.0, 1.0, 1.0]);
|
||||
|
||||
// "shared" should rank higher than "vec-only" despite similar scores
|
||||
// because it appears in two signals
|
||||
expect(result[0].id).toBe("shared");
|
||||
expect(result[1].id).toBe("vec-only");
|
||||
});
|
||||
|
||||
it("should handle ties (same score, same rank) consistently", () => {
|
||||
const signal = [makeSignal("a", 0.5), makeSignal("b", 0.5)];
|
||||
const result = fuseWithConfidenceRRF([signal], 60, [1.0]);
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
// With same score, first in signal should have higher RRF (rank 1 vs rank 2)
|
||||
expect(result[0].id).toBe("a");
|
||||
expect(result[1].id).toBe("b");
|
||||
});
|
||||
|
||||
it("should respect different k values", () => {
|
||||
const signal = [makeSignal("a", 0.9), makeSignal("b", 0.5)];
|
||||
|
||||
// Small k amplifies rank differences, large k smooths them
|
||||
const resultSmallK = fuseWithConfidenceRRF([signal], 1, [1.0]);
|
||||
const resultLargeK = fuseWithConfidenceRRF([signal], 1000, [1.0]);
|
||||
|
||||
// The ratio between first and second should be larger with smaller k
|
||||
const ratioSmallK = resultSmallK[0].rrfScore / resultSmallK[1].rrfScore;
|
||||
const ratioLargeK = resultLargeK[0].rrfScore / resultLargeK[1].rrfScore;
|
||||
expect(ratioSmallK).toBeGreaterThan(ratioLargeK);
|
||||
});
|
||||
|
||||
it("should handle zero-score entries", () => {
|
||||
const signal = [makeSignal("a", 0.9), makeSignal("b", 0)];
|
||||
const result = fuseWithConfidenceRRF([signal], 60, [1.0]);
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
// Zero score entry should have zero RRF contribution
|
||||
expect(result[1].rrfScore).toBe(0);
|
||||
expect(result[0].rrfScore).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("should apply signal weights correctly", () => {
|
||||
// Same item appears in two signals with different weights
|
||||
const signal1 = [makeSignal("a", 0.8)];
|
||||
const signal2 = [makeSignal("a", 0.8)];
|
||||
|
||||
const resultEqual = fuseWithConfidenceRRF([signal1, signal2], 60, [1.0, 1.0]);
|
||||
const resultWeighted = fuseWithConfidenceRRF([signal1, signal2], 60, [2.0, 0.5]);
|
||||
|
||||
// Both should have the same item, but weighted version uses different signal contributions
|
||||
expect(resultEqual[0].id).toBe("a");
|
||||
expect(resultWeighted[0].id).toBe("a");
|
||||
// With unequal weights, overall score differs
|
||||
expect(resultEqual[0].rrfScore).not.toBeCloseTo(resultWeighted[0].rrfScore);
|
||||
});
|
||||
|
||||
it("should sort results by RRF score descending", () => {
|
||||
const signal1 = [makeSignal("low", 0.3)];
|
||||
const signal2 = [makeSignal("high", 0.95)];
|
||||
const signal3 = [makeSignal("mid", 0.6)];
|
||||
|
||||
const result = fuseWithConfidenceRRF([signal1, signal2, signal3], 60, [1.0, 1.0, 1.0]);
|
||||
|
||||
expect(result[0].id).toBe("high");
|
||||
expect(result[1].id).toBe("mid");
|
||||
expect(result[2].id).toBe("low");
|
||||
});
|
||||
|
||||
it("should deduplicate within a single signal (keep first occurrence)", () => {
|
||||
const signal = [
|
||||
makeSignal("dup", 0.9),
|
||||
makeSignal("dup", 0.5), // duplicate — should be ignored
|
||||
makeSignal("other", 0.7),
|
||||
];
|
||||
const result = fuseWithConfidenceRRF([signal], 60, [1.0]);
|
||||
|
||||
// "dup" should appear once using its first occurrence (rank 1, score 0.9)
|
||||
const dupEntry = result.find((r) => r.id === "dup");
|
||||
expect(dupEntry).toBeDefined();
|
||||
// Only 2 unique candidates
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
315
extensions/memory-neo4j/search.ts
Normal file
315
extensions/memory-neo4j/search.ts
Normal file
@@ -0,0 +1,315 @@
|
||||
/**
|
||||
* Three-signal hybrid search with query-adaptive RRF fusion.
|
||||
*
|
||||
* Combines:
|
||||
* Signal 1: Vector similarity (HNSW cosine)
|
||||
* Signal 2: BM25 full-text keyword matching
|
||||
* Signal 3: Graph traversal (entity → MENTIONS ← memory)
|
||||
*
|
||||
* Fused using confidence-weighted Reciprocal Rank Fusion (RRF)
|
||||
* with query-adaptive signal weights.
|
||||
*
|
||||
* Adapted from ontology project RRF implementation.
|
||||
*/
|
||||
|
||||
import type { Embeddings } from "./embeddings.js";
|
||||
import type { Neo4jMemoryClient } from "./neo4j-client.js";
|
||||
import type {
|
||||
HybridSearchResult,
|
||||
Logger,
|
||||
SearchSignalResult,
|
||||
SignalAttribution,
|
||||
} from "./schema.js";
|
||||
|
||||
// ============================================================================
|
||||
// Query Classification
|
||||
// ============================================================================
|
||||
|
||||
export type QueryType = "short" | "entity" | "long" | "default";
|
||||
|
||||
/**
|
||||
* Classify a query to determine adaptive signal weights.
|
||||
*
|
||||
* - short (1-2 words): BM25 excels at exact keyword matching
|
||||
* - entity (proper nouns detected): Graph traversal finds connected memories
|
||||
* - long (5+ words): Vector captures semantic intent better
|
||||
* - default: balanced weights
|
||||
*/
|
||||
export function classifyQuery(query: string): QueryType {
|
||||
const words = query.trim().split(/\s+/);
|
||||
const wordCount = words.length;
|
||||
|
||||
// Entity detection: check for capitalized words (proper nouns)
|
||||
// Runs before word count so "John" or "TypeScript" are classified as entity
|
||||
const commonWords =
|
||||
/^(I|A|An|The|Is|Are|Was|Were|What|Who|Where|When|How|Why|Do|Does|Did|Find|Show|Get|Tell|Me|My|About|For)$/;
|
||||
const capitalizedWords = words.filter((w) => /^[A-Z]/.test(w) && !commonWords.test(w));
|
||||
|
||||
if (capitalizedWords.length > 0) {
|
||||
return "entity";
|
||||
}
|
||||
|
||||
// Short queries: 1-2 words → boost BM25
|
||||
if (wordCount <= 2) {
|
||||
return "short";
|
||||
}
|
||||
|
||||
// Question patterns targeting entities (3-4 word queries only,
|
||||
// so generic long questions like "what is the best framework" fall through to "long")
|
||||
if (wordCount <= 4 && /^(who|where|what)\s+(is|does|did|was|were)\s/i.test(query)) {
|
||||
return "entity";
|
||||
}
|
||||
|
||||
// Long queries: 5+ words → boost vector
|
||||
if (wordCount >= 5) {
|
||||
return "long";
|
||||
}
|
||||
|
||||
return "default";
|
||||
}
|
||||
|
||||
/**
|
||||
* Get adaptive signal weights based on query type.
|
||||
* Returns [vectorWeight, bm25Weight, graphWeight].
|
||||
*
|
||||
* Decision Q7: Query-adaptive RRF weights
|
||||
* - Short → boost BM25 (keyword matching)
|
||||
* - Entity → boost graph (relationship traversal)
|
||||
* - Long → boost vector (semantic similarity)
|
||||
*/
|
||||
export function getAdaptiveWeights(
|
||||
queryType: QueryType,
|
||||
graphEnabled: boolean,
|
||||
): [number, number, number] {
|
||||
const graphBase = graphEnabled ? 1.0 : 0.0;
|
||||
|
||||
switch (queryType) {
|
||||
case "short":
|
||||
return [0.8, 1.2, graphBase * 1.0];
|
||||
case "entity":
|
||||
return [0.8, 1.0, graphBase * 1.3];
|
||||
case "long":
|
||||
return [1.2, 0.7, graphBase * 0.8];
|
||||
case "default":
|
||||
default:
|
||||
return [1.0, 1.0, graphBase * 1.0];
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Confidence-Weighted RRF Fusion
|
||||
// ============================================================================
|
||||
|
||||
type SignalEntry = {
|
||||
rank: number; // 1-indexed
|
||||
score: number; // 0-1 normalized
|
||||
};
|
||||
|
||||
type FusedCandidate = {
|
||||
id: string;
|
||||
text: string;
|
||||
category: string;
|
||||
importance: number;
|
||||
createdAt: string;
|
||||
rrfScore: number;
|
||||
taskId?: string;
|
||||
signals: {
|
||||
vector: SignalAttribution;
|
||||
bm25: SignalAttribution;
|
||||
graph: SignalAttribution;
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Fuse multiple search signals using confidence-weighted RRF.
|
||||
*
|
||||
* Formula: RRF_conf(d) = Σ w_i × score_i(d) / (k + rank_i(d))
|
||||
*
|
||||
* Unlike standard RRF which only uses ranks, this variant preserves
|
||||
* score magnitude: rank-1 with score 0.99 contributes more than
|
||||
* rank-1 with score 0.55.
|
||||
*
|
||||
* Reference: Cormack et al. (2009), extended with confidence weighting.
|
||||
*/
|
||||
export function fuseWithConfidenceRRF(
|
||||
signals: SearchSignalResult[][],
|
||||
k: number,
|
||||
weights: number[],
|
||||
): FusedCandidate[] {
|
||||
// Build per-signal rank/score lookups
|
||||
const signalMaps: Map<string, SignalEntry>[] = signals.map((signal) => {
|
||||
const map = new Map<string, SignalEntry>();
|
||||
for (let i = 0; i < signal.length; i++) {
|
||||
const entry = signal[i];
|
||||
// If duplicate in same signal, keep first (higher ranked)
|
||||
if (!map.has(entry.id)) {
|
||||
map.set(entry.id, { rank: i + 1, score: entry.score });
|
||||
}
|
||||
}
|
||||
return map;
|
||||
});
|
||||
|
||||
// Collect all unique candidate IDs with their metadata
|
||||
const candidateMetadata = new Map<
|
||||
string,
|
||||
{ text: string; category: string; importance: number; createdAt: string; taskId?: string }
|
||||
>();
|
||||
|
||||
for (const signal of signals) {
|
||||
for (const entry of signal) {
|
||||
if (!candidateMetadata.has(entry.id)) {
|
||||
candidateMetadata.set(entry.id, {
|
||||
text: entry.text,
|
||||
category: entry.category,
|
||||
importance: entry.importance,
|
||||
createdAt: entry.createdAt,
|
||||
taskId: entry.taskId,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate confidence-weighted RRF score for each candidate
|
||||
const results: FusedCandidate[] = [];
|
||||
const NO_SIGNAL: SignalAttribution = { rank: 0, score: 0 };
|
||||
|
||||
for (const [id, meta] of candidateMetadata) {
|
||||
let rrfScore = 0;
|
||||
|
||||
for (let i = 0; i < signalMaps.length; i++) {
|
||||
const entry = signalMaps[i].get(id);
|
||||
if (entry && entry.rank > 0) {
|
||||
// Confidence-weighted: multiply by original score
|
||||
rrfScore += weights[i] * entry.score * (1 / (k + entry.rank));
|
||||
}
|
||||
}
|
||||
|
||||
// Build per-signal attribution from the existing signal maps
|
||||
const signals = {
|
||||
vector: signalMaps[0]?.get(id) ?? NO_SIGNAL,
|
||||
bm25: signalMaps[1]?.get(id) ?? NO_SIGNAL,
|
||||
graph: signalMaps[2]?.get(id) ?? NO_SIGNAL,
|
||||
};
|
||||
|
||||
results.push({
|
||||
id,
|
||||
text: meta.text,
|
||||
category: meta.category,
|
||||
importance: meta.importance,
|
||||
createdAt: meta.createdAt,
|
||||
rrfScore,
|
||||
taskId: meta.taskId,
|
||||
signals,
|
||||
});
|
||||
}
|
||||
|
||||
// Sort by RRF score descending
|
||||
results.sort((a, b) => b.rrfScore - a.rrfScore);
|
||||
return results;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Hybrid Search Orchestrator
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Perform a three-signal hybrid search with query-adaptive RRF fusion.
|
||||
*
|
||||
* 1. Embed the query
|
||||
* 2. Classify query for adaptive weights
|
||||
* 3. Run three signals in parallel
|
||||
* 4. Fuse with confidence-weighted RRF
|
||||
* 5. Return top results
|
||||
*
|
||||
* Graceful degradation: if any signal fails, RRF works with remaining signals.
|
||||
* If graph search is not enabled (no extraction API key), uses 2-signal fusion.
|
||||
*/
|
||||
export async function hybridSearch(
|
||||
db: Neo4jMemoryClient,
|
||||
embeddings: Embeddings,
|
||||
query: string,
|
||||
limit: number = 5,
|
||||
agentId: string = "default",
|
||||
graphEnabled: boolean = false,
|
||||
options: {
|
||||
rrfK?: number;
|
||||
candidateMultiplier?: number;
|
||||
graphFiringThreshold?: number;
|
||||
graphSearchDepth?: number;
|
||||
logger?: Logger;
|
||||
} = {},
|
||||
): Promise<HybridSearchResult[]> {
|
||||
// Guard against empty queries
|
||||
if (!query.trim()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const {
|
||||
rrfK = 60,
|
||||
candidateMultiplier = 4,
|
||||
graphFiringThreshold = 0.3,
|
||||
graphSearchDepth = 1,
|
||||
logger,
|
||||
} = options;
|
||||
|
||||
const candidateLimit = Math.floor(Math.min(200, Math.max(1, limit * candidateMultiplier)));
|
||||
|
||||
// 1. Generate query embedding
|
||||
const t0 = performance.now();
|
||||
const queryEmbedding = await embeddings.embed(query);
|
||||
const tEmbed = performance.now();
|
||||
|
||||
// 2. Classify query and get adaptive weights
|
||||
const queryType = classifyQuery(query);
|
||||
const weights = getAdaptiveWeights(queryType, graphEnabled);
|
||||
|
||||
// 3. Run signals in parallel
|
||||
const [vectorResults, bm25Results, graphResults] = await Promise.all([
|
||||
db.vectorSearch(queryEmbedding, candidateLimit, 0.1, agentId),
|
||||
db.bm25Search(query, candidateLimit, agentId),
|
||||
graphEnabled
|
||||
? db.graphSearch(query, candidateLimit, graphFiringThreshold, agentId, graphSearchDepth)
|
||||
: Promise.resolve([] as SearchSignalResult[]),
|
||||
]);
|
||||
const tSignals = performance.now();
|
||||
|
||||
// 4. Fuse with confidence-weighted RRF
|
||||
const fused = fuseWithConfidenceRRF([vectorResults, bm25Results, graphResults], rrfK, weights);
|
||||
const tFuse = performance.now();
|
||||
|
||||
// 5. Return top results, normalized to 0-100% display scores.
|
||||
// Only normalize when maxRrf is above a minimum threshold to avoid
|
||||
// inflating weak matches (e.g., a single low-score result becoming 1.0).
|
||||
const maxRrf = fused.length > 0 ? fused[0].rrfScore : 0;
|
||||
const MIN_RRF_FOR_NORMALIZATION = 0.01;
|
||||
const normalizer = maxRrf >= MIN_RRF_FOR_NORMALIZATION ? 1 / maxRrf : 1;
|
||||
|
||||
const results = fused.slice(0, limit).map((r) => ({
|
||||
id: r.id,
|
||||
text: r.text,
|
||||
category: r.category,
|
||||
importance: r.importance,
|
||||
createdAt: r.createdAt,
|
||||
score: Math.min(1, r.rrfScore * normalizer), // Normalize to 0-1
|
||||
taskId: r.taskId,
|
||||
signals: r.signals,
|
||||
}));
|
||||
|
||||
// 6. Record retrieval events (fire-and-forget for latency)
|
||||
// This tracks which memories are actually being used, enabling
|
||||
// retrieval-based importance adjustment.
|
||||
if (results.length > 0) {
|
||||
const memoryIds = results.map((r) => r.id);
|
||||
db.recordRetrievals(memoryIds).catch(() => {
|
||||
// Silently ignore - retrieval tracking is non-critical
|
||||
});
|
||||
}
|
||||
|
||||
// Log search timing breakdown
|
||||
logger?.info?.(
|
||||
`memory-neo4j: [bench] hybridSearch ${(tFuse - t0).toFixed(0)}ms (embed=${(tEmbed - t0).toFixed(0)}ms, signals=${(tSignals - tEmbed).toFixed(0)}ms, fuse=${(tFuse - tSignals).toFixed(0)}ms) ` +
|
||||
`type=${queryType} vec=${vectorResults.length} bm25=${bm25Results.length} graph=${graphResults.length} → ${results.length} results`,
|
||||
);
|
||||
|
||||
return results;
|
||||
}
|
||||
165
extensions/memory-neo4j/sleep-cycle.credential-scan.test.ts
Normal file
165
extensions/memory-neo4j/sleep-cycle.credential-scan.test.ts
Normal file
@@ -0,0 +1,165 @@
|
||||
/**
|
||||
* Tests for credential scanning in the sleep cycle.
|
||||
*
|
||||
* Verifies that CREDENTIAL_PATTERNS and detectCredential() correctly
|
||||
* identify credential-like content in memory text while not flagging
|
||||
* clean text.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { CREDENTIAL_PATTERNS, detectCredential } from "./sleep-cycle.js";
|
||||
|
||||
describe("Credential Detection", () => {
|
||||
// --------------------------------------------------------------------------
|
||||
// detectCredential() — should flag dangerous content
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe("should detect credentials", () => {
|
||||
it("detects API keys (sk-...)", () => {
|
||||
const result = detectCredential("Use the key sk-abc123def456ghi789jkl012mno345");
|
||||
expect(result).toBe("API key");
|
||||
});
|
||||
|
||||
it("detects api_key patterns", () => {
|
||||
const result = detectCredential("Set api_key_live_abcdef1234567890abcdef");
|
||||
expect(result).toBe("API key");
|
||||
});
|
||||
|
||||
it("detects Bearer tokens", () => {
|
||||
const result = detectCredential(
|
||||
"Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.payload.signature",
|
||||
);
|
||||
// Could match either Bearer token or JWT — both are valid detections
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it("detects password assignments (password: X)", () => {
|
||||
const result = detectCredential("The database password: myS3cretP@ss!");
|
||||
expect(result).toBe("Password assignment");
|
||||
});
|
||||
|
||||
it("detects password assignments (password=X)", () => {
|
||||
const result = detectCredential("config has password=hunter2 in it");
|
||||
expect(result).toBe("Password assignment");
|
||||
});
|
||||
|
||||
it("detects the missed pattern: login with X creds user/pass", () => {
|
||||
const result = detectCredential("login with radarr creds hullah/fuckbar");
|
||||
expect(result).toBe("Credentials (user/pass)");
|
||||
});
|
||||
|
||||
it("detects creds user/pass without login prefix", () => {
|
||||
const result = detectCredential("use creds admin/password123 for the server");
|
||||
expect(result).toBe("Credentials (user/pass)");
|
||||
});
|
||||
|
||||
it("detects URL-embedded credentials", () => {
|
||||
const result = detectCredential("Connect to https://admin:secretpass@db.example.com/mydb");
|
||||
expect(result).toBe("URL credentials");
|
||||
});
|
||||
|
||||
it("detects URL credentials with http://", () => {
|
||||
const result = detectCredential("http://user:pass@192.168.1.1:8080/api");
|
||||
expect(result).toBe("URL credentials");
|
||||
});
|
||||
|
||||
it("detects private keys", () => {
|
||||
const result = detectCredential("-----BEGIN RSA PRIVATE KEY-----\nMIIEow...");
|
||||
expect(result).toBe("Private key");
|
||||
});
|
||||
|
||||
it("detects AWS access keys", () => {
|
||||
const result = detectCredential("AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE");
|
||||
expect(result).toBe("AWS key");
|
||||
});
|
||||
|
||||
it("detects GitHub personal access tokens", () => {
|
||||
const result = detectCredential("Set GITHUB_TOKEN=ghp_ABCDEFabcdef1234567890");
|
||||
expect(result).toBe("GitHub/GitLab token");
|
||||
});
|
||||
|
||||
it("detects GitLab tokens", () => {
|
||||
const result = detectCredential("Use glpat-xxxxxxxxxxxxxxxxxxxx for auth");
|
||||
expect(result).toBe("GitHub/GitLab token");
|
||||
});
|
||||
|
||||
it("detects JWT tokens", () => {
|
||||
const result = detectCredential(
|
||||
"Token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c",
|
||||
);
|
||||
expect(result).toBe("JWT");
|
||||
});
|
||||
|
||||
it("detects token=value patterns", () => {
|
||||
const result = detectCredential(
|
||||
"Set token=abcdef1234567890abcdef1234567890ab for authentication",
|
||||
);
|
||||
expect(result).toBe("Token/secret");
|
||||
});
|
||||
|
||||
it("detects secret: value patterns", () => {
|
||||
const result = detectCredential(
|
||||
"The client secret: abcdef1234567890abcdef1234567890abcdef12",
|
||||
);
|
||||
expect(result).toBe("Token/secret");
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// detectCredential() — should NOT flag clean text
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe("should not flag clean text", () => {
|
||||
it("does not flag normal text", () => {
|
||||
expect(detectCredential("Remember to buy groceries tomorrow")).toBeNull();
|
||||
});
|
||||
|
||||
it("does not flag password advice (without actual password)", () => {
|
||||
expect(
|
||||
detectCredential("Make sure the password is at least 8 characters long for security"),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("does not flag discussion about tokens", () => {
|
||||
expect(detectCredential("We should use JWT tokens for authentication")).toBeNull();
|
||||
});
|
||||
|
||||
it("does not flag short key-like words", () => {
|
||||
expect(detectCredential("The key to success is persistence")).toBeNull();
|
||||
});
|
||||
|
||||
it("does not flag URLs without credentials", () => {
|
||||
expect(detectCredential("Visit https://example.com/api/v1 for docs")).toBeNull();
|
||||
});
|
||||
|
||||
it("does not flag discussion about API key rotation", () => {
|
||||
expect(detectCredential("Rotate your API keys every 90 days as a best practice")).toBeNull();
|
||||
});
|
||||
|
||||
it("does not flag file paths", () => {
|
||||
expect(detectCredential("Credentials are stored in /home/user/.secrets/api.json")).toBeNull();
|
||||
});
|
||||
|
||||
it("does not flag casual use of slash in text", () => {
|
||||
expect(detectCredential("Use the read/write mode for better performance")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// CREDENTIAL_PATTERNS — structural checks
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe("CREDENTIAL_PATTERNS structure", () => {
|
||||
it("has at least 8 patterns", () => {
|
||||
expect(CREDENTIAL_PATTERNS.length).toBeGreaterThanOrEqual(8);
|
||||
});
|
||||
|
||||
it("each pattern has a label and valid RegExp", () => {
|
||||
for (const { pattern, label } of CREDENTIAL_PATTERNS) {
|
||||
expect(pattern).toBeInstanceOf(RegExp);
|
||||
expect(label).toBeTruthy();
|
||||
expect(typeof label).toBe("string");
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
234
extensions/memory-neo4j/sleep-cycle.task-memory.test.ts
Normal file
234
extensions/memory-neo4j/sleep-cycle.task-memory.test.ts
Normal file
@@ -0,0 +1,234 @@
|
||||
/**
|
||||
* Tests for Phase 7: Task-Memory Cleanup in the sleep cycle.
|
||||
*
|
||||
* Tests the LLM classification function and integration with the sleep cycle.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import type { ExtractionConfig } from "./config.js";
|
||||
import { classifyTaskMemory } from "./sleep-cycle.js";
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Mock the LLM client so we don't make real API calls
|
||||
// --------------------------------------------------------------------------
|
||||
vi.mock("./llm-client.js", () => ({
|
||||
callOpenRouter: vi.fn(),
|
||||
callOpenRouterStream: vi.fn(),
|
||||
isTransientError: vi.fn(() => false),
|
||||
}));
|
||||
|
||||
// Import the mocked function for controlling behavior per test
|
||||
import { callOpenRouter } from "./llm-client.js";
|
||||
const mockCallOpenRouter = vi.mocked(callOpenRouter);
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
const baseConfig: ExtractionConfig = {
|
||||
enabled: true,
|
||||
apiKey: "test-key",
|
||||
model: "test-model",
|
||||
baseUrl: "http://localhost:8080",
|
||||
temperature: 0,
|
||||
maxRetries: 0,
|
||||
};
|
||||
|
||||
const disabledConfig: ExtractionConfig = {
|
||||
...baseConfig,
|
||||
enabled: false,
|
||||
};
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// classifyTaskMemory()
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe("classifyTaskMemory", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("returns 'noise' for task-specific progress memory", async () => {
|
||||
mockCallOpenRouter.mockResolvedValueOnce(
|
||||
JSON.stringify({
|
||||
classification: "noise",
|
||||
reason: "This is task-specific progress tracking",
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await classifyTaskMemory(
|
||||
"Currently working on TASK-003, step 2: fixing the column alignment in the LinkedIn dashboard",
|
||||
"Fix LinkedIn Dashboard tab",
|
||||
baseConfig,
|
||||
);
|
||||
|
||||
expect(result).toBe("noise");
|
||||
expect(mockCallOpenRouter).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("returns 'lasting' for decision/fact memory", async () => {
|
||||
mockCallOpenRouter.mockResolvedValueOnce(
|
||||
JSON.stringify({
|
||||
classification: "lasting",
|
||||
reason: "Contains a reusable technical decision",
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await classifyTaskMemory(
|
||||
"ReActor face swap produces better results than Replicate for video face replacement",
|
||||
"Implement face swap pipeline",
|
||||
baseConfig,
|
||||
);
|
||||
|
||||
expect(result).toBe("lasting");
|
||||
expect(mockCallOpenRouter).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("returns 'lasting' when LLM returns null (conservative)", async () => {
|
||||
mockCallOpenRouter.mockResolvedValueOnce(null);
|
||||
|
||||
const result = await classifyTaskMemory("Some ambiguous memory", "Some task", baseConfig);
|
||||
|
||||
expect(result).toBe("lasting");
|
||||
});
|
||||
|
||||
it("returns 'lasting' when LLM throws (conservative)", async () => {
|
||||
mockCallOpenRouter.mockRejectedValueOnce(new Error("network error"));
|
||||
|
||||
const result = await classifyTaskMemory("Some memory", "Some task", baseConfig);
|
||||
|
||||
expect(result).toBe("lasting");
|
||||
});
|
||||
|
||||
it("returns 'lasting' when LLM returns malformed JSON", async () => {
|
||||
mockCallOpenRouter.mockResolvedValueOnce("not json at all");
|
||||
|
||||
const result = await classifyTaskMemory("Some memory", "Some task", baseConfig);
|
||||
|
||||
expect(result).toBe("lasting");
|
||||
});
|
||||
|
||||
it("returns 'lasting' when LLM returns unexpected classification", async () => {
|
||||
mockCallOpenRouter.mockResolvedValueOnce(JSON.stringify({ classification: "unknown_value" }));
|
||||
|
||||
const result = await classifyTaskMemory("Some memory", "Some task", baseConfig);
|
||||
|
||||
expect(result).toBe("lasting");
|
||||
});
|
||||
|
||||
it("returns 'lasting' when config is disabled", async () => {
|
||||
const result = await classifyTaskMemory("Task progress memory", "Some task", disabledConfig);
|
||||
|
||||
expect(result).toBe("lasting");
|
||||
expect(mockCallOpenRouter).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes task title in system prompt", async () => {
|
||||
mockCallOpenRouter.mockResolvedValueOnce(JSON.stringify({ classification: "lasting" }));
|
||||
|
||||
await classifyTaskMemory("Memory text here", "Fix LinkedIn Dashboard tab", baseConfig);
|
||||
|
||||
expect(mockCallOpenRouter).toHaveBeenCalledOnce();
|
||||
const callArgs = mockCallOpenRouter.mock.calls[0];
|
||||
const messages = callArgs[1] as Array<{ role: string; content: string }>;
|
||||
expect(messages[0].content).toContain("Fix LinkedIn Dashboard tab");
|
||||
});
|
||||
|
||||
it("passes memory text as user message", async () => {
|
||||
mockCallOpenRouter.mockResolvedValueOnce(JSON.stringify({ classification: "noise" }));
|
||||
|
||||
await classifyTaskMemory(
|
||||
"Debugging step: checked column B3 alignment",
|
||||
"Fix Dashboard",
|
||||
baseConfig,
|
||||
);
|
||||
|
||||
const callArgs = mockCallOpenRouter.mock.calls[0];
|
||||
const messages = callArgs[1] as Array<{ role: string; content: string }>;
|
||||
expect(messages[1].role).toBe("user");
|
||||
expect(messages[1].content).toBe("Debugging step: checked column B3 alignment");
|
||||
});
|
||||
|
||||
it("passes abort signal to LLM call", async () => {
|
||||
const controller = new AbortController();
|
||||
mockCallOpenRouter.mockResolvedValueOnce(JSON.stringify({ classification: "lasting" }));
|
||||
|
||||
await classifyTaskMemory("Memory text", "Task title", baseConfig, controller.signal);
|
||||
|
||||
const callArgs = mockCallOpenRouter.mock.calls[0];
|
||||
expect(callArgs[2]).toBe(controller.signal);
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Classification examples — verify the prompt produces expected behavior
|
||||
// These test that noise vs lasting classification is passed through correctly
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe("classifyTaskMemory classification examples", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
const noiseExamples = [
|
||||
{
|
||||
memory: "Currently working on TASK-003, step 2: fixing the column alignment",
|
||||
task: "Fix LinkedIn Dashboard tab",
|
||||
reason: "task progress update",
|
||||
},
|
||||
{
|
||||
memory: "ACTIVE TASK: TASK-004 — Fix browser port collision. Step: testing port 18807",
|
||||
task: "Fix browser port collision",
|
||||
reason: "active task checkpoint",
|
||||
},
|
||||
{
|
||||
memory: "Debugging the flight search: Scoot API returned 500, retrying with different dates",
|
||||
task: "Book KL↔Singapore flights for India trip",
|
||||
reason: "debugging steps",
|
||||
},
|
||||
];
|
||||
|
||||
for (const example of noiseExamples) {
|
||||
it(`classifies "${example.reason}" as noise`, async () => {
|
||||
mockCallOpenRouter.mockResolvedValueOnce(
|
||||
JSON.stringify({ classification: "noise", reason: example.reason }),
|
||||
);
|
||||
|
||||
const result = await classifyTaskMemory(example.memory, example.task, baseConfig);
|
||||
|
||||
expect(result).toBe("noise");
|
||||
});
|
||||
}
|
||||
|
||||
const lastingExamples = [
|
||||
{
|
||||
memory:
|
||||
"Port map: 18792 (chrome), 18800 (chetan), 18805 (linkedin), 18806 (tsukhani), 18807 (openclaw)",
|
||||
task: "Fix browser port collision",
|
||||
reason: "useful reference configuration",
|
||||
},
|
||||
{
|
||||
memory:
|
||||
"Dashboard layout: B3:B9 = Total, Accepted, Pending, Not Connected, Follow-ups Sent, Acceptance Rate%, Date",
|
||||
task: "Fix LinkedIn Dashboard tab",
|
||||
reason: "lasting documentation of layout",
|
||||
},
|
||||
{
|
||||
memory: "ReActor face swap produces better results than Replicate for video face replacement",
|
||||
task: "Implement face swap pipeline",
|
||||
reason: "tool comparison decision",
|
||||
},
|
||||
];
|
||||
|
||||
for (const example of lastingExamples) {
|
||||
it(`classifies "${example.reason}" as lasting`, async () => {
|
||||
mockCallOpenRouter.mockResolvedValueOnce(
|
||||
JSON.stringify({ classification: "lasting", reason: example.reason }),
|
||||
);
|
||||
|
||||
const result = await classifyTaskMemory(example.memory, example.task, baseConfig);
|
||||
|
||||
expect(result).toBe("lasting");
|
||||
});
|
||||
}
|
||||
});
|
||||
1168
extensions/memory-neo4j/sleep-cycle.ts
Normal file
1168
extensions/memory-neo4j/sleep-cycle.ts
Normal file
File diff suppressed because it is too large
Load Diff
426
extensions/memory-neo4j/task-filter.test.ts
Normal file
426
extensions/memory-neo4j/task-filter.test.ts
Normal file
@@ -0,0 +1,426 @@
|
||||
/**
|
||||
* Tests for task-filter.ts — Task-aware recall filtering (Layer 1).
|
||||
*
|
||||
* Verifies that memories related to completed tasks are correctly identified
|
||||
* and filtered, while unrelated or loosely-matching memories are preserved.
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildCompletedTaskInfo,
|
||||
clearTaskFilterCache,
|
||||
extractSignificantKeywords,
|
||||
isRelatedToCompletedTask,
|
||||
loadCompletedTaskKeywords,
|
||||
type CompletedTaskInfo,
|
||||
} from "./task-filter.js";
|
||||
|
||||
// ============================================================================
|
||||
// Sample TASKS.md content
|
||||
// ============================================================================
|
||||
|
||||
const SAMPLE_TASKS_MD = `# Active Tasks
|
||||
|
||||
_No active tasks_
|
||||
|
||||
# Completed
|
||||
<!-- Move done tasks here with completion date -->
|
||||
## TASK-002: Book KL↔Singapore flights for India trip
|
||||
- **Completed:** 2026-02-16
|
||||
- **Details:** Tarun booked manually — Scoot TR453 (Feb 23 KUL→SIN) and AirAsia AK720 (Mar 3 SIN→KUL)
|
||||
|
||||
## TASK-003: Fix LinkedIn Dashboard tab
|
||||
- **Completed:** 2026-02-16
|
||||
- **Details:** Fixed misaligned stats, wrong industry numbers, stale data. Added Not Connected row, consolidated industries into 10 groups, cleared residual data.
|
||||
|
||||
## TASK-004: Fix browser port collision
|
||||
- **Completed:** 2026-02-16
|
||||
- **Details:** Added explicit openclaw profile on port 18807 (was colliding with chetan on 18800)
|
||||
`;
|
||||
|
||||
// ============================================================================
|
||||
// extractSignificantKeywords()
|
||||
// ============================================================================
|
||||
|
||||
describe("extractSignificantKeywords", () => {
|
||||
it("extracts words with length >= 4", () => {
|
||||
const keywords = extractSignificantKeywords("Fix the big dashboard bug");
|
||||
expect(keywords).toContain("dashboard");
|
||||
expect(keywords).not.toContain("fix"); // too short
|
||||
expect(keywords).not.toContain("the"); // too short
|
||||
expect(keywords).not.toContain("big"); // too short
|
||||
expect(keywords).not.toContain("bug"); // too short
|
||||
});
|
||||
|
||||
it("removes stop words", () => {
|
||||
const keywords = extractSignificantKeywords("should have been using this work");
|
||||
// All of these are stop words
|
||||
expect(keywords).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("lowercases all keywords", () => {
|
||||
const keywords = extractSignificantKeywords("LinkedIn Dashboard Singapore");
|
||||
expect(keywords).toContain("linkedin");
|
||||
expect(keywords).toContain("dashboard");
|
||||
expect(keywords).toContain("singapore");
|
||||
});
|
||||
|
||||
it("deduplicates keywords", () => {
|
||||
const keywords = extractSignificantKeywords("dashboard dashboard dashboard");
|
||||
expect(keywords).toEqual(["dashboard"]);
|
||||
});
|
||||
|
||||
it("returns empty for empty/null input", () => {
|
||||
expect(extractSignificantKeywords("")).toEqual([]);
|
||||
expect(extractSignificantKeywords(null as unknown as string)).toEqual([]);
|
||||
});
|
||||
|
||||
it("handles special characters", () => {
|
||||
const keywords = extractSignificantKeywords("port 18807 (colliding with chetan)");
|
||||
expect(keywords).toContain("port");
|
||||
expect(keywords).toContain("18807");
|
||||
expect(keywords).toContain("colliding");
|
||||
expect(keywords).toContain("chetan");
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// buildCompletedTaskInfo()
|
||||
// ============================================================================
|
||||
|
||||
describe("buildCompletedTaskInfo", () => {
|
||||
it("extracts keywords from title and details", () => {
|
||||
const info = buildCompletedTaskInfo({
|
||||
id: "TASK-003",
|
||||
title: "Fix LinkedIn Dashboard tab",
|
||||
status: "done",
|
||||
details:
|
||||
"Fixed misaligned stats, wrong industry numbers, stale data. Added Not Connected row, consolidated industries into 10 groups, cleared residual data.",
|
||||
rawLines: [
|
||||
"## TASK-003: Fix LinkedIn Dashboard tab",
|
||||
"- **Completed:** 2026-02-16",
|
||||
"- **Details:** Fixed misaligned stats, wrong industry numbers, stale data.",
|
||||
],
|
||||
isCompleted: true,
|
||||
});
|
||||
|
||||
expect(info.id).toBe("TASK-003");
|
||||
expect(info.keywords).toContain("linkedin");
|
||||
expect(info.keywords).toContain("dashboard");
|
||||
expect(info.keywords).toContain("misaligned");
|
||||
expect(info.keywords).toContain("stats");
|
||||
expect(info.keywords).toContain("industry");
|
||||
});
|
||||
|
||||
it("includes currentStep keywords", () => {
|
||||
const info = buildCompletedTaskInfo({
|
||||
id: "TASK-010",
|
||||
title: "Deploy staging server",
|
||||
status: "done",
|
||||
currentStep: "Verifying nginx configuration",
|
||||
rawLines: ["## TASK-010: Deploy staging server"],
|
||||
isCompleted: true,
|
||||
});
|
||||
|
||||
expect(info.keywords).toContain("deploy");
|
||||
expect(info.keywords).toContain("staging");
|
||||
expect(info.keywords).toContain("server");
|
||||
expect(info.keywords).toContain("nginx");
|
||||
expect(info.keywords).toContain("configuration");
|
||||
});
|
||||
|
||||
it("handles task with minimal fields", () => {
|
||||
const info = buildCompletedTaskInfo({
|
||||
id: "TASK-001",
|
||||
title: "Quick fix",
|
||||
status: "done",
|
||||
rawLines: ["## TASK-001: Quick fix"],
|
||||
isCompleted: true,
|
||||
});
|
||||
|
||||
expect(info.id).toBe("TASK-001");
|
||||
expect(info.keywords).toContain("quick");
|
||||
// "fix" is only 3 chars, should be excluded
|
||||
expect(info.keywords).not.toContain("fix");
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// isRelatedToCompletedTask()
|
||||
// ============================================================================
|
||||
|
||||
describe("isRelatedToCompletedTask", () => {
|
||||
const completedTasks: CompletedTaskInfo[] = [
|
||||
{
|
||||
id: "TASK-002",
|
||||
keywords: [
|
||||
"book",
|
||||
"singapore",
|
||||
"flights",
|
||||
"india",
|
||||
"trip",
|
||||
"scoot",
|
||||
"tr453",
|
||||
"airasia",
|
||||
"ak720",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "TASK-003",
|
||||
keywords: [
|
||||
"linkedin",
|
||||
"dashboard",
|
||||
"misaligned",
|
||||
"stats",
|
||||
"industry",
|
||||
"numbers",
|
||||
"stale",
|
||||
"connected",
|
||||
"consolidated",
|
||||
"industries",
|
||||
"groups",
|
||||
"cleared",
|
||||
"residual",
|
||||
"data",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "TASK-004",
|
||||
keywords: [
|
||||
"browser",
|
||||
"port",
|
||||
"collision",
|
||||
"openclaw",
|
||||
"profile",
|
||||
"18807",
|
||||
"colliding",
|
||||
"chetan",
|
||||
"18800",
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
// --- Task ID matching ---
|
||||
|
||||
it("matches memory containing task ID", () => {
|
||||
expect(
|
||||
isRelatedToCompletedTask("TASK-002 flights have been booked successfully", completedTasks),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("matches task ID case-insensitively", () => {
|
||||
expect(
|
||||
isRelatedToCompletedTask("Completed task-003 — dashboard is fixed", completedTasks),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
// --- Keyword matching ---
|
||||
|
||||
it("matches memory with 2+ keywords from a completed task", () => {
|
||||
expect(
|
||||
isRelatedToCompletedTask(
|
||||
"LinkedIn dashboard stats are now showing correctly",
|
||||
completedTasks,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("matches memory with keywords from flight task", () => {
|
||||
expect(
|
||||
isRelatedToCompletedTask("Booked Singapore flights for the India trip", completedTasks),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
// --- False positive prevention ---
|
||||
|
||||
it("does NOT match memory with only 1 keyword overlap", () => {
|
||||
expect(isRelatedToCompletedTask("Singapore has great food markets", completedTasks)).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("does NOT match memory about LinkedIn that is unrelated to dashboard fix", () => {
|
||||
// "linkedin" alone is only 1 keyword match — should NOT be filtered
|
||||
expect(
|
||||
isRelatedToCompletedTask(
|
||||
"LinkedIn connection request from John Smith accepted",
|
||||
completedTasks,
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("does NOT match memory about browser that is unrelated to port fix", () => {
|
||||
// "browser" alone is only 1 keyword
|
||||
expect(
|
||||
isRelatedToCompletedTask("Browser extension for Flux image generation", completedTasks),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("does NOT match completely unrelated memory", () => {
|
||||
expect(isRelatedToCompletedTask("Tarun's birthday is August 23, 1974", completedTasks)).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
// --- Edge cases ---
|
||||
|
||||
it("returns false for empty memory text", () => {
|
||||
expect(isRelatedToCompletedTask("", completedTasks)).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false for empty completed tasks array", () => {
|
||||
expect(isRelatedToCompletedTask("TASK-002 flights booked", [])).toBe(false);
|
||||
});
|
||||
|
||||
it("handles task with no keywords (only ID matching works)", () => {
|
||||
const tasksNoKeywords: CompletedTaskInfo[] = [{ id: "TASK-099", keywords: [] }];
|
||||
expect(isRelatedToCompletedTask("Completed TASK-099", tasksNoKeywords)).toBe(true);
|
||||
expect(isRelatedToCompletedTask("Some random memory", tasksNoKeywords)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// loadCompletedTaskKeywords()
|
||||
// ============================================================================
|
||||
|
||||
describe("loadCompletedTaskKeywords", () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "task-filter-test-"));
|
||||
clearTaskFilterCache();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
clearTaskFilterCache();
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("parses completed tasks from TASKS.md", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "TASKS.md"), SAMPLE_TASKS_MD);
|
||||
|
||||
const tasks = await loadCompletedTaskKeywords(tmpDir);
|
||||
expect(tasks).toHaveLength(3);
|
||||
expect(tasks.map((t) => t.id)).toEqual(["TASK-002", "TASK-003", "TASK-004"]);
|
||||
});
|
||||
|
||||
it("extracts keywords from completed tasks", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "TASKS.md"), SAMPLE_TASKS_MD);
|
||||
|
||||
const tasks = await loadCompletedTaskKeywords(tmpDir);
|
||||
const flightTask = tasks.find((t) => t.id === "TASK-002");
|
||||
expect(flightTask).toBeDefined();
|
||||
expect(flightTask!.keywords).toContain("singapore");
|
||||
expect(flightTask!.keywords).toContain("flights");
|
||||
});
|
||||
|
||||
it("returns empty array when TASKS.md does not exist", async () => {
|
||||
const tasks = await loadCompletedTaskKeywords(tmpDir);
|
||||
expect(tasks).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns empty array for empty TASKS.md", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "TASKS.md"), "");
|
||||
|
||||
const tasks = await loadCompletedTaskKeywords(tmpDir);
|
||||
expect(tasks).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns empty array for TASKS.md with no completed tasks", async () => {
|
||||
const content = `# Active Tasks
|
||||
|
||||
## TASK-001: Do something
|
||||
- **Status:** in_progress
|
||||
- **Details:** Working on it
|
||||
|
||||
# Completed
|
||||
<!-- Move done tasks here with completion date -->
|
||||
`;
|
||||
await fs.writeFile(path.join(tmpDir, "TASKS.md"), content);
|
||||
|
||||
const tasks = await loadCompletedTaskKeywords(tmpDir);
|
||||
expect(tasks).toEqual([]);
|
||||
});
|
||||
|
||||
it("handles malformed TASKS.md gracefully", async () => {
|
||||
const content = `This is not a valid TASKS.md file
|
||||
Just some random text
|
||||
No headers or structure at all`;
|
||||
await fs.writeFile(path.join(tmpDir, "TASKS.md"), content);
|
||||
|
||||
const tasks = await loadCompletedTaskKeywords(tmpDir);
|
||||
expect(tasks).toEqual([]);
|
||||
});
|
||||
|
||||
// --- Cache behavior ---
|
||||
|
||||
it("returns cached data within TTL", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "TASKS.md"), SAMPLE_TASKS_MD);
|
||||
|
||||
const first = await loadCompletedTaskKeywords(tmpDir);
|
||||
expect(first).toHaveLength(3);
|
||||
|
||||
// Modify the file — should still return cached result
|
||||
await fs.writeFile(path.join(tmpDir, "TASKS.md"), "# Active Tasks\n\n# Completed\n");
|
||||
|
||||
const second = await loadCompletedTaskKeywords(tmpDir);
|
||||
expect(second).toHaveLength(3); // Still cached
|
||||
expect(second).toBe(first); // Same reference (from cache)
|
||||
});
|
||||
|
||||
it("refreshes after cache is cleared", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "TASKS.md"), SAMPLE_TASKS_MD);
|
||||
|
||||
const first = await loadCompletedTaskKeywords(tmpDir);
|
||||
expect(first).toHaveLength(3);
|
||||
|
||||
// Modify file and clear cache
|
||||
await fs.writeFile(path.join(tmpDir, "TASKS.md"), "# Active Tasks\n\n# Completed\n");
|
||||
clearTaskFilterCache();
|
||||
|
||||
const second = await loadCompletedTaskKeywords(tmpDir);
|
||||
expect(second).toHaveLength(0); // Re-read from disk
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Integration: end-to-end filtering
|
||||
// ============================================================================
|
||||
|
||||
describe("end-to-end recall filtering", () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "task-filter-e2e-"));
|
||||
clearTaskFilterCache();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
clearTaskFilterCache();
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("filters memories related to completed tasks while keeping unrelated ones", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "TASKS.md"), SAMPLE_TASKS_MD);
|
||||
|
||||
const completedTasks = await loadCompletedTaskKeywords(tmpDir);
|
||||
|
||||
const memories = [
|
||||
{ text: "TASK-002 flights have been booked — Scoot TR453 confirmed", keep: false },
|
||||
{ text: "LinkedIn dashboard stats fixed — industry numbers corrected", keep: false },
|
||||
{ text: "Browser port collision resolved — openclaw on 18807", keep: false },
|
||||
{ text: "Tarun's birthday is August 23, 1974", keep: true },
|
||||
{ text: "Singapore has great food markets", keep: true },
|
||||
{ text: "LinkedIn connection from Jane Doe accepted", keep: true },
|
||||
{ text: "Memory-neo4j sleep cycle runs at 3am", keep: true },
|
||||
];
|
||||
|
||||
for (const m of memories) {
|
||||
const isRelated = isRelatedToCompletedTask(m.text, completedTasks);
|
||||
expect(isRelated).toBe(!m.keep);
|
||||
}
|
||||
});
|
||||
});
|
||||
324
extensions/memory-neo4j/task-filter.ts
Normal file
324
extensions/memory-neo4j/task-filter.ts
Normal file
@@ -0,0 +1,324 @@
|
||||
/**
|
||||
* Task-aware recall filter (Layer 1).
|
||||
*
|
||||
* Filters out auto-recalled memories that relate to completed tasks,
|
||||
* preventing stale task-state memories from being injected into agent context.
|
||||
*
|
||||
* Design principles:
|
||||
* - Conservative: false positives (filtering useful memories) are worse than
|
||||
* false negatives (letting some stale ones through).
|
||||
* - Fast: runs on every message, targeting < 5ms with caching.
|
||||
* - Graceful: missing/malformed TASKS.md is silently ignored.
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { parseTaskLedger, type ParsedTask } from "./task-ledger.js";
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
|
||||
/** Extracted keyword info for a single completed task. */
|
||||
export type CompletedTaskInfo = {
|
||||
/** Task ID (e.g. "TASK-002") */
|
||||
id: string;
|
||||
/** Significant keywords extracted from the task title + details + currentStep */
|
||||
keywords: string[];
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Constants
|
||||
// ============================================================================
|
||||
|
||||
/** Cache TTL in milliseconds — avoids re-reading TASKS.md on every message. */
|
||||
const CACHE_TTL_MS = 60_000;
|
||||
|
||||
/** Minimum keyword length to be considered "significant". */
|
||||
const MIN_KEYWORD_LENGTH = 4;
|
||||
|
||||
/**
|
||||
* Common English stop words that should be excluded from keyword matching.
|
||||
* Only words ≥ MIN_KEYWORD_LENGTH are included (shorter ones are filtered by length).
|
||||
*/
|
||||
const STOP_WORDS = new Set([
|
||||
"about",
|
||||
"also",
|
||||
"been",
|
||||
"before",
|
||||
"being",
|
||||
"between",
|
||||
"both",
|
||||
"came",
|
||||
"come",
|
||||
"could",
|
||||
"does",
|
||||
"done",
|
||||
"each",
|
||||
"even",
|
||||
"find",
|
||||
"first",
|
||||
"found",
|
||||
"from",
|
||||
"going",
|
||||
"good",
|
||||
"great",
|
||||
"have",
|
||||
"here",
|
||||
"high",
|
||||
"however",
|
||||
"into",
|
||||
"just",
|
||||
"keep",
|
||||
"know",
|
||||
"last",
|
||||
"like",
|
||||
"long",
|
||||
"look",
|
||||
"made",
|
||||
"make",
|
||||
"many",
|
||||
"more",
|
||||
"most",
|
||||
"much",
|
||||
"must",
|
||||
"need",
|
||||
"next",
|
||||
"only",
|
||||
"other",
|
||||
"over",
|
||||
"part",
|
||||
"said",
|
||||
"same",
|
||||
"should",
|
||||
"show",
|
||||
"since",
|
||||
"some",
|
||||
"still",
|
||||
"such",
|
||||
"take",
|
||||
"than",
|
||||
"that",
|
||||
"their",
|
||||
"them",
|
||||
"then",
|
||||
"there",
|
||||
"these",
|
||||
"they",
|
||||
"this",
|
||||
"through",
|
||||
"time",
|
||||
"under",
|
||||
"used",
|
||||
"using",
|
||||
"very",
|
||||
"want",
|
||||
"were",
|
||||
"what",
|
||||
"when",
|
||||
"where",
|
||||
"which",
|
||||
"while",
|
||||
"will",
|
||||
"with",
|
||||
"without",
|
||||
"work",
|
||||
"would",
|
||||
"your",
|
||||
// Task-related generic words that shouldn't be matching keywords:
|
||||
"task",
|
||||
"tasks",
|
||||
"active",
|
||||
"completed",
|
||||
"details",
|
||||
"status",
|
||||
"started",
|
||||
"updated",
|
||||
"blocked",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Minimum number of keyword matches required to consider a memory related
|
||||
* to a completed task (when matching by keywords rather than task ID).
|
||||
*/
|
||||
const MIN_KEYWORD_MATCHES = 2;
|
||||
|
||||
// ============================================================================
|
||||
// Cache
|
||||
// ============================================================================
|
||||
|
||||
type CacheEntry = {
|
||||
tasks: CompletedTaskInfo[];
|
||||
timestamp: number;
|
||||
};
|
||||
|
||||
const cache = new Map<string, CacheEntry>();
|
||||
|
||||
/** Clear the cache (exposed for testing). */
|
||||
export function clearTaskFilterCache(): void {
|
||||
cache.clear();
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Keyword Extraction
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Extract significant keywords from a text string.
|
||||
*
|
||||
* Filters out short words, stop words, and common noise to produce
|
||||
* a set of meaningful terms that can identify task-specific content.
|
||||
*/
|
||||
export function extractSignificantKeywords(text: string): string[] {
|
||||
if (!text) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const words = text
|
||||
.toLowerCase()
|
||||
// Replace non-alphanumeric chars (except hyphens in task IDs) with spaces
|
||||
.replace(/[^a-z0-9\-]/g, " ")
|
||||
.split(/\s+/)
|
||||
.filter((w) => w.length >= MIN_KEYWORD_LENGTH && !STOP_WORDS.has(w));
|
||||
|
||||
// Deduplicate while preserving order
|
||||
return [...new Set(words)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a {@link CompletedTaskInfo} from a parsed completed task.
|
||||
*
|
||||
* Extracts keywords from the task's title, details, and current step.
|
||||
*/
|
||||
export function buildCompletedTaskInfo(task: ParsedTask): CompletedTaskInfo {
|
||||
const parts: string[] = [task.title];
|
||||
if (task.details) {
|
||||
parts.push(task.details);
|
||||
}
|
||||
if (task.currentStep) {
|
||||
parts.push(task.currentStep);
|
||||
}
|
||||
|
||||
// Also extract from raw lines to capture fields the parser doesn't map
|
||||
// (e.g. "- **Completed:** 2026-02-16")
|
||||
for (const line of task.rawLines) {
|
||||
const trimmed = line.trim();
|
||||
// Skip the header line (already have title) and empty lines
|
||||
if (trimmed.startsWith("##") || trimmed === "") {
|
||||
continue;
|
||||
}
|
||||
// Include field values from bullet lines
|
||||
const fieldMatch = trimmed.match(/^-\s+\*\*.+?:\*\*\s*(.+)$/);
|
||||
if (fieldMatch) {
|
||||
parts.push(fieldMatch[1]);
|
||||
}
|
||||
}
|
||||
|
||||
const keywords = extractSignificantKeywords(parts.join(" "));
|
||||
|
||||
return {
|
||||
id: task.id,
|
||||
keywords,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Core API
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Load completed task info from TASKS.md in the given workspace directory.
|
||||
*
|
||||
* Results are cached per workspace dir with a 60-second TTL to avoid
|
||||
* re-reading and re-parsing on every message.
|
||||
*
|
||||
* @param workspaceDir - Path to the workspace directory containing TASKS.md
|
||||
* @returns Array of completed task info (empty if TASKS.md is missing or has no completed tasks)
|
||||
*/
|
||||
export async function loadCompletedTaskKeywords(
|
||||
workspaceDir: string,
|
||||
): Promise<CompletedTaskInfo[]> {
|
||||
const now = Date.now();
|
||||
|
||||
// Check cache
|
||||
const cached = cache.get(workspaceDir);
|
||||
if (cached && now - cached.timestamp < CACHE_TTL_MS) {
|
||||
return cached.tasks;
|
||||
}
|
||||
|
||||
// Read and parse TASKS.md
|
||||
const tasksPath = path.join(workspaceDir, "TASKS.md");
|
||||
let content: string;
|
||||
try {
|
||||
content = await fs.readFile(tasksPath, "utf-8");
|
||||
} catch {
|
||||
// File doesn't exist or isn't readable — cache empty result
|
||||
cache.set(workspaceDir, { tasks: [], timestamp: now });
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!content.trim()) {
|
||||
cache.set(workspaceDir, { tasks: [], timestamp: now });
|
||||
return [];
|
||||
}
|
||||
|
||||
const ledger = parseTaskLedger(content);
|
||||
const tasks = ledger.completedTasks.map(buildCompletedTaskInfo);
|
||||
|
||||
// Cache the result
|
||||
cache.set(workspaceDir, { tasks, timestamp: now });
|
||||
|
||||
return tasks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a memory's text is related to a completed task.
|
||||
*
|
||||
* Uses two matching strategies:
|
||||
* 1. **Task ID match** — if the memory text contains a completed task's ID
|
||||
* (e.g. "TASK-002"), it's considered related.
|
||||
* 2. **Keyword match** — if the memory text matches {@link MIN_KEYWORD_MATCHES}
|
||||
* or more significant keywords from a completed task, it's considered related.
|
||||
*
|
||||
* The filter is intentionally conservative: a memory about "Flux 2" won't be
|
||||
* filtered just because a completed task mentioned "Flux", unless the memory
|
||||
* also matches additional task-specific keywords.
|
||||
*
|
||||
* @param memoryText - The text content of the recalled memory
|
||||
* @param completedTasks - Completed task info from {@link loadCompletedTaskKeywords}
|
||||
* @returns `true` if the memory appears related to a completed task
|
||||
*/
|
||||
export function isRelatedToCompletedTask(
|
||||
memoryText: string,
|
||||
completedTasks: CompletedTaskInfo[],
|
||||
): boolean {
|
||||
if (!memoryText || completedTasks.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const lowerText = memoryText.toLowerCase();
|
||||
|
||||
for (const task of completedTasks) {
|
||||
// Strategy 1: Direct task ID match (case-insensitive)
|
||||
if (lowerText.includes(task.id.toLowerCase())) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Strategy 2: Keyword overlap — require MIN_KEYWORD_MATCHES distinct keywords
|
||||
if (task.keywords.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let matchCount = 0;
|
||||
for (const keyword of task.keywords) {
|
||||
if (lowerText.includes(keyword)) {
|
||||
matchCount++;
|
||||
if (matchCount >= MIN_KEYWORD_MATCHES) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
466
extensions/memory-neo4j/task-ledger.test.ts
Normal file
466
extensions/memory-neo4j/task-ledger.test.ts
Normal file
@@ -0,0 +1,466 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
findStaleTasks,
|
||||
parseTaskDate,
|
||||
parseTaskLedger,
|
||||
reviewAndArchiveStaleTasks,
|
||||
serializeTask,
|
||||
serializeTaskLedger,
|
||||
} from "./task-ledger.js";
|
||||
|
||||
// ============================================================================
|
||||
// parseTaskDate
|
||||
// ============================================================================
|
||||
|
||||
describe("parseTaskDate", () => {
|
||||
it("parses YYYY-MM-DD HH:MM format", () => {
|
||||
const date = parseTaskDate("2026-02-14 09:15");
|
||||
expect(date).not.toBeNull();
|
||||
expect(date!.getFullYear()).toBe(2026);
|
||||
expect(date!.getMonth()).toBe(1); // February is month 1
|
||||
expect(date!.getDate()).toBe(14);
|
||||
});
|
||||
|
||||
it("parses YYYY-MM-DD HH:MM with timezone abbreviation", () => {
|
||||
const date = parseTaskDate("2026-02-14 09:15 MYT");
|
||||
expect(date).not.toBeNull();
|
||||
expect(date!.getFullYear()).toBe(2026);
|
||||
});
|
||||
|
||||
it("parses ISO format", () => {
|
||||
const date = parseTaskDate("2026-02-14T09:15:00");
|
||||
expect(date).not.toBeNull();
|
||||
expect(date!.getFullYear()).toBe(2026);
|
||||
});
|
||||
|
||||
it("returns null for empty string", () => {
|
||||
expect(parseTaskDate("")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null for invalid date", () => {
|
||||
expect(parseTaskDate("not-a-date")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// parseTaskLedger
|
||||
// ============================================================================
|
||||
|
||||
describe("parseTaskLedger", () => {
|
||||
it("parses a simple task ledger", () => {
|
||||
const content = [
|
||||
"# Active Tasks",
|
||||
"",
|
||||
"## TASK-001: Restaurant Booking",
|
||||
"- **Status:** in_progress",
|
||||
"- **Started:** 2026-02-14 09:15",
|
||||
"- **Updated:** 2026-02-14 09:30",
|
||||
"- **Details:** Graze, 4 pax, 19:30",
|
||||
"- **Current Step:** Form filled, awaiting confirmation",
|
||||
"",
|
||||
"# Completed",
|
||||
"<!-- Move done tasks here with completion date -->",
|
||||
].join("\n");
|
||||
|
||||
const ledger = parseTaskLedger(content);
|
||||
expect(ledger.activeTasks).toHaveLength(1);
|
||||
expect(ledger.completedTasks).toHaveLength(0);
|
||||
|
||||
const task = ledger.activeTasks[0];
|
||||
expect(task.id).toBe("TASK-001");
|
||||
expect(task.title).toBe("Restaurant Booking");
|
||||
expect(task.status).toBe("in_progress");
|
||||
expect(task.started).toBe("2026-02-14 09:15");
|
||||
expect(task.updated).toBe("2026-02-14 09:30");
|
||||
expect(task.details).toBe("Graze, 4 pax, 19:30");
|
||||
expect(task.currentStep).toBe("Form filled, awaiting confirmation");
|
||||
expect(task.isCompleted).toBe(false);
|
||||
});
|
||||
|
||||
it("parses multiple active tasks", () => {
|
||||
const content = [
|
||||
"# Active Tasks",
|
||||
"",
|
||||
"## TASK-001: Task One",
|
||||
"- **Status:** in_progress",
|
||||
"- **Started:** 2026-02-14 09:00",
|
||||
"",
|
||||
"## TASK-002: Task Two",
|
||||
"- **Status:** awaiting_input",
|
||||
"- **Started:** 2026-02-14 10:00",
|
||||
"",
|
||||
"# Completed",
|
||||
].join("\n");
|
||||
|
||||
const ledger = parseTaskLedger(content);
|
||||
expect(ledger.activeTasks).toHaveLength(2);
|
||||
expect(ledger.activeTasks[0].id).toBe("TASK-001");
|
||||
expect(ledger.activeTasks[1].id).toBe("TASK-002");
|
||||
});
|
||||
|
||||
it("parses completed tasks", () => {
|
||||
const content = [
|
||||
"# Active Tasks",
|
||||
"",
|
||||
"# Completed",
|
||||
"",
|
||||
"## ~~TASK-001: Old Task~~",
|
||||
"- **Status:** done",
|
||||
"- **Started:** 2026-02-13 09:00",
|
||||
"- **Updated:** 2026-02-13 15:00",
|
||||
].join("\n");
|
||||
|
||||
const ledger = parseTaskLedger(content);
|
||||
expect(ledger.activeTasks).toHaveLength(0);
|
||||
expect(ledger.completedTasks).toHaveLength(1);
|
||||
expect(ledger.completedTasks[0].id).toBe("TASK-001");
|
||||
expect(ledger.completedTasks[0].isCompleted).toBe(true);
|
||||
});
|
||||
|
||||
it("parses blocked tasks", () => {
|
||||
const content = [
|
||||
"# Active Tasks",
|
||||
"",
|
||||
"## TASK-001: Blocked Task",
|
||||
"- **Status:** blocked",
|
||||
"- **Started:** 2026-02-14 09:00",
|
||||
"- **Blocked On:** Waiting for API key",
|
||||
"",
|
||||
"# Completed",
|
||||
].join("\n");
|
||||
|
||||
const ledger = parseTaskLedger(content);
|
||||
expect(ledger.activeTasks).toHaveLength(1);
|
||||
expect(ledger.activeTasks[0].blockedOn).toBe("Waiting for API key");
|
||||
});
|
||||
|
||||
it("handles empty task ledger", () => {
|
||||
const content = [
|
||||
"# Active Tasks",
|
||||
"",
|
||||
"# Completed",
|
||||
"<!-- Move done tasks here with completion date -->",
|
||||
].join("\n");
|
||||
|
||||
const ledger = parseTaskLedger(content);
|
||||
expect(ledger.activeTasks).toHaveLength(0);
|
||||
expect(ledger.completedTasks).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("handles Last Updated field variant", () => {
|
||||
const content = [
|
||||
"# Active Tasks",
|
||||
"",
|
||||
"## TASK-001: Some Task",
|
||||
"- **Status:** in_progress",
|
||||
"- **Last Updated:** 2026-02-14 10:00",
|
||||
"",
|
||||
"# Completed",
|
||||
].join("\n");
|
||||
|
||||
const ledger = parseTaskLedger(content);
|
||||
expect(ledger.activeTasks[0].updated).toBe("2026-02-14 10:00");
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// findStaleTasks
|
||||
// ============================================================================
|
||||
|
||||
describe("findStaleTasks", () => {
|
||||
const now = new Date("2026-02-15T10:00:00");
|
||||
const twentyFourHoursMs = 24 * 60 * 60 * 1000;
|
||||
|
||||
it("identifies tasks older than 24h as stale", () => {
|
||||
const tasks = [
|
||||
{
|
||||
id: "TASK-001",
|
||||
title: "Old Task",
|
||||
status: "in_progress" as const,
|
||||
updated: "2026-02-14 08:00",
|
||||
rawLines: [],
|
||||
isCompleted: false,
|
||||
},
|
||||
];
|
||||
|
||||
const stale = findStaleTasks(tasks, now, twentyFourHoursMs);
|
||||
expect(stale).toHaveLength(1);
|
||||
expect(stale[0].id).toBe("TASK-001");
|
||||
});
|
||||
|
||||
it("does not mark recent tasks as stale", () => {
|
||||
const tasks = [
|
||||
{
|
||||
id: "TASK-001",
|
||||
title: "Recent Task",
|
||||
status: "in_progress" as const,
|
||||
updated: "2026-02-15 09:00",
|
||||
rawLines: [],
|
||||
isCompleted: false,
|
||||
},
|
||||
];
|
||||
|
||||
const stale = findStaleTasks(tasks, now, twentyFourHoursMs);
|
||||
expect(stale).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("skips done tasks", () => {
|
||||
const tasks = [
|
||||
{
|
||||
id: "TASK-001",
|
||||
title: "Done Task",
|
||||
status: "done" as const,
|
||||
updated: "2026-02-13 08:00",
|
||||
rawLines: [],
|
||||
isCompleted: false,
|
||||
},
|
||||
];
|
||||
|
||||
const stale = findStaleTasks(tasks, now, twentyFourHoursMs);
|
||||
expect(stale).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("skips already-stale tasks", () => {
|
||||
const tasks = [
|
||||
{
|
||||
id: "TASK-001",
|
||||
title: "Already Stale",
|
||||
status: "stale" as const,
|
||||
updated: "2026-02-13 08:00",
|
||||
rawLines: [],
|
||||
isCompleted: false,
|
||||
},
|
||||
];
|
||||
|
||||
const stale = findStaleTasks(tasks, now, twentyFourHoursMs);
|
||||
expect(stale).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("uses started date when updated is missing", () => {
|
||||
const tasks = [
|
||||
{
|
||||
id: "TASK-001",
|
||||
title: "No Update Date",
|
||||
status: "in_progress" as const,
|
||||
started: "2026-02-14 08:00",
|
||||
rawLines: [],
|
||||
isCompleted: false,
|
||||
},
|
||||
];
|
||||
|
||||
const stale = findStaleTasks(tasks, now, twentyFourHoursMs);
|
||||
expect(stale).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("marks tasks with no dates as stale", () => {
|
||||
const tasks = [
|
||||
{
|
||||
id: "TASK-001",
|
||||
title: "No Dates",
|
||||
status: "in_progress" as const,
|
||||
rawLines: [],
|
||||
isCompleted: false,
|
||||
},
|
||||
];
|
||||
|
||||
const stale = findStaleTasks(tasks, now, twentyFourHoursMs);
|
||||
expect(stale).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// serializeTask / serializeTaskLedger
|
||||
// ============================================================================
|
||||
|
||||
describe("serializeTask", () => {
|
||||
it("serializes an active task", () => {
|
||||
const task = {
|
||||
id: "TASK-001",
|
||||
title: "My Task",
|
||||
status: "in_progress" as const,
|
||||
started: "2026-02-14 09:00",
|
||||
updated: "2026-02-14 10:00",
|
||||
details: "Some details",
|
||||
currentStep: "Step 1",
|
||||
rawLines: [],
|
||||
isCompleted: false,
|
||||
};
|
||||
|
||||
const lines = serializeTask(task);
|
||||
expect(lines[0]).toBe("## TASK-001: My Task");
|
||||
expect(lines).toContain("- **Status:** in_progress");
|
||||
expect(lines).toContain("- **Started:** 2026-02-14 09:00");
|
||||
expect(lines).toContain("- **Updated:** 2026-02-14 10:00");
|
||||
expect(lines).toContain("- **Details:** Some details");
|
||||
expect(lines).toContain("- **Current Step:** Step 1");
|
||||
});
|
||||
|
||||
it("serializes a completed task with strikethrough", () => {
|
||||
const task = {
|
||||
id: "TASK-001",
|
||||
title: "Done Task",
|
||||
status: "done" as const,
|
||||
started: "2026-02-14 09:00",
|
||||
rawLines: [],
|
||||
isCompleted: true,
|
||||
};
|
||||
|
||||
const lines = serializeTask(task);
|
||||
expect(lines[0]).toBe("## ~~TASK-001: Done Task~~");
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeTaskLedger", () => {
|
||||
it("round-trips a task ledger", () => {
|
||||
const ledger = {
|
||||
activeTasks: [
|
||||
{
|
||||
id: "TASK-001",
|
||||
title: "Active Task",
|
||||
status: "in_progress" as const,
|
||||
started: "2026-02-14 09:00",
|
||||
updated: "2026-02-14 10:00",
|
||||
details: "Details here",
|
||||
rawLines: [],
|
||||
isCompleted: false,
|
||||
},
|
||||
],
|
||||
completedTasks: [
|
||||
{
|
||||
id: "TASK-000",
|
||||
title: "Old Task",
|
||||
status: "done" as const,
|
||||
started: "2026-02-13 09:00",
|
||||
rawLines: [],
|
||||
isCompleted: true,
|
||||
},
|
||||
],
|
||||
preamble: [],
|
||||
sectionSeparator: [],
|
||||
postamble: [],
|
||||
};
|
||||
|
||||
const serialized = serializeTaskLedger(ledger);
|
||||
expect(serialized).toContain("# Active Tasks");
|
||||
expect(serialized).toContain("## TASK-001: Active Task");
|
||||
expect(serialized).toContain("# Completed");
|
||||
expect(serialized).toContain("## ~~TASK-000: Old Task~~");
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// reviewAndArchiveStaleTasks (integration with filesystem)
|
||||
// ============================================================================
|
||||
|
||||
describe("reviewAndArchiveStaleTasks", () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "task-ledger-test-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("returns null when TASKS.md does not exist", async () => {
|
||||
const result = await reviewAndArchiveStaleTasks(tmpDir);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null for empty TASKS.md", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "TASKS.md"), "", "utf-8");
|
||||
const result = await reviewAndArchiveStaleTasks(tmpDir);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("archives stale tasks", async () => {
|
||||
const content = [
|
||||
"# Active Tasks",
|
||||
"",
|
||||
"## TASK-001: Stale Task",
|
||||
"- **Status:** in_progress",
|
||||
"- **Started:** 2026-02-13 08:00",
|
||||
"- **Updated:** 2026-02-13 09:00",
|
||||
"",
|
||||
"## TASK-002: Fresh Task",
|
||||
"- **Status:** in_progress",
|
||||
"- **Started:** 2026-02-14 09:00",
|
||||
"- **Updated:** 2026-02-14 23:00",
|
||||
"",
|
||||
"# Completed",
|
||||
"<!-- Move done tasks here with completion date -->",
|
||||
].join("\n");
|
||||
|
||||
await fs.writeFile(path.join(tmpDir, "TASKS.md"), content, "utf-8");
|
||||
|
||||
// "now" is Feb 15, 10:00 — TASK-001 updated Feb 13, 09:00 (>24h ago), TASK-002 updated Feb 14, 23:00 (<24h ago)
|
||||
const now = new Date("2026-02-15T10:00:00");
|
||||
const result = await reviewAndArchiveStaleTasks(tmpDir, undefined, now);
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.staleCount).toBe(1);
|
||||
expect(result!.archivedCount).toBe(1);
|
||||
expect(result!.archivedIds).toEqual(["TASK-001"]);
|
||||
|
||||
// Verify the file was updated
|
||||
const updated = await fs.readFile(path.join(tmpDir, "TASKS.md"), "utf-8");
|
||||
expect(updated).toContain("## TASK-002: Fresh Task");
|
||||
expect(updated).toContain("## ~~TASK-001: Stale Task~~");
|
||||
|
||||
// Re-parse to verify structure
|
||||
const ledger = parseTaskLedger(updated);
|
||||
expect(ledger.activeTasks).toHaveLength(1);
|
||||
expect(ledger.activeTasks[0].id).toBe("TASK-002");
|
||||
expect(ledger.completedTasks).toHaveLength(1);
|
||||
expect(ledger.completedTasks[0].id).toBe("TASK-001");
|
||||
expect(ledger.completedTasks[0].status).toBe("stale");
|
||||
});
|
||||
|
||||
it("does nothing when no tasks are stale", async () => {
|
||||
const content = [
|
||||
"# Active Tasks",
|
||||
"",
|
||||
"## TASK-001: Fresh Task",
|
||||
"- **Status:** in_progress",
|
||||
"- **Started:** 2026-02-15 09:00",
|
||||
"- **Updated:** 2026-02-15 09:30",
|
||||
"",
|
||||
"# Completed",
|
||||
].join("\n");
|
||||
|
||||
await fs.writeFile(path.join(tmpDir, "TASKS.md"), content, "utf-8");
|
||||
|
||||
const now = new Date("2026-02-15T10:00:00");
|
||||
const result = await reviewAndArchiveStaleTasks(tmpDir, undefined, now);
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.staleCount).toBe(0);
|
||||
expect(result!.archivedCount).toBe(0);
|
||||
});
|
||||
|
||||
it("supports custom maxAgeMs", async () => {
|
||||
const content = [
|
||||
"# Active Tasks",
|
||||
"",
|
||||
"## TASK-001: Semi-Fresh Task",
|
||||
"- **Status:** in_progress",
|
||||
"- **Started:** 2026-02-15 06:00",
|
||||
"- **Updated:** 2026-02-15 06:00",
|
||||
"",
|
||||
"# Completed",
|
||||
].join("\n");
|
||||
|
||||
await fs.writeFile(path.join(tmpDir, "TASKS.md"), content, "utf-8");
|
||||
|
||||
const now = new Date("2026-02-15T10:00:00");
|
||||
const oneHourMs = 60 * 60 * 1000;
|
||||
|
||||
// With 1-hour threshold, task is stale (4 hours old)
|
||||
const result = await reviewAndArchiveStaleTasks(tmpDir, oneHourMs, now);
|
||||
expect(result!.archivedCount).toBe(1);
|
||||
});
|
||||
});
|
||||
424
extensions/memory-neo4j/task-ledger.ts
Normal file
424
extensions/memory-neo4j/task-ledger.ts
Normal file
@@ -0,0 +1,424 @@
|
||||
/**
|
||||
* Task Ledger (TASKS.md) maintenance utilities.
|
||||
*
|
||||
* Parses and updates the structured task ledger file used by agents
|
||||
* to track active work across compaction events. The sleep cycle uses
|
||||
* these utilities to archive stale tasks (>24h with no activity).
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
|
||||
export type TaskStatus = "in_progress" | "awaiting_input" | "blocked" | "done" | "stale" | string;
|
||||
|
||||
export type ParsedTask = {
|
||||
/** Task ID (e.g. "TASK-001") */
|
||||
id: string;
|
||||
/** Short title */
|
||||
title: string;
|
||||
/** Current status */
|
||||
status: TaskStatus;
|
||||
/** When the task was started (ISO-ish string) */
|
||||
started?: string;
|
||||
/** When the task was last updated (ISO-ish string) */
|
||||
updated?: string;
|
||||
/** Task details/description */
|
||||
details?: string;
|
||||
/** Current step being worked on */
|
||||
currentStep?: string;
|
||||
/** What's blocking progress */
|
||||
blockedOn?: string;
|
||||
/** Raw markdown lines for this task section (for round-tripping) */
|
||||
rawLines: string[];
|
||||
/** Whether this task is in the completed section */
|
||||
isCompleted: boolean;
|
||||
};
|
||||
|
||||
export type TaskLedger = {
|
||||
activeTasks: ParsedTask[];
|
||||
completedTasks: ParsedTask[];
|
||||
/** Lines before the first task section (header, etc.) */
|
||||
preamble: string[];
|
||||
/** Lines between active and completed sections */
|
||||
sectionSeparator: string[];
|
||||
/** Lines after the completed section */
|
||||
postamble: string[];
|
||||
};
|
||||
|
||||
export type StaleTaskResult = {
|
||||
/** Number of tasks found that are stale */
|
||||
staleCount: number;
|
||||
/** Number of tasks archived (moved to completed) */
|
||||
archivedCount: number;
|
||||
/** Task IDs that were archived */
|
||||
archivedIds: string[];
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Parsing
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Parse a TASKS.md file content into structured task data.
|
||||
*/
|
||||
export function parseTaskLedger(content: string): TaskLedger {
|
||||
const lines = content.split("\n");
|
||||
const activeTasks: ParsedTask[] = [];
|
||||
const completedTasks: ParsedTask[] = [];
|
||||
const preamble: string[] = [];
|
||||
const sectionSeparator: string[] = [];
|
||||
const postamble: string[] = [];
|
||||
|
||||
let currentSection: "preamble" | "active" | "completed" | "postamble" = "preamble";
|
||||
let currentTask: ParsedTask | null = null;
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
|
||||
// Detect section headers
|
||||
if (/^#\s+Active\s+Tasks/i.test(trimmed)) {
|
||||
if (currentTask) {
|
||||
pushTask(currentTask, activeTasks, completedTasks);
|
||||
currentTask = null;
|
||||
}
|
||||
currentSection = "active";
|
||||
preamble.push(line);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (/^#\s+Completed/i.test(trimmed)) {
|
||||
if (currentTask) {
|
||||
pushTask(currentTask, activeTasks, completedTasks);
|
||||
currentTask = null;
|
||||
}
|
||||
currentSection = "completed";
|
||||
sectionSeparator.push(line);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Detect task headers (## TASK-NNN: Title or ## ~~TASK-NNN: Title~~)
|
||||
const taskMatch = trimmed.match(/^##\s+(?:~~)?(TASK-\d+):\s*(.+?)(?:~~)?$/);
|
||||
if (taskMatch) {
|
||||
if (currentTask) {
|
||||
pushTask(currentTask, activeTasks, completedTasks);
|
||||
}
|
||||
const isStrikethrough = trimmed.includes("~~");
|
||||
currentTask = {
|
||||
id: taskMatch[1],
|
||||
title: taskMatch[2].replace(/~~/g, "").trim(),
|
||||
status: isStrikethrough ? "done" : "in_progress",
|
||||
rawLines: [line],
|
||||
isCompleted: currentSection === "completed" || isStrikethrough,
|
||||
};
|
||||
continue;
|
||||
}
|
||||
|
||||
// Parse task fields (- **Field:** Value)
|
||||
if (currentTask) {
|
||||
const fieldMatch = trimmed.match(/^-\s+\*\*(.+?):\*\*\s*(.*)$/);
|
||||
if (fieldMatch) {
|
||||
const fieldName = fieldMatch[1].toLowerCase();
|
||||
const value = fieldMatch[2].trim();
|
||||
switch (fieldName) {
|
||||
case "status":
|
||||
currentTask.status = value;
|
||||
break;
|
||||
case "started":
|
||||
currentTask.started = value;
|
||||
break;
|
||||
case "updated":
|
||||
case "last updated":
|
||||
currentTask.updated = value;
|
||||
break;
|
||||
case "details":
|
||||
currentTask.details = value;
|
||||
break;
|
||||
case "current step":
|
||||
currentTask.currentStep = value;
|
||||
break;
|
||||
case "blocked on":
|
||||
currentTask.blockedOn = value;
|
||||
break;
|
||||
}
|
||||
currentTask.rawLines.push(line);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Non-field lines within a task
|
||||
if (trimmed !== "" && !trimmed.startsWith("#")) {
|
||||
currentTask.rawLines.push(line);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Empty line within a task — include it
|
||||
if (trimmed === "") {
|
||||
currentTask.rawLines.push(line);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
currentSection === "completed" &&
|
||||
trimmed.startsWith("#") &&
|
||||
!/^#\s+Completed/i.test(trimmed)
|
||||
) {
|
||||
currentSection = "postamble";
|
||||
}
|
||||
|
||||
// Lines not part of a task
|
||||
switch (currentSection) {
|
||||
case "preamble":
|
||||
case "active":
|
||||
preamble.push(line);
|
||||
break;
|
||||
case "completed":
|
||||
sectionSeparator.push(line);
|
||||
break;
|
||||
case "postamble":
|
||||
postamble.push(line);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Push the last task
|
||||
if (currentTask) {
|
||||
pushTask(currentTask, activeTasks, completedTasks);
|
||||
}
|
||||
|
||||
return { activeTasks, completedTasks, preamble, sectionSeparator, postamble };
|
||||
}
|
||||
|
||||
function pushTask(task: ParsedTask, active: ParsedTask[], completed: ParsedTask[]) {
|
||||
if (task.isCompleted || task.status === "done") {
|
||||
completed.push(task);
|
||||
} else {
|
||||
active.push(task);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Staleness Detection
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Parse a date string from the task ledger.
|
||||
* Accepts formats like "2026-02-14 09:15", "2026-02-14 09:15 MYT",
|
||||
* "2026-02-14T09:15:00", etc.
|
||||
*/
|
||||
export function parseTaskDate(dateStr: string): Date | null {
|
||||
if (!dateStr) {
|
||||
return null;
|
||||
}
|
||||
const cleaned = dateStr
|
||||
.trim()
|
||||
// Remove timezone abbreviations like MYT, UTC, PST
|
||||
.replace(/\s+[A-Z]{2,5}$/, "")
|
||||
// Normalize space-separated date time to ISO
|
||||
.replace(/^(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2})/, "$1T$2");
|
||||
|
||||
const date = new Date(cleaned);
|
||||
if (Number.isNaN(date.getTime())) {
|
||||
return null;
|
||||
}
|
||||
return date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find tasks that are stale (no update in more than `maxAgeMs` milliseconds).
|
||||
* Default: 24 hours.
|
||||
*/
|
||||
export function findStaleTasks(
|
||||
tasks: ParsedTask[],
|
||||
now: Date = new Date(),
|
||||
maxAgeMs: number = 24 * 60 * 60 * 1000,
|
||||
): ParsedTask[] {
|
||||
return tasks.filter((task) => {
|
||||
// Only check active tasks (not already done/stale)
|
||||
if (task.status === "done" || task.status === "stale") {
|
||||
return false;
|
||||
}
|
||||
|
||||
const lastUpdate = task.updated || task.started;
|
||||
if (!lastUpdate) {
|
||||
// No date info — consider stale if we can't determine age
|
||||
return true;
|
||||
}
|
||||
|
||||
const date = parseTaskDate(lastUpdate);
|
||||
if (!date) {
|
||||
return false; // Can't parse date — don't mark as stale
|
||||
}
|
||||
|
||||
const ageMs = now.getTime() - date.getTime();
|
||||
return ageMs > maxAgeMs;
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Task Ledger Serialization
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Serialize a task back to markdown lines.
|
||||
* If the task has rawLines from parsing, regenerate only the header and status
|
||||
* (which may have changed) while preserving other raw content.
|
||||
* For new/modified tasks without rawLines, generate from parsed fields.
|
||||
*/
|
||||
export function serializeTask(task: ParsedTask): string[] {
|
||||
const titlePrefix = task.isCompleted
|
||||
? `## ~~${task.id}: ${task.title}~~`
|
||||
: `## ${task.id}: ${task.title}`;
|
||||
|
||||
// If we have rawLines and the task was only modified (status/updated changed
|
||||
// by archival), rebuild from rawLines with updated field values.
|
||||
if (task.rawLines.length > 0) {
|
||||
const lines: string[] = [titlePrefix];
|
||||
for (const line of task.rawLines.slice(1)) {
|
||||
const trimmed = line.trim();
|
||||
// Replace Status field with current value
|
||||
if (/^-\s+\*\*Status:\*\*/.test(trimmed)) {
|
||||
lines.push(`- **Status:** ${task.status}`);
|
||||
} else if (/^-\s+\*\*(?:Updated|Last Updated):\*\*/.test(trimmed)) {
|
||||
lines.push(`- **Updated:** ${task.updated ?? ""}`);
|
||||
} else {
|
||||
lines.push(line);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
// Fallback: generate from parsed fields (for newly created tasks)
|
||||
const lines: string[] = [titlePrefix];
|
||||
lines.push(`- **Status:** ${task.status}`);
|
||||
if (task.started) {
|
||||
lines.push(`- **Started:** ${task.started}`);
|
||||
}
|
||||
if (task.updated) {
|
||||
lines.push(`- **Updated:** ${task.updated}`);
|
||||
}
|
||||
if (task.details) {
|
||||
lines.push(`- **Details:** ${task.details}`);
|
||||
}
|
||||
if (task.currentStep) {
|
||||
lines.push(`- **Current Step:** ${task.currentStep}`);
|
||||
}
|
||||
if (task.blockedOn) {
|
||||
lines.push(`- **Blocked On:** ${task.blockedOn}`);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize the full task ledger back to markdown.
|
||||
* Preserves preamble, section separators, and postamble from the original parse.
|
||||
*/
|
||||
export function serializeTaskLedger(ledger: TaskLedger): string {
|
||||
const lines: string[] = [];
|
||||
|
||||
// Use original preamble if available, otherwise generate header
|
||||
if (ledger.preamble.length > 0) {
|
||||
lines.push(...ledger.preamble);
|
||||
} else {
|
||||
lines.push("# Active Tasks");
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
// Active tasks
|
||||
for (const task of ledger.activeTasks) {
|
||||
lines.push(...serializeTask(task));
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
// Use original section separator if available, otherwise generate
|
||||
if (ledger.sectionSeparator.length > 0) {
|
||||
lines.push(...ledger.sectionSeparator);
|
||||
} else {
|
||||
lines.push("# Completed");
|
||||
lines.push("<!-- Move done tasks here with completion date -->");
|
||||
}
|
||||
lines.push("");
|
||||
|
||||
// Completed tasks
|
||||
for (const task of ledger.completedTasks) {
|
||||
lines.push(...serializeTask(task));
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
// Preserve postamble
|
||||
if (ledger.postamble.length > 0) {
|
||||
lines.push(...ledger.postamble);
|
||||
}
|
||||
|
||||
return lines.join("\n").trimEnd() + "\n";
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Sleep Cycle Integration
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Review TASKS.md for stale tasks and archive them.
|
||||
* This is called during the sleep cycle.
|
||||
*
|
||||
* @param workspaceDir - Path to the workspace directory
|
||||
* @param maxAgeMs - Maximum age before a task is considered stale (default: 24h)
|
||||
* @param now - Current time (for testing)
|
||||
* @returns Result of the stale task review, or null if TASKS.md doesn't exist
|
||||
*/
|
||||
export async function reviewAndArchiveStaleTasks(
|
||||
workspaceDir: string,
|
||||
maxAgeMs: number = 24 * 60 * 60 * 1000,
|
||||
now: Date = new Date(),
|
||||
): Promise<StaleTaskResult | null> {
|
||||
const tasksPath = path.join(workspaceDir, "TASKS.md");
|
||||
|
||||
let content: string;
|
||||
try {
|
||||
content = await fs.readFile(tasksPath, "utf-8");
|
||||
} catch {
|
||||
// TASKS.md doesn't exist — nothing to do
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!content.trim()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const ledger = parseTaskLedger(content);
|
||||
const staleTasks = findStaleTasks(ledger.activeTasks, now, maxAgeMs);
|
||||
|
||||
if (staleTasks.length === 0) {
|
||||
return { staleCount: 0, archivedCount: 0, archivedIds: [] };
|
||||
}
|
||||
|
||||
const archivedIds: string[] = [];
|
||||
const nowStr = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, "0")}-${String(now.getDate()).padStart(2, "0")} ${String(now.getHours()).padStart(2, "0")}:${String(now.getMinutes()).padStart(2, "0")}`;
|
||||
|
||||
for (const task of staleTasks) {
|
||||
task.status = "stale";
|
||||
task.updated = nowStr;
|
||||
task.isCompleted = true;
|
||||
|
||||
// Move from active to completed
|
||||
const idx = ledger.activeTasks.indexOf(task);
|
||||
if (idx !== -1) {
|
||||
ledger.activeTasks.splice(idx, 1);
|
||||
}
|
||||
ledger.completedTasks.push(task);
|
||||
archivedIds.push(task.id);
|
||||
}
|
||||
|
||||
// Write back
|
||||
const updated = serializeTaskLedger(ledger);
|
||||
await fs.writeFile(tasksPath, updated, "utf-8");
|
||||
|
||||
return {
|
||||
staleCount: staleTasks.length,
|
||||
archivedCount: archivedIds.length,
|
||||
archivedIds,
|
||||
};
|
||||
}
|
||||
606
extensions/memory-neo4j/task-metadata.test.ts
Normal file
606
extensions/memory-neo4j/task-metadata.test.ts
Normal file
@@ -0,0 +1,606 @@
|
||||
/**
|
||||
* Tests for Layer 3: Task Metadata on memories.
|
||||
*
|
||||
* Tests that memories can be linked to specific tasks via taskId,
|
||||
* enabling precise task-aware filtering at recall and cleanup time.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import type { StoreMemoryInput } from "./schema.js";
|
||||
import { Neo4jMemoryClient } from "./neo4j-client.js";
|
||||
import { fuseWithConfidenceRRF } from "./search.js";
|
||||
import { parseTaskLedger } from "./task-ledger.js";
|
||||
|
||||
// ============================================================================
|
||||
// Test Helpers
|
||||
// ============================================================================
|
||||
|
||||
function createMockSession() {
|
||||
return {
|
||||
run: vi.fn().mockResolvedValue({ records: [] }),
|
||||
close: vi.fn().mockResolvedValue(undefined),
|
||||
executeWrite: vi.fn(
|
||||
async (work: (tx: { run: ReturnType<typeof vi.fn> }) => Promise<unknown>) => {
|
||||
const mockTx = { run: vi.fn().mockResolvedValue({ records: [] }) };
|
||||
return work(mockTx);
|
||||
},
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function createMockDriver() {
|
||||
return {
|
||||
session: vi.fn().mockReturnValue(createMockSession()),
|
||||
close: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
function createMockLogger() {
|
||||
return {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function createMockRecord(data: Record<string, unknown>) {
|
||||
return {
|
||||
get: (key: string) => data[key],
|
||||
keys: Object.keys(data),
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Neo4jMemoryClient: storeMemory with taskId
|
||||
// ============================================================================
|
||||
|
||||
describe("Task Metadata: storeMemory", () => {
|
||||
let client: Neo4jMemoryClient;
|
||||
let mockDriver: ReturnType<typeof createMockDriver>;
|
||||
let mockSession: ReturnType<typeof createMockSession>;
|
||||
|
||||
beforeEach(() => {
|
||||
const mockLogger = createMockLogger();
|
||||
mockDriver = createMockDriver();
|
||||
mockSession = createMockSession();
|
||||
mockDriver.session.mockReturnValue(mockSession);
|
||||
|
||||
client = new Neo4jMemoryClient("bolt://localhost:7687", "neo4j", "password", 1024, mockLogger);
|
||||
(client as any).driver = mockDriver;
|
||||
(client as any).indexesReady = true;
|
||||
});
|
||||
|
||||
it("should store memory with taskId when provided", async () => {
|
||||
mockSession.run.mockResolvedValue({
|
||||
records: [createMockRecord({ id: "mem-1" })],
|
||||
});
|
||||
|
||||
const input: StoreMemoryInput = {
|
||||
id: "mem-1",
|
||||
text: "test memory with task",
|
||||
embedding: [0.1, 0.2],
|
||||
importance: 0.7,
|
||||
category: "fact",
|
||||
source: "user",
|
||||
extractionStatus: "pending",
|
||||
agentId: "agent-1",
|
||||
taskId: "TASK-001",
|
||||
};
|
||||
|
||||
await client.storeMemory(input);
|
||||
|
||||
const runCall = mockSession.run.mock.calls[0];
|
||||
const cypher = runCall[0] as string;
|
||||
const params = runCall[1] as Record<string, unknown>;
|
||||
|
||||
// Cypher should include taskId clause
|
||||
expect(cypher).toContain("taskId");
|
||||
// Params should include the taskId value
|
||||
expect(params.taskId).toBe("TASK-001");
|
||||
});
|
||||
|
||||
it("should store memory without taskId when not provided", async () => {
|
||||
mockSession.run.mockResolvedValue({
|
||||
records: [createMockRecord({ id: "mem-2" })],
|
||||
});
|
||||
|
||||
const input: StoreMemoryInput = {
|
||||
id: "mem-2",
|
||||
text: "test memory without task",
|
||||
embedding: [0.1, 0.2],
|
||||
importance: 0.7,
|
||||
category: "fact",
|
||||
source: "user",
|
||||
extractionStatus: "pending",
|
||||
agentId: "agent-1",
|
||||
};
|
||||
|
||||
await client.storeMemory(input);
|
||||
|
||||
const runCall = mockSession.run.mock.calls[0];
|
||||
const cypher = runCall[0] as string;
|
||||
|
||||
// Cypher should NOT include taskId clause when not provided
|
||||
// The dynamic clause is only added when taskId is present
|
||||
expect(cypher).not.toContain(", taskId: $taskId");
|
||||
});
|
||||
|
||||
it("backward compatibility: existing memories without taskId still work", async () => {
|
||||
// Storing without taskId should work exactly as before
|
||||
mockSession.run.mockResolvedValue({
|
||||
records: [createMockRecord({ id: "mem-3" })],
|
||||
});
|
||||
|
||||
const input: StoreMemoryInput = {
|
||||
id: "mem-3",
|
||||
text: "legacy memory",
|
||||
embedding: [0.1],
|
||||
importance: 0.5,
|
||||
category: "other",
|
||||
source: "auto-capture",
|
||||
extractionStatus: "skipped",
|
||||
agentId: "default",
|
||||
};
|
||||
|
||||
const id = await client.storeMemory(input);
|
||||
expect(id).toBe("mem-3");
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Neo4jMemoryClient: findMemoriesByTaskId
|
||||
// ============================================================================
|
||||
|
||||
describe("Task Metadata: findMemoriesByTaskId", () => {
|
||||
let client: Neo4jMemoryClient;
|
||||
let mockDriver: ReturnType<typeof createMockDriver>;
|
||||
let mockSession: ReturnType<typeof createMockSession>;
|
||||
|
||||
beforeEach(() => {
|
||||
const mockLogger = createMockLogger();
|
||||
mockDriver = createMockDriver();
|
||||
mockSession = createMockSession();
|
||||
mockDriver.session.mockReturnValue(mockSession);
|
||||
|
||||
client = new Neo4jMemoryClient("bolt://localhost:7687", "neo4j", "password", 1024, mockLogger);
|
||||
(client as any).driver = mockDriver;
|
||||
(client as any).indexesReady = true;
|
||||
});
|
||||
|
||||
it("should find memories by taskId", async () => {
|
||||
mockSession.run.mockResolvedValue({
|
||||
records: [
|
||||
createMockRecord({
|
||||
id: "mem-1",
|
||||
text: "task-related memory",
|
||||
category: "fact",
|
||||
importance: 0.8,
|
||||
}),
|
||||
createMockRecord({
|
||||
id: "mem-2",
|
||||
text: "another task memory",
|
||||
category: "other",
|
||||
importance: 0.6,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const results = await client.findMemoriesByTaskId("TASK-001");
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results[0].id).toBe("mem-1");
|
||||
expect(results[1].id).toBe("mem-2");
|
||||
|
||||
const runCall = mockSession.run.mock.calls[0];
|
||||
const cypher = runCall[0] as string;
|
||||
const params = runCall[1] as Record<string, unknown>;
|
||||
|
||||
expect(cypher).toContain("m.taskId = $taskId");
|
||||
expect(params.taskId).toBe("TASK-001");
|
||||
});
|
||||
|
||||
it("should filter by agentId when provided", async () => {
|
||||
mockSession.run.mockResolvedValue({ records: [] });
|
||||
|
||||
await client.findMemoriesByTaskId("TASK-001", "agent-1");
|
||||
|
||||
const runCall = mockSession.run.mock.calls[0];
|
||||
const cypher = runCall[0] as string;
|
||||
const params = runCall[1] as Record<string, unknown>;
|
||||
|
||||
expect(cypher).toContain("m.agentId = $agentId");
|
||||
expect(params.agentId).toBe("agent-1");
|
||||
});
|
||||
|
||||
it("should return empty array when no memories match", async () => {
|
||||
mockSession.run.mockResolvedValue({ records: [] });
|
||||
|
||||
const results = await client.findMemoriesByTaskId("TASK-999");
|
||||
expect(results).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Neo4jMemoryClient: clearTaskIdFromMemories
|
||||
// ============================================================================
|
||||
|
||||
describe("Task Metadata: clearTaskIdFromMemories", () => {
|
||||
let client: Neo4jMemoryClient;
|
||||
let mockDriver: ReturnType<typeof createMockDriver>;
|
||||
let mockSession: ReturnType<typeof createMockSession>;
|
||||
|
||||
beforeEach(() => {
|
||||
const mockLogger = createMockLogger();
|
||||
mockDriver = createMockDriver();
|
||||
mockSession = createMockSession();
|
||||
mockDriver.session.mockReturnValue(mockSession);
|
||||
|
||||
client = new Neo4jMemoryClient("bolt://localhost:7687", "neo4j", "password", 1024, mockLogger);
|
||||
(client as any).driver = mockDriver;
|
||||
(client as any).indexesReady = true;
|
||||
});
|
||||
|
||||
it("should clear taskId from all matching memories", async () => {
|
||||
mockSession.run.mockResolvedValue({
|
||||
records: [createMockRecord({ cleared: 3 })],
|
||||
});
|
||||
|
||||
const count = await client.clearTaskIdFromMemories("TASK-001");
|
||||
|
||||
expect(count).toBe(3);
|
||||
|
||||
const runCall = mockSession.run.mock.calls[0];
|
||||
const cypher = runCall[0] as string;
|
||||
const params = runCall[1] as Record<string, unknown>;
|
||||
|
||||
expect(cypher).toContain("m.taskId = $taskId");
|
||||
expect(cypher).toContain("SET m.taskId = null");
|
||||
expect(params.taskId).toBe("TASK-001");
|
||||
});
|
||||
|
||||
it("should filter by agentId when provided", async () => {
|
||||
mockSession.run.mockResolvedValue({
|
||||
records: [createMockRecord({ cleared: 1 })],
|
||||
});
|
||||
|
||||
await client.clearTaskIdFromMemories("TASK-001", "agent-1");
|
||||
|
||||
const runCall = mockSession.run.mock.calls[0];
|
||||
const cypher = runCall[0] as string;
|
||||
const params = runCall[1] as Record<string, unknown>;
|
||||
|
||||
expect(cypher).toContain("m.agentId = $agentId");
|
||||
expect(params.agentId).toBe("agent-1");
|
||||
});
|
||||
|
||||
it("should return 0 when no memories match", async () => {
|
||||
mockSession.run.mockResolvedValue({
|
||||
records: [createMockRecord({ cleared: 0 })],
|
||||
});
|
||||
|
||||
const count = await client.clearTaskIdFromMemories("TASK-999");
|
||||
expect(count).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Hybrid search results include taskId
|
||||
// ============================================================================
|
||||
|
||||
describe("Task Metadata: hybrid search includes taskId", () => {
|
||||
it("should carry taskId through RRF fusion", () => {
|
||||
const vectorResults = [
|
||||
{
|
||||
id: "mem-1",
|
||||
text: "memory with task",
|
||||
category: "fact",
|
||||
importance: 0.8,
|
||||
createdAt: "2026-01-01",
|
||||
score: 0.9,
|
||||
taskId: "TASK-001",
|
||||
},
|
||||
{
|
||||
id: "mem-2",
|
||||
text: "memory without task",
|
||||
category: "other",
|
||||
importance: 0.5,
|
||||
createdAt: "2026-01-02",
|
||||
score: 0.8,
|
||||
},
|
||||
];
|
||||
|
||||
const bm25Results = [
|
||||
{
|
||||
id: "mem-1",
|
||||
text: "memory with task",
|
||||
category: "fact",
|
||||
importance: 0.8,
|
||||
createdAt: "2026-01-01",
|
||||
score: 0.7,
|
||||
taskId: "TASK-001",
|
||||
},
|
||||
];
|
||||
|
||||
const graphResults: typeof vectorResults = [];
|
||||
|
||||
const fused = fuseWithConfidenceRRF(
|
||||
[vectorResults, bm25Results, graphResults],
|
||||
60,
|
||||
[1.0, 1.0, 1.0],
|
||||
);
|
||||
|
||||
// mem-1 should have taskId preserved
|
||||
const mem1 = fused.find((r) => r.id === "mem-1");
|
||||
expect(mem1).toBeDefined();
|
||||
expect(mem1!.taskId).toBe("TASK-001");
|
||||
|
||||
// mem-2 should have undefined taskId
|
||||
const mem2 = fused.find((r) => r.id === "mem-2");
|
||||
expect(mem2).toBeDefined();
|
||||
expect(mem2!.taskId).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should include taskId in fused results when present in any signal", () => {
|
||||
// taskId present only in BM25 signal
|
||||
const vectorResults = [
|
||||
{
|
||||
id: "mem-1",
|
||||
text: "test",
|
||||
category: "fact",
|
||||
importance: 0.8,
|
||||
createdAt: "2026-01-01",
|
||||
score: 0.9,
|
||||
// no taskId
|
||||
},
|
||||
];
|
||||
|
||||
const bm25Results = [
|
||||
{
|
||||
id: "mem-1",
|
||||
text: "test",
|
||||
category: "fact",
|
||||
importance: 0.8,
|
||||
createdAt: "2026-01-01",
|
||||
score: 0.7,
|
||||
taskId: "TASK-002",
|
||||
},
|
||||
];
|
||||
|
||||
const fused = fuseWithConfidenceRRF([vectorResults, bm25Results, []], 60, [1.0, 1.0, 1.0]);
|
||||
|
||||
// The first signal (vector) is used for metadata — taskId would be undefined
|
||||
// because candidateMetadata takes the first occurrence
|
||||
const mem1 = fused.find((r) => r.id === "mem-1");
|
||||
expect(mem1).toBeDefined();
|
||||
// The first signal to contribute metadata wins
|
||||
// vector came first and has no taskId
|
||||
expect(mem1!.taskId).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Auto-tagging: parseTaskLedger for active task detection
|
||||
// ============================================================================
|
||||
|
||||
describe("Task Metadata: auto-tagging via parseTaskLedger", () => {
|
||||
it("should detect single active task for auto-tagging", () => {
|
||||
const content = `# Active Tasks
|
||||
|
||||
## TASK-005: Fix login bug
|
||||
- **Status:** in_progress
|
||||
- **Started:** 2026-02-16
|
||||
|
||||
# Completed
|
||||
## TASK-004: Fix browser port collision
|
||||
- **Completed:** 2026-02-16
|
||||
`;
|
||||
|
||||
const ledger = parseTaskLedger(content);
|
||||
expect(ledger.activeTasks).toHaveLength(1);
|
||||
expect(ledger.activeTasks[0].id).toBe("TASK-005");
|
||||
});
|
||||
|
||||
it("should not auto-tag when multiple active tasks exist", () => {
|
||||
const content = `# Active Tasks
|
||||
|
||||
## TASK-005: Fix login bug
|
||||
- **Status:** in_progress
|
||||
|
||||
## TASK-006: Update docs
|
||||
- **Status:** in_progress
|
||||
|
||||
# Completed
|
||||
`;
|
||||
|
||||
const ledger = parseTaskLedger(content);
|
||||
// Multiple active tasks — should NOT auto-tag
|
||||
expect(ledger.activeTasks.length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it("should not auto-tag when no active tasks exist", () => {
|
||||
const content = `# Active Tasks
|
||||
|
||||
_No active tasks_
|
||||
|
||||
# Completed
|
||||
## TASK-004: Fix browser port collision
|
||||
- **Completed:** 2026-02-16
|
||||
`;
|
||||
|
||||
const ledger = parseTaskLedger(content);
|
||||
expect(ledger.activeTasks).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("should extract completed task IDs for recall filtering", () => {
|
||||
const content = `# Active Tasks
|
||||
|
||||
## TASK-007: New feature
|
||||
- **Status:** in_progress
|
||||
|
||||
# Completed
|
||||
## TASK-002: Book flights
|
||||
- **Completed:** 2026-02-16
|
||||
|
||||
## TASK-003: Fix dashboard
|
||||
- **Completed:** 2026-02-16
|
||||
`;
|
||||
|
||||
const ledger = parseTaskLedger(content);
|
||||
const completedTaskIds = new Set(ledger.completedTasks.map((t) => t.id));
|
||||
expect(completedTaskIds.has("TASK-002")).toBe(true);
|
||||
expect(completedTaskIds.has("TASK-003")).toBe(true);
|
||||
expect(completedTaskIds.has("TASK-007")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Recall filter: taskId-based completed task filtering
|
||||
// ============================================================================
|
||||
|
||||
describe("Task Metadata: recall filter", () => {
|
||||
it("should filter out memories linked to completed tasks", () => {
|
||||
const completedTaskIds = new Set(["TASK-002", "TASK-003"]);
|
||||
|
||||
const results = [
|
||||
{
|
||||
id: "1",
|
||||
text: "active task memory",
|
||||
taskId: "TASK-007",
|
||||
score: 0.9,
|
||||
category: "fact",
|
||||
importance: 0.8,
|
||||
createdAt: "2026-01-01",
|
||||
},
|
||||
{
|
||||
id: "2",
|
||||
text: "completed task memory",
|
||||
taskId: "TASK-002",
|
||||
score: 0.85,
|
||||
category: "fact",
|
||||
importance: 0.7,
|
||||
createdAt: "2026-01-01",
|
||||
},
|
||||
{
|
||||
id: "3",
|
||||
text: "no task memory",
|
||||
score: 0.8,
|
||||
category: "other",
|
||||
importance: 0.5,
|
||||
createdAt: "2026-01-01",
|
||||
},
|
||||
{
|
||||
id: "4",
|
||||
text: "another completed",
|
||||
taskId: "TASK-003",
|
||||
score: 0.75,
|
||||
category: "fact",
|
||||
importance: 0.6,
|
||||
createdAt: "2026-01-01",
|
||||
},
|
||||
];
|
||||
|
||||
const filtered = results.filter((r) => !r.taskId || !completedTaskIds.has(r.taskId));
|
||||
|
||||
expect(filtered).toHaveLength(2);
|
||||
expect(filtered[0].id).toBe("1"); // active task — kept
|
||||
expect(filtered[1].id).toBe("3"); // no task — kept
|
||||
});
|
||||
|
||||
it("should keep all memories when no completed task IDs", () => {
|
||||
const completedTaskIds = new Set<string>();
|
||||
|
||||
const results = [
|
||||
{ id: "1", text: "memory A", taskId: "TASK-001", score: 0.9 },
|
||||
{ id: "2", text: "memory B", score: 0.8 },
|
||||
];
|
||||
|
||||
const filtered = results.filter((r) => !r.taskId || !completedTaskIds.has(r.taskId));
|
||||
|
||||
expect(filtered).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("should keep memories without taskId regardless of filter", () => {
|
||||
const completedTaskIds = new Set(["TASK-001", "TASK-002"]);
|
||||
|
||||
const results = [
|
||||
{ id: "1", text: "old memory without task", score: 0.9 },
|
||||
{ id: "2", text: "another old one", taskId: undefined, score: 0.8 },
|
||||
];
|
||||
|
||||
const filtered = results.filter((r) => !r.taskId || !completedTaskIds.has(r.taskId));
|
||||
|
||||
expect(filtered).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Vector/BM25 search results include taskId
|
||||
// ============================================================================
|
||||
|
||||
describe("Task Metadata: search signal taskId", () => {
|
||||
let client: Neo4jMemoryClient;
|
||||
let mockDriver: ReturnType<typeof createMockDriver>;
|
||||
let mockSession: ReturnType<typeof createMockSession>;
|
||||
|
||||
beforeEach(() => {
|
||||
const mockLogger = createMockLogger();
|
||||
mockDriver = createMockDriver();
|
||||
mockSession = createMockSession();
|
||||
mockDriver.session.mockReturnValue(mockSession);
|
||||
|
||||
client = new Neo4jMemoryClient("bolt://localhost:7687", "neo4j", "password", 1024, mockLogger);
|
||||
(client as any).driver = mockDriver;
|
||||
(client as any).indexesReady = true;
|
||||
});
|
||||
|
||||
it("vector search should include taskId in results", async () => {
|
||||
mockSession.run.mockResolvedValue({
|
||||
records: [
|
||||
createMockRecord({
|
||||
id: "mem-1",
|
||||
text: "test",
|
||||
category: "fact",
|
||||
importance: 0.8,
|
||||
createdAt: "2026-01-01",
|
||||
taskId: "TASK-001",
|
||||
similarity: 0.95,
|
||||
}),
|
||||
createMockRecord({
|
||||
id: "mem-2",
|
||||
text: "test2",
|
||||
category: "other",
|
||||
importance: 0.5,
|
||||
createdAt: "2026-01-02",
|
||||
taskId: null, // Legacy memory without taskId
|
||||
similarity: 0.85,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const results = await client.vectorSearch([0.1, 0.2], 10, 0.1);
|
||||
|
||||
expect(results[0].taskId).toBe("TASK-001");
|
||||
expect(results[1].taskId).toBeUndefined(); // null → undefined
|
||||
});
|
||||
|
||||
it("BM25 search should include taskId in results", async () => {
|
||||
mockSession.run.mockResolvedValue({
|
||||
records: [
|
||||
createMockRecord({
|
||||
id: "mem-1",
|
||||
text: "test query",
|
||||
category: "fact",
|
||||
importance: 0.8,
|
||||
createdAt: "2026-01-01",
|
||||
taskId: "TASK-002",
|
||||
bm25Score: 5.0,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const results = await client.bm25Search("test query", 10);
|
||||
|
||||
expect(results[0].taskId).toBe("TASK-002");
|
||||
});
|
||||
});
|
||||
19
extensions/memory-neo4j/tsconfig.json
Normal file
19
extensions/memory-neo4j/tsconfig.json
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2023",
|
||||
"lib": ["ES2023"],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"outDir": "./dist",
|
||||
"rootDir": "."
|
||||
},
|
||||
"include": ["*.ts"],
|
||||
"exclude": ["node_modules", "dist", "*.test.ts"]
|
||||
}
|
||||
@@ -3,7 +3,7 @@ import { escapeXml } from "../voice-mapping.js";
|
||||
export function generateNotifyTwiml(message: string, voice: string): string {
|
||||
return `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Response>
|
||||
<Say voice="${voice}">${escapeXml(message)}</Say>
|
||||
<Say voice="${escapeXml(voice)}">${escapeXml(message)}</Say>
|
||||
<Hangup/>
|
||||
</Response>`;
|
||||
}
|
||||
|
||||
@@ -244,6 +244,23 @@ export class PlivoProvider implements VoiceCallProvider {
|
||||
callStatus === "no-answer" ||
|
||||
callStatus === "failed"
|
||||
) {
|
||||
// Clean up internal maps on terminal state
|
||||
if (callUuid) {
|
||||
this.callUuidToWebhookUrl.delete(callUuid);
|
||||
// Also clean up the reverse mapping
|
||||
for (const [reqId, cUuid] of this.requestUuidToCallUuid) {
|
||||
if (cUuid === callUuid) {
|
||||
this.requestUuidToCallUuid.delete(reqId);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (callIdOverride) {
|
||||
this.callIdToWebhookUrl.delete(callIdOverride);
|
||||
this.pendingSpeakByCallId.delete(callIdOverride);
|
||||
this.pendingListenByCallId.delete(callIdOverride);
|
||||
}
|
||||
|
||||
return {
|
||||
...baseEvent,
|
||||
type: "call.ended",
|
||||
|
||||
@@ -174,6 +174,7 @@ class OpenAIRealtimeSTTSession implements RealtimeSTTSession {
|
||||
|
||||
setTimeout(() => {
|
||||
if (!this.connected) {
|
||||
this.ws?.close();
|
||||
reject(new Error("Realtime STT connection timeout"));
|
||||
}
|
||||
}, 10000);
|
||||
|
||||
@@ -130,11 +130,23 @@ export class TelnyxProvider implements VoiceCallProvider {
|
||||
callId = data.payload?.call_control_id || "";
|
||||
}
|
||||
|
||||
const direction =
|
||||
data.payload?.direction === "incoming"
|
||||
? ("inbound" as const)
|
||||
: data.payload?.direction === "outgoing"
|
||||
? ("outbound" as const)
|
||||
: undefined;
|
||||
const from = typeof data.payload?.from === "string" ? data.payload.from : undefined;
|
||||
const to = typeof data.payload?.to === "string" ? data.payload.to : undefined;
|
||||
|
||||
const baseEvent = {
|
||||
id: data.id || crypto.randomUUID(),
|
||||
callId,
|
||||
providerCallId: data.payload?.call_control_id,
|
||||
timestamp: Date.now(),
|
||||
...(direction && { direction }),
|
||||
...(from && { from }),
|
||||
...(to && { to }),
|
||||
};
|
||||
|
||||
switch (data.event_type) {
|
||||
|
||||
@@ -143,7 +143,7 @@ export class OpenAITTSProvider {
|
||||
}
|
||||
|
||||
/**
|
||||
* Resample 24kHz PCM to 8kHz using linear interpolation.
|
||||
* Resample 24kHz PCM to 8kHz by picking every 3rd sample.
|
||||
* Input/output: 16-bit signed little-endian mono.
|
||||
*/
|
||||
function resample24kTo8k(input: Buffer): Buffer {
|
||||
@@ -152,20 +152,11 @@ function resample24kTo8k(input: Buffer): Buffer {
|
||||
const output = Buffer.alloc(outputSamples * 2);
|
||||
|
||||
for (let i = 0; i < outputSamples; i++) {
|
||||
// Calculate position in input (3:1 ratio)
|
||||
const srcPos = i * 3;
|
||||
const srcIdx = srcPos * 2;
|
||||
// Pick every 3rd sample (3:1 ratio for 24kHz -> 8kHz)
|
||||
const srcByteOffset = i * 3 * 2;
|
||||
|
||||
if (srcIdx + 3 < input.length) {
|
||||
// Linear interpolation between samples
|
||||
const s0 = input.readInt16LE(srcIdx);
|
||||
const s1 = input.readInt16LE(srcIdx + 2);
|
||||
const frac = srcPos % 1 || 0;
|
||||
const sample = Math.round(s0 + frac * (s1 - s0));
|
||||
output.writeInt16LE(clamp16(sample), i * 2);
|
||||
} else {
|
||||
// Last sample
|
||||
output.writeInt16LE(input.readInt16LE(srcIdx), i * 2);
|
||||
if (srcByteOffset + 1 < input.length) {
|
||||
output.writeInt16LE(input.readInt16LE(srcByteOffset), i * 2);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -290,12 +290,14 @@ export class TwilioProvider implements VoiceCallProvider {
|
||||
case "no-answer":
|
||||
case "failed":
|
||||
this.streamAuthTokens.delete(callSid);
|
||||
this.callWebhookUrls.delete(callSid);
|
||||
if (callIdOverride) {
|
||||
this.deleteStoredTwiml(callIdOverride);
|
||||
}
|
||||
return { ...baseEvent, type: "call.ended", reason: callStatus };
|
||||
case "canceled":
|
||||
this.streamAuthTokens.delete(callSid);
|
||||
this.callWebhookUrls.delete(callSid);
|
||||
if (callIdOverride) {
|
||||
this.deleteStoredTwiml(callIdOverride);
|
||||
}
|
||||
@@ -544,7 +546,7 @@ export class TwilioProvider implements VoiceCallProvider {
|
||||
const pollyVoice = mapVoiceToPolly(input.voice);
|
||||
const twiml = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Response>
|
||||
<Say voice="${pollyVoice}" language="${input.locale || "en-US"}">${escapeXml(input.text)}</Say>
|
||||
<Say voice="${escapeXml(pollyVoice)}" language="${escapeXml(input.locale || "en-US")}">${escapeXml(input.text)}</Say>
|
||||
<Gather input="speech" speechTimeout="auto" action="${escapeXml(webhookUrl)}" method="POST">
|
||||
<Say>.</Say>
|
||||
</Gather>
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
*/
|
||||
export function escapeXml(text: string): string {
|
||||
return text
|
||||
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, "")
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
|
||||
@@ -209,7 +209,7 @@
|
||||
"engines": {
|
||||
"node": ">=22.12.0"
|
||||
},
|
||||
"packageManager": "pnpm@10.23.0",
|
||||
"packageManager": "pnpm@10.29.3",
|
||||
"pnpm": {
|
||||
"minimumReleaseAge": 2880,
|
||||
"overrides": {
|
||||
|
||||
749
pnpm-lock.yaml
generated
749
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
141
scripts/update-and-restart.sh
Executable file
141
scripts/update-and-restart.sh
Executable file
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# update-and-restart.sh — Rebase, build, link, push, and restart OpenClaw gateway
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
REPO_DIR="$HOME/openclaw"
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
log() { echo -e "${CYAN}[$(date '+%H:%M:%S')]${NC} $1"; }
|
||||
ok() { echo -e "${GREEN}✅ $1${NC}"; }
|
||||
warn() { echo -e "${YELLOW}⚠️ $1${NC}"; }
|
||||
fail() { echo -e "${RED}❌ $1${NC}"; exit 1; }
|
||||
|
||||
cd "$REPO_DIR" || fail "Cannot cd to $REPO_DIR"
|
||||
|
||||
# --- Check for uncommitted changes ---
|
||||
if ! git diff --quiet || ! git diff --cached --quiet; then
|
||||
echo ""
|
||||
git status --short
|
||||
echo ""
|
||||
fail "Working tree is dirty — commit or stash your changes first."
|
||||
fi
|
||||
|
||||
# --- Record pre-pull state ---
|
||||
OLD_SHORT=$(git rev-parse --short HEAD)
|
||||
log "Current commit: ${OLD_SHORT}"
|
||||
|
||||
# --- Rebase adabot on top of origin/main ---
|
||||
# This repo is the single source of truth for adabot.
|
||||
# We rebase our commits on top of upstream (origin/main), then force-push out.
|
||||
# Never pull/rebase from bitbucket/fork — those are downstream mirrors.
|
||||
log "Fetching origin..."
|
||||
UPSTREAM_BEFORE=$(git rev-parse origin/main)
|
||||
git fetch origin 2>&1 || fail "Could not fetch origin"
|
||||
UPSTREAM_AFTER=$(git rev-parse origin/main)
|
||||
log "Rebasing onto origin/main..."
|
||||
if git rebase origin/main 2>&1; then
|
||||
ok "Rebase onto origin/main complete"
|
||||
else
|
||||
warn "Rebase failed — aborting rebase and stopping."
|
||||
git rebase --abort 2>/dev/null || true
|
||||
fail "Rebase onto origin/main failed (conflicts?). Resolve manually."
|
||||
fi
|
||||
|
||||
BUILT_SHA=$(git rev-parse HEAD)
|
||||
BUILT_SHORT=$(git rev-parse --short HEAD)
|
||||
|
||||
if [ "$UPSTREAM_BEFORE" = "$UPSTREAM_AFTER" ]; then
|
||||
log "Already up to date with origin/main (${BUILT_SHORT})"
|
||||
else
|
||||
log "Rebased onto new upstream commits (${BUILT_SHORT})"
|
||||
echo ""
|
||||
UPSTREAM_COUNT=$(git rev-list --count "${UPSTREAM_BEFORE}..${UPSTREAM_AFTER}")
|
||||
if [ "$UPSTREAM_COUNT" -gt 20 ]; then
|
||||
log "(showing last 20 of ${UPSTREAM_COUNT} new upstream commits)"
|
||||
fi
|
||||
git --no-pager log --oneline -20 "${UPSTREAM_BEFORE}..${UPSTREAM_AFTER}"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# --- Force-push to remotes (this repo is source of truth) ---
|
||||
# Push early so mirrors stay in sync even if the build/lint/link steps fail.
|
||||
BRANCH=$(git branch --show-current)
|
||||
log "Force-pushing ${BRANCH} to bitbucket and fork..."
|
||||
BB_OK=0; FK_OK=0
|
||||
git push --force-with-lease bitbucket "HEAD:${BRANCH}" 2>&1 && BB_OK=1 || warn "Could not push to bitbucket"
|
||||
git push --force-with-lease fork "HEAD:${BRANCH}" 2>&1 && FK_OK=1 || warn "Could not push to fork"
|
||||
if [ "$BB_OK" -eq 1 ] && [ "$FK_OK" -eq 1 ]; then
|
||||
ok "Pushed to both remotes"
|
||||
elif [ "$BB_OK" -eq 1 ] || [ "$FK_OK" -eq 1 ]; then
|
||||
warn "Pushed to one remote only (see warnings above)"
|
||||
else
|
||||
fail "Could not push to either remote"
|
||||
fi
|
||||
|
||||
# --- pnpm install ---
|
||||
log "Installing dependencies..."
|
||||
if pnpm install --frozen-lockfile 2>&1; then
|
||||
ok "pnpm install complete"
|
||||
else
|
||||
warn "Frozen lockfile failed, trying regular install..."
|
||||
pnpm install 2>&1 || fail "pnpm install failed"
|
||||
ok "pnpm install complete"
|
||||
fi
|
||||
|
||||
# --- pnpm format (check only) ---
|
||||
log "Checking code formatting..."
|
||||
pnpm format:check 2>&1 || fail "Format check failed — run 'pnpm exec oxfmt --write <file>' to fix"
|
||||
ok "Format check passed"
|
||||
|
||||
# --- pnpm build ---
|
||||
log "Building TypeScript..."
|
||||
pnpm build 2>&1 || fail "pnpm build failed"
|
||||
ok "Build complete"
|
||||
|
||||
# --- pnpm lint ---
|
||||
log "Running linter..."
|
||||
pnpm lint 2>&1 || fail "Lint check failed — run 'pnpm exec oxlint <file>' to fix"
|
||||
ok "Lint check passed"
|
||||
|
||||
# --- pnpm link ---
|
||||
log "Linking globally..."
|
||||
pnpm link --global 2>&1 || fail "pnpm link --global failed"
|
||||
ok "Linked globally"
|
||||
|
||||
log "Built commit: ${BUILT_SHORT} (${BUILT_SHA})"
|
||||
|
||||
# --- Restart gateway ---
|
||||
log "Restarting gateway..."
|
||||
openclaw gateway restart 2>&1 || fail "Gateway restart failed"
|
||||
|
||||
# --- Wait for gateway to come back and verify ---
|
||||
log "Waiting for gateway health..."
|
||||
HEALTHY=0
|
||||
for i in {1..10}; do
|
||||
if openclaw gateway health 2>&1; then
|
||||
HEALTHY=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "$HEALTHY" -eq 1 ]; then
|
||||
ok "Gateway is healthy"
|
||||
else
|
||||
warn "Gateway health check failed after 10 attempts — may need manual inspection"
|
||||
fi
|
||||
|
||||
# --- Summary ---
|
||||
echo ""
|
||||
echo -e "${GREEN}════════════════════════════════════════${NC}"
|
||||
echo -e "${GREEN} OpenClaw updated and restarted!${NC}"
|
||||
echo -e "${GREEN} Commit: ${BUILT_SHORT}${NC}"
|
||||
if [ "$UPSTREAM_BEFORE" != "$UPSTREAM_AFTER" ]; then
|
||||
echo -e "${GREEN} Upstream: ${UPSTREAM_COUNT} new commit(s) from origin/main${NC}"
|
||||
fi
|
||||
echo -e "${GREEN}════════════════════════════════════════${NC}"
|
||||
@@ -2,6 +2,7 @@ import type { OpenClawConfig } from "../config/config.js";
|
||||
import type { AgentBootstrapHookContext } from "../hooks/internal-hooks.js";
|
||||
import type { WorkspaceBootstrapFile } from "./workspace.js";
|
||||
import { createInternalHookEvent, triggerInternalHook } from "../hooks/internal-hooks.js";
|
||||
import { getGlobalHookRunner } from "../plugins/hook-runner-global.js";
|
||||
import { resolveAgentIdFromSessionKey } from "../routing/session-key.js";
|
||||
|
||||
export async function applyBootstrapHookOverrides(params: {
|
||||
@@ -27,5 +28,30 @@ export async function applyBootstrapHookOverrides(params: {
|
||||
const event = createInternalHookEvent("agent", "bootstrap", sessionKey, context);
|
||||
await triggerInternalHook(event);
|
||||
const updated = (event.context as AgentBootstrapHookContext).bootstrapFiles;
|
||||
return Array.isArray(updated) ? updated : params.files;
|
||||
const internalResult = Array.isArray(updated) ? updated : params.files;
|
||||
|
||||
// After internal hooks, run plugin hooks
|
||||
const hookRunner = getGlobalHookRunner();
|
||||
if (hookRunner?.hasHooks("agent_bootstrap")) {
|
||||
const result = await hookRunner.runAgentBootstrap(
|
||||
{
|
||||
files: internalResult.map((f) => ({
|
||||
name: f.name,
|
||||
path: f.path,
|
||||
content: f.content,
|
||||
missing: f.missing,
|
||||
})),
|
||||
},
|
||||
{
|
||||
agentId,
|
||||
sessionKey: params.sessionKey,
|
||||
workspaceDir: params.workspaceDir,
|
||||
},
|
||||
);
|
||||
if (result?.files) {
|
||||
return result.files as WorkspaceBootstrapFile[];
|
||||
}
|
||||
}
|
||||
|
||||
return internalResult;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { AgentMessage } from "@mariozechner/pi-agent-core";
|
||||
import type { ExtensionContext } from "@mariozechner/pi-coding-agent";
|
||||
import { estimateTokens, generateSummary } from "@mariozechner/pi-coding-agent";
|
||||
import { completeSimple } from "@mariozechner/pi-ai";
|
||||
import { convertToLlm, estimateTokens, serializeConversation } from "@mariozechner/pi-coding-agent";
|
||||
import { DEFAULT_CONTEXT_TOKENS } from "./defaults.js";
|
||||
import { repairToolUseResultPairing, stripToolResultDetails } from "./session-transcript-repair.js";
|
||||
|
||||
@@ -13,6 +14,163 @@ const MERGE_SUMMARIES_INSTRUCTIONS =
|
||||
"Merge these partial summaries into a single cohesive summary. Preserve decisions," +
|
||||
" TODOs, open questions, and any constraints.";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Enhanced summarization prompts with "Immediate Context" section
|
||||
// ---------------------------------------------------------------------------
|
||||
// These replace the upstream pi-coding-agent prompts to add recency awareness.
|
||||
// The key addition is "## Immediate Context" which captures what was being
|
||||
// actively discussed/worked on in the most recent messages, solving the problem
|
||||
// of losing the "last thing we were doing" after compaction.
|
||||
|
||||
const ENHANCED_SUMMARIZATION_SYSTEM_PROMPT =
|
||||
"You are a context summarization assistant. Your task is to read a conversation " +
|
||||
"between a user and an AI assistant, then produce a structured summary following " +
|
||||
"the exact format specified.\n\n" +
|
||||
"Do NOT continue the conversation. Do NOT respond to any questions in the " +
|
||||
"conversation. ONLY output the structured summary.";
|
||||
|
||||
const ENHANCED_SUMMARIZATION_PROMPT =
|
||||
"The messages above are a conversation to summarize. Create a structured context " +
|
||||
"checkpoint summary that another LLM will use to continue the work.\n\n" +
|
||||
"Use this EXACT format:\n\n" +
|
||||
"## Immediate Context\n" +
|
||||
"[What was the user MOST RECENTLY asking about or working on? Describe the active " +
|
||||
"conversation topic from the last few exchanges in detail. Include any pending " +
|
||||
"questions, partial results, or the exact state of the task right before this " +
|
||||
"summary. This section should read like a handoff note: 'You were just working " +
|
||||
"on X, the user asked Y, and you were in the middle of Z.']\n\n" +
|
||||
"## Goal\n" +
|
||||
"[What is the user trying to accomplish? Can be multiple items if the session " +
|
||||
"covers different tasks.]\n\n" +
|
||||
"## Constraints & Preferences\n" +
|
||||
"- [Any constraints, preferences, or requirements mentioned by user]\n" +
|
||||
'- [Or "(none)" if none were mentioned]\n\n' +
|
||||
"## Progress\n" +
|
||||
"### Done\n" +
|
||||
"- [x] [Completed tasks/changes]\n\n" +
|
||||
"### In Progress\n" +
|
||||
"- [ ] [Current work]\n\n" +
|
||||
"### Blocked\n" +
|
||||
"- [Issues preventing progress, if any]\n\n" +
|
||||
"## Key Decisions\n" +
|
||||
"- **[Decision]**: [Brief rationale]\n\n" +
|
||||
"## Next Steps\n" +
|
||||
"1. [Ordered list of what should happen next]\n\n" +
|
||||
"## Critical Context\n" +
|
||||
"- [Any data, examples, or references needed to continue]\n" +
|
||||
'- [Or "(none)" if not applicable]\n\n' +
|
||||
"Keep each section concise. Preserve exact file paths, function names, and error messages.";
|
||||
|
||||
const ENHANCED_UPDATE_SUMMARIZATION_PROMPT =
|
||||
"The messages above are NEW conversation messages to incorporate into the existing " +
|
||||
"summary provided in <previous-summary> tags.\n\n" +
|
||||
"Update the existing structured summary with new information. RULES:\n" +
|
||||
"- REPLACE the Immediate Context section entirely with what the NEWEST messages " +
|
||||
"are about — this must always reflect the most recent conversation topic\n" +
|
||||
"- PRESERVE all existing information from the previous summary in other sections\n" +
|
||||
"- ADD new progress, decisions, and context from the new messages\n" +
|
||||
'- UPDATE the Progress section: move items from "In Progress" to "Done" when completed\n' +
|
||||
'- UPDATE "Next Steps" based on what was accomplished\n' +
|
||||
"- PRESERVE exact file paths, function names, and error messages\n" +
|
||||
"- If something is no longer relevant, you may remove it\n\n" +
|
||||
"Use this EXACT format:\n\n" +
|
||||
"## Immediate Context\n" +
|
||||
"[What is the conversation CURRENTLY about based on these newest messages? " +
|
||||
"Describe the active topic, any pending questions, and the exact state of work. " +
|
||||
"This REPLACES any previous immediate context — always reflect the latest exchanges.]\n\n" +
|
||||
"## Goal\n" +
|
||||
"[Preserve existing goals, add new ones if the task expanded]\n\n" +
|
||||
"## Constraints & Preferences\n" +
|
||||
"- [Preserve existing, add new ones discovered]\n\n" +
|
||||
"## Progress\n" +
|
||||
"### Done\n" +
|
||||
"- [x] [Include previously done items AND newly completed items]\n\n" +
|
||||
"### In Progress\n" +
|
||||
"- [ ] [Current work - update based on progress]\n\n" +
|
||||
"### Blocked\n" +
|
||||
"- [Current blockers - remove if resolved]\n\n" +
|
||||
"## Key Decisions\n" +
|
||||
"- **[Decision]**: [Brief rationale] (preserve all previous, add new)\n\n" +
|
||||
"## Next Steps\n" +
|
||||
"1. [Update based on current state]\n\n" +
|
||||
"## Critical Context\n" +
|
||||
"- [Preserve important context, add new if needed]\n\n" +
|
||||
"Keep each section concise. Preserve exact file paths, function names, and error messages.";
|
||||
|
||||
/**
|
||||
* Enhanced version of generateSummary that includes an "Immediate Context" section
|
||||
* in the compaction summary. This ensures that the most recent conversation topic
|
||||
* is prominently captured, solving the "can't remember what we were just doing"
|
||||
* problem after compaction.
|
||||
*/
|
||||
async function generateSummary(
|
||||
currentMessages: AgentMessage[],
|
||||
model: NonNullable<ExtensionContext["model"]>,
|
||||
reserveTokens: number,
|
||||
apiKey: string,
|
||||
signal: AbortSignal,
|
||||
customInstructions?: string,
|
||||
previousSummary?: string,
|
||||
): Promise<string> {
|
||||
const maxTokens = Math.floor(0.8 * reserveTokens);
|
||||
|
||||
// Use update prompt if we have a previous summary, otherwise initial prompt
|
||||
let basePrompt = previousSummary
|
||||
? ENHANCED_UPDATE_SUMMARIZATION_PROMPT
|
||||
: ENHANCED_SUMMARIZATION_PROMPT;
|
||||
if (customInstructions) {
|
||||
basePrompt = `${basePrompt}\n\nAdditional focus: ${customInstructions}`;
|
||||
}
|
||||
|
||||
// Serialize conversation to text so model doesn't try to continue it
|
||||
// Use type assertion since convertToLlm accepts AgentMessage[] at runtime
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const llmMessages = convertToLlm(currentMessages as any);
|
||||
const conversationText = serializeConversation(llmMessages);
|
||||
|
||||
// Build the prompt with conversation wrapped in tags
|
||||
let promptText = `<conversation>\n${conversationText}\n</conversation>\n\n`;
|
||||
if (previousSummary) {
|
||||
promptText += `<previous-summary>\n${previousSummary}\n</previous-summary>\n\n`;
|
||||
}
|
||||
promptText += basePrompt;
|
||||
|
||||
// Build user message for summarization request
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const summarizationMessages: any[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: promptText }],
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
];
|
||||
|
||||
const response = await completeSimple(
|
||||
model,
|
||||
{
|
||||
systemPrompt: ENHANCED_SUMMARIZATION_SYSTEM_PROMPT,
|
||||
messages: summarizationMessages,
|
||||
},
|
||||
{ maxTokens, signal, apiKey, reasoning: "high" },
|
||||
);
|
||||
|
||||
if (response.stopReason === "error") {
|
||||
throw new Error(
|
||||
`Summarization failed: ${
|
||||
(response as { errorMessage?: string }).errorMessage || "Unknown error"
|
||||
}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Extract text content from response
|
||||
const textContent = (response.content as Array<{ type: string; text?: string }>)
|
||||
.filter((c) => c.type === "text" && c.text)
|
||||
.map((c) => c.text!)
|
||||
.join("\n");
|
||||
|
||||
return textContent;
|
||||
}
|
||||
|
||||
export function estimateMessagesTokens(messages: AgentMessage[]): number {
|
||||
// SECURITY: toolResult.details can contain untrusted/verbose payloads; never include in LLM-facing compaction.
|
||||
const safe = stripToolResultDetails(messages);
|
||||
|
||||
@@ -8,17 +8,17 @@ import {
|
||||
} from "./context-window-guard.js";
|
||||
|
||||
describe("context-window-guard", () => {
|
||||
it("blocks below 16k (model metadata)", () => {
|
||||
it("blocks below 1024 (model metadata)", () => {
|
||||
const info = resolveContextWindowInfo({
|
||||
cfg: undefined,
|
||||
provider: "openrouter",
|
||||
modelId: "tiny",
|
||||
modelContextWindow: 8000,
|
||||
modelContextWindow: 512,
|
||||
defaultTokens: 200_000,
|
||||
});
|
||||
const guard = evaluateContextWindowGuard({ info });
|
||||
expect(guard.source).toBe("model");
|
||||
expect(guard.tokens).toBe(8000);
|
||||
expect(guard.tokens).toBe(512);
|
||||
expect(guard.shouldWarn).toBe(true);
|
||||
expect(guard.shouldBlock).toBe(true);
|
||||
});
|
||||
@@ -64,7 +64,7 @@ describe("context-window-guard", () => {
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 12_000,
|
||||
contextWindow: 512,
|
||||
maxTokens: 256,
|
||||
},
|
||||
],
|
||||
@@ -142,8 +142,45 @@ describe("context-window-guard", () => {
|
||||
expect(guard.shouldBlock).toBe(false);
|
||||
});
|
||||
|
||||
it("does not warn when context window is explicitly configured via modelsConfig", () => {
|
||||
const cfg = {
|
||||
models: {
|
||||
providers: {
|
||||
ollama: {
|
||||
baseUrl: "http://localhost:11434/v1",
|
||||
apiKey: "x",
|
||||
models: [
|
||||
{
|
||||
id: "qwen2.5:7b-2k",
|
||||
name: "Qwen 2.5 7B 2k",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 2_048,
|
||||
maxTokens: 2_048,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
|
||||
const info = resolveContextWindowInfo({
|
||||
cfg,
|
||||
provider: "ollama",
|
||||
modelId: "qwen2.5:7b-2k",
|
||||
modelContextWindow: undefined,
|
||||
defaultTokens: 200_000,
|
||||
});
|
||||
const guard = evaluateContextWindowGuard({ info });
|
||||
expect(info.source).toBe("modelsConfig");
|
||||
expect(guard.tokens).toBe(2_048);
|
||||
expect(guard.shouldWarn).toBe(false);
|
||||
expect(guard.shouldBlock).toBe(true);
|
||||
});
|
||||
|
||||
it("exports thresholds as expected", () => {
|
||||
expect(CONTEXT_WINDOW_HARD_MIN_TOKENS).toBe(16_000);
|
||||
expect(CONTEXT_WINDOW_HARD_MIN_TOKENS).toBe(1_024);
|
||||
expect(CONTEXT_WINDOW_WARN_BELOW_TOKENS).toBe(32_000);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
|
||||
export const CONTEXT_WINDOW_HARD_MIN_TOKENS = 16_000;
|
||||
export const CONTEXT_WINDOW_HARD_MIN_TOKENS = 1_024;
|
||||
export const CONTEXT_WINDOW_WARN_BELOW_TOKENS = 32_000;
|
||||
|
||||
export type ContextWindowSource = "model" | "modelsConfig" | "agentContextTokens" | "default";
|
||||
@@ -68,7 +68,7 @@ export function evaluateContextWindowGuard(params: {
|
||||
return {
|
||||
...params.info,
|
||||
tokens,
|
||||
shouldWarn: tokens > 0 && tokens < warnBelow,
|
||||
shouldWarn: tokens > 0 && tokens < warnBelow && params.info.source !== "modelsConfig",
|
||||
shouldBlock: tokens > 0 && tokens < hardMin,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -107,5 +107,39 @@ export function lookupContextTokens(modelId?: string): number | undefined {
|
||||
}
|
||||
// Best-effort: kick off loading, but don't block.
|
||||
void loadPromise;
|
||||
return MODEL_CACHE.get(modelId);
|
||||
|
||||
// Try exact match first (only if it contains a slash, i.e., already has provider prefix)
|
||||
if (modelId.includes("/")) {
|
||||
const exact = MODEL_CACHE.get(modelId);
|
||||
if (exact !== undefined) {
|
||||
return exact;
|
||||
}
|
||||
}
|
||||
|
||||
// For bare model names (no slash), try common provider prefixes first
|
||||
// to prefer our custom config over built-in defaults.
|
||||
// Priority order: prefer anthropic, then openai, then google
|
||||
const prefixes = ["anthropic", "openai", "google"];
|
||||
for (const prefix of prefixes) {
|
||||
const prefixedKey = `${prefix}/${modelId}`;
|
||||
const prefixed = MODEL_CACHE.get(prefixedKey);
|
||||
if (prefixed !== undefined) {
|
||||
return prefixed;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to exact match for bare model names (built-in defaults)
|
||||
const exact = MODEL_CACHE.get(modelId);
|
||||
if (exact !== undefined) {
|
||||
return exact;
|
||||
}
|
||||
|
||||
// Final fallback: any matching suffix
|
||||
for (const [key, value] of MODEL_CACHE) {
|
||||
if (key.endsWith(`/${modelId}`)) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -297,6 +297,13 @@ export function resolveMemorySearchConfig(
|
||||
cfg: OpenClawConfig,
|
||||
agentId: string,
|
||||
): ResolvedMemorySearchConfig | null {
|
||||
// Only one memory system can be active at a time.
|
||||
// When a memory plugin owns the slot, core memory-search is unconditionally disabled.
|
||||
const memoryPluginSlot = cfg.plugins?.slots?.memory;
|
||||
if (memoryPluginSlot && memoryPluginSlot !== "none") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const defaults = cfg.agents?.defaults?.memorySearch;
|
||||
const overrides = resolveAgentConfig(cfg, agentId)?.memorySearch;
|
||||
const resolved = mergeConfig(defaults, overrides, agentId);
|
||||
|
||||
@@ -199,15 +199,16 @@ export function buildBootstrapContextFiles(
|
||||
if (remainingTotalChars <= 0) {
|
||||
break;
|
||||
}
|
||||
const filePath = file.path ?? file.name;
|
||||
if (file.missing) {
|
||||
const missingText = `[MISSING] Expected at: ${file.path}`;
|
||||
const missingText = `[MISSING] Expected at: ${filePath}`;
|
||||
const cappedMissingText = clampToBudget(missingText, remainingTotalChars);
|
||||
if (!cappedMissingText) {
|
||||
break;
|
||||
}
|
||||
remainingTotalChars = Math.max(0, remainingTotalChars - cappedMissingText.length);
|
||||
result.push({
|
||||
path: file.path,
|
||||
path: filePath,
|
||||
content: cappedMissingText,
|
||||
});
|
||||
continue;
|
||||
@@ -231,7 +232,7 @@ export function buildBootstrapContextFiles(
|
||||
}
|
||||
remainingTotalChars = Math.max(0, remainingTotalChars - contentWithinBudget.length);
|
||||
result.push({
|
||||
path: file.path,
|
||||
path: filePath,
|
||||
content: contentWithinBudget,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { DEFAULT_COMPACTION_INSTRUCTIONS } from "./compact.js";
|
||||
|
||||
describe("DEFAULT_COMPACTION_INSTRUCTIONS", () => {
|
||||
it("contains priority ordering with numbered items", () => {
|
||||
expect(DEFAULT_COMPACTION_INSTRUCTIONS).toContain("1.");
|
||||
expect(DEFAULT_COMPACTION_INSTRUCTIONS).toContain("2.");
|
||||
expect(DEFAULT_COMPACTION_INSTRUCTIONS).toContain("3.");
|
||||
expect(DEFAULT_COMPACTION_INSTRUCTIONS).toContain("4.");
|
||||
expect(DEFAULT_COMPACTION_INSTRUCTIONS).toContain("5.");
|
||||
});
|
||||
|
||||
it("prioritizes active tasks first", () => {
|
||||
const taskLine = DEFAULT_COMPACTION_INSTRUCTIONS.indexOf("active or in-progress tasks");
|
||||
const decisionsLine = DEFAULT_COMPACTION_INSTRUCTIONS.indexOf("Key decisions");
|
||||
expect(taskLine).toBeLessThan(decisionsLine);
|
||||
expect(taskLine).toBeGreaterThan(-1);
|
||||
});
|
||||
|
||||
it("mentions TASKS.md for task ledger continuity", () => {
|
||||
expect(DEFAULT_COMPACTION_INSTRUCTIONS).toContain("TASKS.md");
|
||||
});
|
||||
|
||||
it("includes de-prioritization guidance", () => {
|
||||
expect(DEFAULT_COMPACTION_INSTRUCTIONS).toContain("De-prioritize");
|
||||
expect(DEFAULT_COMPACTION_INSTRUCTIONS).toContain("casual conversation");
|
||||
expect(DEFAULT_COMPACTION_INSTRUCTIONS).toContain("completed tasks");
|
||||
});
|
||||
|
||||
it("mentions exact values needed to resume work", () => {
|
||||
expect(DEFAULT_COMPACTION_INSTRUCTIONS).toContain("file paths");
|
||||
expect(DEFAULT_COMPACTION_INSTRUCTIONS).toContain("URLs");
|
||||
expect(DEFAULT_COMPACTION_INSTRUCTIONS).toContain("IDs");
|
||||
});
|
||||
|
||||
it("includes tool state preservation", () => {
|
||||
expect(DEFAULT_COMPACTION_INSTRUCTIONS).toContain("Tool state");
|
||||
expect(DEFAULT_COMPACTION_INSTRUCTIONS).toContain("browser sessions");
|
||||
});
|
||||
});
|
||||
|
||||
describe("compaction instructions merging", () => {
|
||||
it("custom instructions are appended to defaults", () => {
|
||||
const customInstructions = "Also remember to include user preferences.";
|
||||
const merged = `${DEFAULT_COMPACTION_INSTRUCTIONS}\n\n${customInstructions}`;
|
||||
|
||||
// Defaults come first
|
||||
expect(merged.indexOf("When summarizing")).toBeLessThan(merged.indexOf(customInstructions));
|
||||
// Custom instructions are present
|
||||
expect(merged).toContain(customInstructions);
|
||||
// Defaults are not lost
|
||||
expect(merged).toContain("active or in-progress tasks");
|
||||
});
|
||||
|
||||
it("when no custom instructions, defaults are used alone", () => {
|
||||
// Simulate the compaction path where customInstructions is undefined
|
||||
const resolve = (custom?: string) =>
|
||||
custom ? `${DEFAULT_COMPACTION_INSTRUCTIONS}\n\n${custom}` : DEFAULT_COMPACTION_INSTRUCTIONS;
|
||||
|
||||
const result = resolve(undefined);
|
||||
expect(result).toBe(DEFAULT_COMPACTION_INSTRUCTIONS);
|
||||
expect(result).not.toContain("\n\nundefined");
|
||||
});
|
||||
});
|
||||
@@ -79,6 +79,17 @@ import { splitSdkTools } from "./tool-split.js";
|
||||
import { describeUnknownError, mapThinkingLevel } from "./utils.js";
|
||||
import { flushPendingToolResultsAfterIdle } from "./wait-for-idle-before-flush.js";
|
||||
|
||||
export const DEFAULT_COMPACTION_INSTRUCTIONS = [
|
||||
"When summarizing this conversation, prioritize the following:",
|
||||
"1. Any active or in-progress tasks: include task name, current step, what has been done, what remains, and any pending user decisions.",
|
||||
"2. Key decisions made and their rationale.",
|
||||
"3. Exact values that would be needed to resume work: names, URLs, file paths, configuration values, row numbers, IDs.",
|
||||
"4. What the user was last working on and their most recent request.",
|
||||
"5. Tool state: any browser sessions, file operations, or API calls in progress.",
|
||||
"6. If TASKS.md was updated during this conversation, note which tasks changed and their current status.",
|
||||
"De-prioritize: casual conversation, greetings, completed tasks with no follow-up needed, resolved errors.",
|
||||
].join("\n");
|
||||
|
||||
export type CompactEmbeddedPiSessionParams = {
|
||||
sessionId: string;
|
||||
runId?: string;
|
||||
@@ -585,6 +596,48 @@ export async function compactEmbeddedPiSessionDirect(
|
||||
if (limited.length > 0) {
|
||||
session.agent.replaceMessages(limited);
|
||||
}
|
||||
|
||||
// Pre-check: detect "already compacted but context is high" scenario
|
||||
// The SDK rejects compaction if the last entry is a compaction, but this is
|
||||
// too aggressive when context has grown back to threshold levels.
|
||||
const branchEntries = sessionManager.getBranch();
|
||||
const lastEntry = branchEntries.length > 0 ? branchEntries[branchEntries.length - 1] : null;
|
||||
const isLastEntryCompaction = lastEntry?.type === "compaction";
|
||||
|
||||
if (isLastEntryCompaction) {
|
||||
// Check if there's actually new content since the compaction
|
||||
const compactionIndex = branchEntries.findIndex((e) => e.id === lastEntry.id);
|
||||
const hasNewContent = branchEntries
|
||||
.slice(compactionIndex + 1)
|
||||
.some((e) => e.type === "message" || e.type === "custom_message");
|
||||
|
||||
if (!hasNewContent) {
|
||||
// No new content since last compaction - estimate current context
|
||||
let currentTokens = 0;
|
||||
for (const message of session.messages) {
|
||||
currentTokens += estimateTokens(message);
|
||||
}
|
||||
const contextWindow = model.contextWindow ?? 200000;
|
||||
const contextPercent = (currentTokens / contextWindow) * 100;
|
||||
|
||||
// If context is still high (>70%) but no new content, provide clear error
|
||||
if (contextPercent > 70) {
|
||||
return {
|
||||
ok: false,
|
||||
compacted: false,
|
||||
reason: `Already compacted • Context ${Math.round(currentTokens / 1000)}k/${Math.round(contextWindow / 1000)}k (${Math.round(contextPercent)}%) — the compaction summary itself is large. Consider starting a new session with /new`,
|
||||
};
|
||||
}
|
||||
// Context is fine, just skip compaction gracefully
|
||||
return {
|
||||
ok: true,
|
||||
compacted: false,
|
||||
reason: "Already compacted",
|
||||
};
|
||||
}
|
||||
// Has new content - fall through to let SDK handle it (it should work now)
|
||||
}
|
||||
|
||||
// Run before_compaction hooks (fire-and-forget).
|
||||
// The session JSONL already contains all messages on disk, so plugins
|
||||
// can read sessionFile asynchronously and process in parallel with
|
||||
|
||||
@@ -31,6 +31,8 @@ import {
|
||||
listChannelSupportedActions,
|
||||
resolveChannelMessageToolHints,
|
||||
} from "../../channel-tools.js";
|
||||
import { estimateMessagesTokens } from "../../compaction.js";
|
||||
import { DEFAULT_CONTEXT_TOKENS } from "../../defaults.js";
|
||||
import { resolveOpenClawDocsPath } from "../../docs-path.js";
|
||||
import { isTimeoutError } from "../../failover-error.js";
|
||||
import { resolveModelAuthMode } from "../../model-auth.js";
|
||||
@@ -850,10 +852,16 @@ export async function runEmbeddedAttempt(
|
||||
let effectivePrompt = params.prompt;
|
||||
if (hookRunner?.hasHooks("before_agent_start")) {
|
||||
try {
|
||||
// Calculate context usage for mid-session memory refresh
|
||||
const contextWindowTokens = params.model.contextWindow ?? DEFAULT_CONTEXT_TOKENS;
|
||||
const estimatedUsedTokens = estimateMessagesTokens(activeSession.messages);
|
||||
|
||||
const hookResult = await hookRunner.runBeforeAgentStart(
|
||||
{
|
||||
prompt: params.prompt,
|
||||
messages: activeSession.messages,
|
||||
contextWindowTokens,
|
||||
estimatedUsedTokens,
|
||||
},
|
||||
{
|
||||
agentId: hookAgentId,
|
||||
|
||||
@@ -184,6 +184,28 @@ describe("buildEmbeddedRunPayloads", () => {
|
||||
expect(payloads[0]?.text).toContain("code 1");
|
||||
});
|
||||
|
||||
it("suppresses exec tool errors when mutatingAction is false and assistant produced a reply", () => {
|
||||
const payloads = buildPayloads({
|
||||
assistantTexts: ["I searched for PDF files but some directories were inaccessible."],
|
||||
lastAssistant: makeAssistant({
|
||||
stopReason: "stop",
|
||||
errorMessage: undefined,
|
||||
content: [],
|
||||
}),
|
||||
lastToolError: {
|
||||
toolName: "exec",
|
||||
error: "Command exited with code 1",
|
||||
mutatingAction: false,
|
||||
},
|
||||
});
|
||||
|
||||
expect(payloads).toHaveLength(1);
|
||||
expect(payloads[0]?.text).toBe(
|
||||
"I searched for PDF files but some directories were inaccessible.",
|
||||
);
|
||||
expect(payloads[0]?.isError).toBeUndefined();
|
||||
});
|
||||
|
||||
it("suppresses recoverable tool errors containing 'required' for non-mutating tools", () => {
|
||||
const payloads = buildPayloads({
|
||||
lastToolError: { toolName: "browser", error: "url required" },
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildAgentSystemPrompt } from "../system-prompt.js";
|
||||
|
||||
describe("Task Ledger section", () => {
|
||||
it("includes the Task Ledger section in full prompt mode", () => {
|
||||
const prompt = buildAgentSystemPrompt({
|
||||
workspaceDir: "/tmp/openclaw",
|
||||
});
|
||||
|
||||
expect(prompt).toContain("## Task Ledger (TASKS.md)");
|
||||
});
|
||||
|
||||
it("describes the task format with required fields", () => {
|
||||
const prompt = buildAgentSystemPrompt({
|
||||
workspaceDir: "/tmp/openclaw",
|
||||
});
|
||||
|
||||
expect(prompt).toContain("**Status:**");
|
||||
expect(prompt).toContain("**Started:**");
|
||||
expect(prompt).toContain("**Updated:**");
|
||||
expect(prompt).toContain("**Current Step:**");
|
||||
});
|
||||
|
||||
it("mentions stale task archival by sleep cycle", () => {
|
||||
const prompt = buildAgentSystemPrompt({
|
||||
workspaceDir: "/tmp/openclaw",
|
||||
});
|
||||
|
||||
expect(prompt).toContain("sleep cycle");
|
||||
expect(prompt).toContain(">24h");
|
||||
});
|
||||
|
||||
it("omits the section in minimal (subagent) prompt mode", () => {
|
||||
const prompt = buildAgentSystemPrompt({
|
||||
workspaceDir: "/tmp/openclaw",
|
||||
promptMode: "minimal",
|
||||
});
|
||||
|
||||
expect(prompt).not.toContain("## Task Ledger (TASKS.md)");
|
||||
});
|
||||
|
||||
it("omits the section in none prompt mode", () => {
|
||||
const prompt = buildAgentSystemPrompt({
|
||||
workspaceDir: "/tmp/openclaw",
|
||||
promptMode: "none",
|
||||
});
|
||||
|
||||
expect(prompt).not.toContain("## Task Ledger (TASKS.md)");
|
||||
});
|
||||
});
|
||||
|
||||
describe("Post-Compaction Recovery", () => {
|
||||
it("does NOT include a static recovery section (handled by framework injection)", () => {
|
||||
const prompt = buildAgentSystemPrompt({
|
||||
workspaceDir: "/tmp/openclaw",
|
||||
});
|
||||
|
||||
// Recovery instructions are injected dynamically via post-compaction-recovery.ts,
|
||||
// not baked into the system prompt (avoids wasting tokens on every turn).
|
||||
expect(prompt).not.toContain("## Post-Compaction Recovery");
|
||||
});
|
||||
});
|
||||
@@ -442,6 +442,25 @@ export async function ensureSandboxContainer(params: {
|
||||
});
|
||||
} else if (!running) {
|
||||
await execDocker(["start", containerName]);
|
||||
} else {
|
||||
// Container was already running – verify the workspace bind mount is still
|
||||
// valid. When the host directory backing the mount is deleted while the
|
||||
// container is running, any `docker exec` against it fails with an OCI
|
||||
// "mount namespace" error. Detect this and recreate the container.
|
||||
const probe = await execDocker(["exec", containerName, "true"], { allowFailure: true });
|
||||
if (probe.code !== 0 && probe.stderr.includes("mount namespace")) {
|
||||
defaultRuntime.log(`Sandbox mount stale for ${containerName}; recreating.`);
|
||||
await execDocker(["rm", "-f", containerName], { allowFailure: true });
|
||||
await createSandboxContainer({
|
||||
name: containerName,
|
||||
cfg: params.cfg.docker,
|
||||
workspaceDir: params.workspaceDir,
|
||||
workspaceAccess: params.cfg.workspaceAccess,
|
||||
agentWorkspaceDir: params.agentWorkspaceDir,
|
||||
scopeKey,
|
||||
configHash: expectedHash,
|
||||
});
|
||||
}
|
||||
}
|
||||
await updateRegistry({
|
||||
containerName,
|
||||
|
||||
@@ -49,7 +49,7 @@ async function listSandboxRegistryItems<
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
const agentId = resolveSandboxAgentId(entry.sessionKey);
|
||||
const agentId = resolveSandboxAgentId(entry.sessionKey) ?? "";
|
||||
const configuredImage = params.resolveConfiguredImage(agentId);
|
||||
results.push({
|
||||
...entry,
|
||||
|
||||
@@ -129,6 +129,54 @@ describe("buildWorkspaceSkillCommandSpecs", () => {
|
||||
const cmd = commands.find((entry) => entry.skillName === "tool-dispatch");
|
||||
expect(cmd?.dispatch).toEqual({ kind: "tool", toolName: "sessions_send", argMode: "raw" });
|
||||
});
|
||||
|
||||
it("includes thinking and model from skill config", async () => {
|
||||
const workspaceDir = await makeWorkspace();
|
||||
await writeSkill({
|
||||
dir: path.join(workspaceDir, "skills", "browser"),
|
||||
name: "browser",
|
||||
description: "Browser automation",
|
||||
});
|
||||
await writeSkill({
|
||||
dir: path.join(workspaceDir, "skills", "replicate-image"),
|
||||
name: "replicate-image",
|
||||
description: "Image generation",
|
||||
});
|
||||
await writeSkill({
|
||||
dir: path.join(workspaceDir, "skills", "no-config"),
|
||||
name: "no-config",
|
||||
description: "No special config",
|
||||
});
|
||||
|
||||
const commands = buildWorkspaceSkillCommandSpecs(workspaceDir, {
|
||||
config: {
|
||||
skills: {
|
||||
entries: {
|
||||
browser: {
|
||||
thinking: "xhigh",
|
||||
model: "anthropic/claude-opus-4-5",
|
||||
},
|
||||
"replicate-image": {
|
||||
thinking: "low",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const browserCmd = commands.find((entry) => entry.skillName === "browser");
|
||||
const replicateCmd = commands.find((entry) => entry.skillName === "replicate-image");
|
||||
const noConfigCmd = commands.find((entry) => entry.skillName === "no-config");
|
||||
|
||||
expect(browserCmd?.thinking).toBe("xhigh");
|
||||
expect(browserCmd?.model).toBe("anthropic/claude-opus-4-5");
|
||||
|
||||
expect(replicateCmd?.thinking).toBe("low");
|
||||
expect(replicateCmd?.model).toBeUndefined();
|
||||
|
||||
expect(noConfigCmd?.thinking).toBeUndefined();
|
||||
expect(noConfigCmd?.model).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildWorkspaceSkillsPrompt", () => {
|
||||
|
||||
@@ -54,6 +54,8 @@ export type SkillCommandSpec = {
|
||||
description: string;
|
||||
/** Optional deterministic dispatch behavior for this command. */
|
||||
dispatch?: SkillCommandDispatchSpec;
|
||||
thinking?: "off" | "minimal" | "low" | "medium" | "high" | "xhigh";
|
||||
model?: string;
|
||||
};
|
||||
|
||||
export type SkillsInstallPreferences = {
|
||||
|
||||
@@ -18,12 +18,13 @@ import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
import { CONFIG_DIR, resolveUserPath } from "../../utils.js";
|
||||
import { resolveSandboxPath } from "../sandbox-paths.js";
|
||||
import { resolveBundledSkillsDir } from "./bundled-dir.js";
|
||||
import { shouldIncludeSkill } from "./config.js";
|
||||
import { resolveSkillConfig, shouldIncludeSkill } from "./config.js";
|
||||
import { normalizeSkillFilter } from "./filter.js";
|
||||
import {
|
||||
parseFrontmatter,
|
||||
resolveOpenClawMetadata,
|
||||
resolveSkillInvocationPolicy,
|
||||
resolveSkillKey,
|
||||
} from "./frontmatter.js";
|
||||
import { resolvePluginSkillDirs } from "./plugin-skills.js";
|
||||
import { serializeByKey } from "./serialize.js";
|
||||
@@ -107,6 +108,8 @@ function loadSkillEntries(
|
||||
config?: OpenClawConfig;
|
||||
managedSkillsDir?: string;
|
||||
bundledSkillsDir?: string;
|
||||
/** When true, only load skills from the workspace dir (skip managed/bundled/extra). */
|
||||
scopeToWorkspace?: boolean;
|
||||
},
|
||||
): SkillEntry[] {
|
||||
const loadSkills = (params: { dir: string; source: string }): Skill[] => {
|
||||
@@ -125,8 +128,34 @@ function loadSkillEntries(
|
||||
return [];
|
||||
};
|
||||
|
||||
const workspaceSkillsDir = path.join(workspaceDir, "skills");
|
||||
|
||||
// When scoped to workspace, only load skills from the workspace dir.
|
||||
// This prevents managed/bundled skill paths from leaking into sandboxed
|
||||
// agents where those paths are outside the sandbox root.
|
||||
if (opts?.scopeToWorkspace) {
|
||||
const workspaceSkills = loadSkills({
|
||||
dir: workspaceSkillsDir,
|
||||
source: "openclaw-workspace",
|
||||
});
|
||||
return workspaceSkills.map((skill) => {
|
||||
let frontmatter: ParsedSkillFrontmatter = {};
|
||||
try {
|
||||
const raw = fs.readFileSync(skill.filePath, "utf-8");
|
||||
frontmatter = parseFrontmatter(raw);
|
||||
} catch {
|
||||
// ignore malformed skills
|
||||
}
|
||||
return {
|
||||
skill,
|
||||
frontmatter,
|
||||
metadata: resolveOpenClawMetadata(frontmatter),
|
||||
invocation: resolveSkillInvocationPolicy(frontmatter),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
const managedSkillsDir = opts?.managedSkillsDir ?? path.join(CONFIG_DIR, "skills");
|
||||
const workspaceSkillsDir = path.resolve(workspaceDir, "skills");
|
||||
const bundledSkillsDir = opts?.bundledSkillsDir ?? resolveBundledSkillsDir();
|
||||
const extraDirsRaw = opts?.config?.skills?.load?.extraDirs ?? [];
|
||||
const extraDirs = extraDirsRaw
|
||||
@@ -220,6 +249,8 @@ export function buildWorkspaceSkillSnapshot(
|
||||
skillFilter?: string[];
|
||||
eligibility?: SkillEligibilityContext;
|
||||
snapshotVersion?: number;
|
||||
/** When true, only load skills from the workspace dir (for sandboxed agents). */
|
||||
scopeToWorkspace?: boolean;
|
||||
},
|
||||
): SkillSnapshot {
|
||||
const skillEntries = opts?.entries ?? loadSkillEntries(workspaceDir, opts);
|
||||
@@ -355,6 +386,7 @@ export async function syncSkillsToWorkspace(params: {
|
||||
}) {
|
||||
const sourceDir = resolveUserPath(params.sourceWorkspaceDir);
|
||||
const targetDir = resolveUserPath(params.targetWorkspaceDir);
|
||||
|
||||
if (sourceDir === targetDir) {
|
||||
return;
|
||||
}
|
||||
@@ -511,11 +543,18 @@ export function buildWorkspaceSkillCommandSpecs(
|
||||
return { kind: "tool", toolName, argMode: "raw" } as const;
|
||||
})();
|
||||
|
||||
const skillKey = resolveSkillKey(entry.skill, entry);
|
||||
const skillConfig = resolveSkillConfig(opts?.config, skillKey);
|
||||
const thinking = skillConfig?.thinking;
|
||||
const model = skillConfig?.model;
|
||||
|
||||
specs.push({
|
||||
name: unique,
|
||||
skillName: rawName,
|
||||
description,
|
||||
...(dispatch ? { dispatch } : {}),
|
||||
...(thinking ? { thinking } : {}),
|
||||
...(model ? { model } : {}),
|
||||
});
|
||||
}
|
||||
return specs;
|
||||
|
||||
@@ -44,6 +44,9 @@ function buildInjectedWorkspaceFiles(params: {
|
||||
const injectedByPath = new Map(params.injectedFiles.map((f) => [f.path, f.content]));
|
||||
const injectedByBaseName = new Map<string, string>();
|
||||
for (const file of params.injectedFiles) {
|
||||
if (!file.path) {
|
||||
continue;
|
||||
}
|
||||
const normalizedPath = file.path.replace(/\\/g, "/");
|
||||
const baseName = path.posix.basename(normalizedPath);
|
||||
if (!injectedByBaseName.has(baseName)) {
|
||||
|
||||
@@ -73,11 +73,19 @@ function buildUserIdentitySection(ownerLine: string | undefined, isMinimal: bool
|
||||
return ["## User Identity", ownerLine, ""];
|
||||
}
|
||||
|
||||
function buildTimeSection(params: { userTimezone?: string }) {
|
||||
function buildTimeSection(params: { userTimezone?: string; userTime?: string }) {
|
||||
if (!params.userTimezone) {
|
||||
return [];
|
||||
}
|
||||
return ["## Current Date & Time", `Time zone: ${params.userTimezone}`, ""];
|
||||
const lines = ["## Current Date & Time", `Time zone: ${params.userTimezone}`];
|
||||
if (params.userTime) {
|
||||
lines.push(`Current time: ${params.userTime}`);
|
||||
}
|
||||
lines.push(
|
||||
"If you need the current date, time, or day of week, use the session_status tool.",
|
||||
"",
|
||||
);
|
||||
return lines;
|
||||
}
|
||||
|
||||
function buildReplyTagsSection(isMinimal: boolean) {
|
||||
@@ -340,6 +348,7 @@ export function buildAgentSystemPrompt(params: {
|
||||
: undefined;
|
||||
const reasoningLevel = params.reasoningLevel ?? "off";
|
||||
const userTimezone = params.userTimezone?.trim();
|
||||
const userTime = params.userTime?.trim();
|
||||
const skillsPrompt = params.skillsPrompt?.trim();
|
||||
const heartbeatPrompt = params.heartbeatPrompt?.trim();
|
||||
const heartbeatPromptLine = heartbeatPrompt
|
||||
@@ -526,6 +535,7 @@ export function buildAgentSystemPrompt(params: {
|
||||
...buildUserIdentitySection(ownerLine, isMinimal),
|
||||
...buildTimeSection({
|
||||
userTimezone,
|
||||
userTime,
|
||||
}),
|
||||
"## Workspace Files (injected)",
|
||||
"These user-editable files are loaded by OpenClaw and included below in Project Context.",
|
||||
@@ -615,6 +625,38 @@ export function buildAgentSystemPrompt(params: {
|
||||
);
|
||||
}
|
||||
|
||||
// Task Ledger instructions (skip for subagent/none modes)
|
||||
if (!isMinimal) {
|
||||
lines.push(
|
||||
"## Task Ledger (TASKS.md)",
|
||||
"Maintain a TASKS.md file in the workspace root to track active work across compaction events.",
|
||||
"Update it whenever you start, progress, or complete a task. Format:",
|
||||
"",
|
||||
"```markdown",
|
||||
"# Active Tasks",
|
||||
"",
|
||||
"## TASK-001: <short title>",
|
||||
"- **Status:** in_progress | awaiting_input | blocked | done",
|
||||
"- **Started:** YYYY-MM-DD HH:MM",
|
||||
"- **Updated:** YYYY-MM-DD HH:MM",
|
||||
"- **Details:** What this task is about",
|
||||
"- **Current Step:** What you're doing right now",
|
||||
"- **Blocked On:** (if applicable) What's preventing progress",
|
||||
"",
|
||||
"# Completed",
|
||||
"<!-- Move done tasks here with completion date -->",
|
||||
"```",
|
||||
"",
|
||||
"Rules:",
|
||||
"- Create TASKS.md on first task if it doesn't exist.",
|
||||
"- Update **Updated** timestamp and **Current Step** as you make progress.",
|
||||
"- Move tasks to Completed when done; include completion date.",
|
||||
"- Keep IDs sequential (TASK-001, TASK-002, etc.).",
|
||||
"- Stale tasks (>24h with no update) may be auto-archived by the sleep cycle.",
|
||||
"",
|
||||
);
|
||||
}
|
||||
|
||||
// Skip heartbeats for subagent/none modes
|
||||
if (!isMinimal) {
|
||||
lines.push(
|
||||
|
||||
@@ -61,6 +61,35 @@ describe("tool mutation helpers", () => {
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("classifies read-only exec/bash commands as non-mutating", () => {
|
||||
expect(isMutatingToolCall("exec", { command: "find ~ -iname '*.pdf' 2>/dev/null" })).toBe(
|
||||
false,
|
||||
);
|
||||
expect(isMutatingToolCall("bash", { command: "ls -la" })).toBe(false);
|
||||
expect(isMutatingToolCall("exec", { command: "grep pattern file.txt" })).toBe(false);
|
||||
expect(isMutatingToolCall("exec", { command: "echo hello" })).toBe(false);
|
||||
expect(isMutatingToolCall("bash", { command: "cat file | grep foo" })).toBe(false);
|
||||
expect(isMutatingToolCall("exec", { command: "FOO=bar find ." })).toBe(false);
|
||||
expect(isMutatingToolCall("bash", { command: "/usr/bin/find . -name '*.ts'" })).toBe(false);
|
||||
expect(isMutatingToolCall("exec", { command: "sudo ls /root" })).toBe(false);
|
||||
expect(isMutatingToolCall("bash", { command: "time grep -r pattern src/" })).toBe(false);
|
||||
expect(isMutatingToolCall("exec", { command: "jq '.name' package.json" })).toBe(false);
|
||||
});
|
||||
|
||||
it("classifies mutating exec/bash commands conservatively", () => {
|
||||
expect(isMutatingToolCall("exec", { command: "rm -rf /tmp/foo" })).toBe(true);
|
||||
expect(isMutatingToolCall("bash", { command: "npm install" })).toBe(true);
|
||||
expect(isMutatingToolCall("exec", { command: "git push origin main" })).toBe(true);
|
||||
expect(isMutatingToolCall("bash", { command: "mv file1.txt file2.txt" })).toBe(true);
|
||||
});
|
||||
|
||||
it("treats empty or missing exec/bash command as mutating (conservative)", () => {
|
||||
expect(isMutatingToolCall("exec", {})).toBe(true);
|
||||
expect(isMutatingToolCall("bash", { command: "" })).toBe(true);
|
||||
expect(isMutatingToolCall("exec", { command: " " })).toBe(true);
|
||||
expect(isMutatingToolCall("bash", undefined)).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps legacy name-only mutating heuristics for payload fallback", () => {
|
||||
expect(isLikelyMutatingToolName("sessions_send")).toBe(true);
|
||||
expect(isLikelyMutatingToolName("browser_actions")).toBe(true);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import path from "node:path";
|
||||
|
||||
const MUTATING_TOOL_NAMES = new Set([
|
||||
"write",
|
||||
"edit",
|
||||
@@ -14,6 +16,131 @@ const MUTATING_TOOL_NAMES = new Set([
|
||||
"session_status",
|
||||
]);
|
||||
|
||||
const READ_ONLY_EXEC_COMMANDS = new Set([
|
||||
"find",
|
||||
"locate",
|
||||
"ls",
|
||||
"dir",
|
||||
"tree",
|
||||
"cat",
|
||||
"head",
|
||||
"tail",
|
||||
"less",
|
||||
"more",
|
||||
"tac",
|
||||
"grep",
|
||||
"egrep",
|
||||
"fgrep",
|
||||
"rg",
|
||||
"ag",
|
||||
"ack",
|
||||
"wc",
|
||||
"sort",
|
||||
"uniq",
|
||||
"cut",
|
||||
"tr",
|
||||
"fold",
|
||||
"paste",
|
||||
"column",
|
||||
"diff",
|
||||
"comm",
|
||||
"cmp",
|
||||
"which",
|
||||
"whereis",
|
||||
"whence",
|
||||
"type",
|
||||
"command",
|
||||
"hash",
|
||||
"file",
|
||||
"stat",
|
||||
"readlink",
|
||||
"realpath",
|
||||
"du",
|
||||
"df",
|
||||
"free",
|
||||
"lsblk",
|
||||
"date",
|
||||
"cal",
|
||||
"uptime",
|
||||
"w",
|
||||
"who",
|
||||
"whoami",
|
||||
"id",
|
||||
"groups",
|
||||
"logname",
|
||||
"uname",
|
||||
"hostname",
|
||||
"hostnamectl",
|
||||
"arch",
|
||||
"nproc",
|
||||
"lscpu",
|
||||
"env",
|
||||
"printenv",
|
||||
"locale",
|
||||
"echo",
|
||||
"printf",
|
||||
"test",
|
||||
"[",
|
||||
"true",
|
||||
"false",
|
||||
"basename",
|
||||
"dirname",
|
||||
"seq",
|
||||
"yes",
|
||||
"md5sum",
|
||||
"sha256sum",
|
||||
"sha1sum",
|
||||
"shasum",
|
||||
"cksum",
|
||||
"strings",
|
||||
"xxd",
|
||||
"od",
|
||||
"hexdump",
|
||||
"jq",
|
||||
"yq",
|
||||
"xq",
|
||||
"ps",
|
||||
"pgrep",
|
||||
"lsof",
|
||||
"ss",
|
||||
"netstat",
|
||||
"dig",
|
||||
"nslookup",
|
||||
"host",
|
||||
"ping",
|
||||
"curl",
|
||||
"wget",
|
||||
]);
|
||||
|
||||
const SKIP_PREFIXES = new Set(["sudo", "nice", "time", "env", "ionice", "strace", "ltrace"]);
|
||||
|
||||
function isReadOnlyShellCommand(command: string): boolean {
|
||||
if (!command) {
|
||||
return false;
|
||||
}
|
||||
const tokens = command.split(/\s+/);
|
||||
let i = 0;
|
||||
// Skip env-var assignments (FOO=bar) and common prefixes
|
||||
while (i < tokens.length) {
|
||||
const token = tokens[i];
|
||||
if (/^[A-Za-z_][A-Za-z0-9_]*=/.test(token)) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (SKIP_PREFIXES.has(token)) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
const firstCmd = tokens[i];
|
||||
if (!firstCmd) {
|
||||
return false;
|
||||
}
|
||||
const baseName = path.basename(firstCmd);
|
||||
return READ_ONLY_EXEC_COMMANDS.has(baseName);
|
||||
}
|
||||
|
||||
const READ_ONLY_ACTIONS = new Set([
|
||||
"get",
|
||||
"list",
|
||||
@@ -104,10 +231,13 @@ export function isMutatingToolCall(toolName: string, args: unknown): boolean {
|
||||
case "write":
|
||||
case "edit":
|
||||
case "apply_patch":
|
||||
case "exec":
|
||||
case "bash":
|
||||
case "sessions_send":
|
||||
return true;
|
||||
case "exec":
|
||||
case "bash": {
|
||||
const command = typeof record?.command === "string" ? record.command.trim() : "";
|
||||
return !isReadOnlyShellCommand(command);
|
||||
}
|
||||
case "process":
|
||||
return action != null && PROCESS_MUTATING_ACTIONS.has(action);
|
||||
case "message":
|
||||
|
||||
79
src/agents/workspace.tasks-bootstrap.test.ts
Normal file
79
src/agents/workspace.tasks-bootstrap.test.ts
Normal file
@@ -0,0 +1,79 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { makeTempWorkspace, writeWorkspaceFile } from "../test-helpers/workspace.js";
|
||||
import {
|
||||
DEFAULT_TASKS_FILENAME,
|
||||
filterBootstrapFilesForSession,
|
||||
loadWorkspaceBootstrapFiles,
|
||||
} from "./workspace.js";
|
||||
|
||||
describe("TASKS.md bootstrap", () => {
|
||||
it("DEFAULT_TASKS_FILENAME equals TASKS.md", () => {
|
||||
expect(DEFAULT_TASKS_FILENAME).toBe("TASKS.md");
|
||||
});
|
||||
|
||||
it("loadWorkspaceBootstrapFiles includes TASKS.md entry", async () => {
|
||||
const tempDir = await makeTempWorkspace("openclaw-tasks-");
|
||||
|
||||
const files = await loadWorkspaceBootstrapFiles(tempDir);
|
||||
const tasksEntry = files.find((f) => f.name === DEFAULT_TASKS_FILENAME);
|
||||
|
||||
expect(tasksEntry).toBeDefined();
|
||||
});
|
||||
|
||||
it("loads TASKS.md content when the file exists", async () => {
|
||||
const tempDir = await makeTempWorkspace("openclaw-tasks-");
|
||||
await writeWorkspaceFile({ dir: tempDir, name: "TASKS.md", content: "- [ ] finish tests" });
|
||||
|
||||
const files = await loadWorkspaceBootstrapFiles(tempDir);
|
||||
const tasksEntry = files.find((f) => f.name === DEFAULT_TASKS_FILENAME);
|
||||
|
||||
expect(tasksEntry).toBeDefined();
|
||||
expect(tasksEntry!.missing).toBe(false);
|
||||
expect(tasksEntry!.content).toBe("- [ ] finish tests");
|
||||
});
|
||||
|
||||
it("marks TASKS.md as missing (not error) when the file does not exist", async () => {
|
||||
const tempDir = await makeTempWorkspace("openclaw-tasks-");
|
||||
|
||||
const files = await loadWorkspaceBootstrapFiles(tempDir);
|
||||
const tasksEntry = files.find((f) => f.name === DEFAULT_TASKS_FILENAME);
|
||||
|
||||
expect(tasksEntry).toBeDefined();
|
||||
expect(tasksEntry!.missing).toBe(true);
|
||||
expect(tasksEntry!.content).toBeUndefined();
|
||||
});
|
||||
|
||||
it("TASKS.md is in SUBAGENT_BOOTSTRAP_ALLOWLIST (kept for subagent sessions)", () => {
|
||||
const files = [
|
||||
{
|
||||
name: DEFAULT_TASKS_FILENAME as const,
|
||||
path: "/tmp/TASKS.md",
|
||||
missing: false,
|
||||
content: "tasks",
|
||||
},
|
||||
{ name: "SOUL.md" as const, path: "/tmp/SOUL.md", missing: false, content: "soul" },
|
||||
];
|
||||
|
||||
const filtered = filterBootstrapFilesForSession(files, "agent:main:subagent:test-123");
|
||||
|
||||
const tasksKept = filtered.find((f) => f.name === DEFAULT_TASKS_FILENAME);
|
||||
expect(tasksKept).toBeDefined();
|
||||
});
|
||||
|
||||
it("filterBootstrapFilesForSession drops non-allowlisted files for subagent sessions", () => {
|
||||
const files = [
|
||||
{
|
||||
name: DEFAULT_TASKS_FILENAME as const,
|
||||
path: "/tmp/TASKS.md",
|
||||
missing: false,
|
||||
content: "tasks",
|
||||
},
|
||||
{ name: "SOUL.md" as const, path: "/tmp/SOUL.md", missing: false, content: "soul" },
|
||||
];
|
||||
|
||||
const filtered = filterBootstrapFilesForSession(files, "agent:main:subagent:test-123");
|
||||
|
||||
const soulKept = filtered.find((f) => f.name === "SOUL.md");
|
||||
expect(soulKept).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -28,6 +28,7 @@ export const DEFAULT_USER_FILENAME = "USER.md";
|
||||
export const DEFAULT_HEARTBEAT_FILENAME = "HEARTBEAT.md";
|
||||
export const DEFAULT_BOOTSTRAP_FILENAME = "BOOTSTRAP.md";
|
||||
export const DEFAULT_MEMORY_FILENAME = "MEMORY.md";
|
||||
export const DEFAULT_TASKS_FILENAME = "TASKS.md";
|
||||
export const DEFAULT_MEMORY_ALT_FILENAME = "memory.md";
|
||||
const WORKSPACE_STATE_DIRNAME = ".openclaw";
|
||||
const WORKSPACE_STATE_FILENAME = "workspace-state.json";
|
||||
@@ -87,6 +88,7 @@ export type WorkspaceBootstrapFileName =
|
||||
| typeof DEFAULT_HEARTBEAT_FILENAME
|
||||
| typeof DEFAULT_BOOTSTRAP_FILENAME
|
||||
| typeof DEFAULT_MEMORY_FILENAME
|
||||
| typeof DEFAULT_TASKS_FILENAME
|
||||
| typeof DEFAULT_MEMORY_ALT_FILENAME;
|
||||
|
||||
export type WorkspaceBootstrapFile = {
|
||||
@@ -444,6 +446,10 @@ export async function loadWorkspaceBootstrapFiles(dir: string): Promise<Workspac
|
||||
name: DEFAULT_BOOTSTRAP_FILENAME,
|
||||
filePath: path.join(resolvedDir, DEFAULT_BOOTSTRAP_FILENAME),
|
||||
},
|
||||
{
|
||||
name: DEFAULT_TASKS_FILENAME,
|
||||
filePath: path.join(resolvedDir, DEFAULT_TASKS_FILENAME),
|
||||
},
|
||||
];
|
||||
|
||||
entries.push(...(await resolveMemoryBootstrapEntries(resolvedDir)));
|
||||
@@ -465,7 +471,11 @@ export async function loadWorkspaceBootstrapFiles(dir: string): Promise<Workspac
|
||||
return result;
|
||||
}
|
||||
|
||||
const MINIMAL_BOOTSTRAP_ALLOWLIST = new Set([DEFAULT_AGENTS_FILENAME, DEFAULT_TOOLS_FILENAME]);
|
||||
const SUBAGENT_BOOTSTRAP_ALLOWLIST = new Set([
|
||||
DEFAULT_AGENTS_FILENAME,
|
||||
DEFAULT_TOOLS_FILENAME,
|
||||
DEFAULT_TASKS_FILENAME,
|
||||
]);
|
||||
|
||||
export function filterBootstrapFilesForSession(
|
||||
files: WorkspaceBootstrapFile[],
|
||||
@@ -474,7 +484,7 @@ export function filterBootstrapFilesForSession(
|
||||
if (!sessionKey || (!isSubagentSessionKey(sessionKey) && !isCronSessionKey(sessionKey))) {
|
||||
return files;
|
||||
}
|
||||
return files.filter((file) => MINIMAL_BOOTSTRAP_ALLOWLIST.has(file.name));
|
||||
return files.filter((file) => SUBAGENT_BOOTSTRAP_ALLOWLIST.has(file.name));
|
||||
}
|
||||
|
||||
export async function loadExtraBootstrapFiles(
|
||||
|
||||
@@ -34,9 +34,12 @@ export function hasControlCommand(
|
||||
if (lowered === normalized) {
|
||||
return true;
|
||||
}
|
||||
if (lowered === `${normalized}:`) {
|
||||
return true;
|
||||
}
|
||||
if (command.acceptsArgs && lowered.startsWith(normalized)) {
|
||||
const nextChar = normalizedBody.charAt(normalized.length);
|
||||
if (nextChar && /\s/.test(nextChar)) {
|
||||
if (nextChar === ":" || (nextChar && /\s/.test(nextChar))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,8 +111,8 @@ function assertCommandRegistry(commands: ChatCommandDefinition[]): void {
|
||||
}
|
||||
|
||||
for (const alias of command.textAliases) {
|
||||
if (!alias.startsWith("/")) {
|
||||
throw new Error(`Command alias missing leading '/': ${alias}`);
|
||||
if (!alias.startsWith("/") && !alias.startsWith(".")) {
|
||||
throw new Error(`Command alias missing leading '/' or '.': ${alias}`);
|
||||
}
|
||||
const aliasKey = alias.toLowerCase();
|
||||
if (textAliases.has(aliasKey)) {
|
||||
@@ -618,6 +618,8 @@ function buildChatCommands(): ChatCommandDefinition[] {
|
||||
registerAlias(commands, "reasoning", "/reason");
|
||||
registerAlias(commands, "elevated", "/elev");
|
||||
registerAlias(commands, "steer", "/tell");
|
||||
registerAlias(commands, "model", ".model");
|
||||
registerAlias(commands, "models", ".models");
|
||||
|
||||
assertCommandRegistry(commands);
|
||||
return commands;
|
||||
|
||||
@@ -14,7 +14,7 @@ export function extractModelDirective(
|
||||
}
|
||||
|
||||
const modelMatch = body.match(
|
||||
/(?:^|\s)\/model(?=$|\s|:)\s*:?\s*([A-Za-z0-9_.:@-]+(?:\/[A-Za-z0-9_.:@-]+)*)?/i,
|
||||
/(?:^|\s)[/.]model(?=$|\s|:)\s*:?\s*([A-Za-z0-9_.:@-]+(?:\/[A-Za-z0-9_.:@-]+)*)?/i,
|
||||
);
|
||||
|
||||
const aliases = (options?.aliases ?? []).map((alias) => alias.trim()).filter(Boolean);
|
||||
@@ -23,7 +23,7 @@ export function extractModelDirective(
|
||||
? null
|
||||
: body.match(
|
||||
new RegExp(
|
||||
`(?:^|\\s)\\/(${aliases.map(escapeRegExp).join("|")})(?=$|\\s|:)(?:\\s*:\\s*)?`,
|
||||
`(?:^|\\s)[/.](${aliases.map(escapeRegExp).join("|")})(?=$|\\s|:)(?:\\s*:\\s*)?`,
|
||||
"i",
|
||||
),
|
||||
);
|
||||
|
||||
@@ -540,7 +540,10 @@ export async function runAgentTurnWithFallback(params: {
|
||||
continue;
|
||||
}
|
||||
|
||||
defaultRuntime.error(`Embedded agent failed before reply: ${message}`);
|
||||
const stack = err instanceof Error ? err.stack : undefined;
|
||||
defaultRuntime.error(
|
||||
`Embedded agent failed before reply: ${message}${stack ? `\n${stack}` : ""}`,
|
||||
);
|
||||
const safeMessage = isTransientHttp
|
||||
? sanitizeUserFacingText(message, { errorContext: true })
|
||||
: message;
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
resolveMemoryFlushSettings,
|
||||
shouldRunMemoryFlush,
|
||||
} from "./memory-flush.js";
|
||||
import { markNeedsPostCompactionRecovery } from "./post-compaction-recovery.js";
|
||||
import { incrementCompactionCount } from "./session-updates.js";
|
||||
|
||||
export async function runMemoryFlushIfNeeded(params: {
|
||||
@@ -179,6 +180,16 @@ export async function runMemoryFlushIfNeeded(params: {
|
||||
if (typeof nextCount === "number") {
|
||||
memoryFlushCompactionCount = nextCount;
|
||||
}
|
||||
// P3: Mark session for post-compaction recovery on the next turn.
|
||||
// This path handles flush-triggered compaction (memory flush forces a compact).
|
||||
// The main path in agent-runner.ts handles SDK auto-compaction.
|
||||
// These are mutually exclusive; setting true is idempotent.
|
||||
await markNeedsPostCompactionRecovery({
|
||||
sessionEntry: activeSessionEntry,
|
||||
sessionStore: activeSessionStore,
|
||||
sessionKey: params.sessionKey,
|
||||
storePath: params.storePath,
|
||||
});
|
||||
}
|
||||
if (params.storePath && params.sessionKey) {
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
DEFAULT_MEMORY_FLUSH_PROMPT,
|
||||
DEFAULT_MEMORY_FLUSH_SYSTEM_PROMPT,
|
||||
resolveMemoryFlushSettings,
|
||||
} from "./memory-flush.js";
|
||||
|
||||
describe("memory flush task checkpoint", () => {
|
||||
describe("DEFAULT_MEMORY_FLUSH_PROMPT", () => {
|
||||
it("includes task state extraction language", () => {
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toContain("active task");
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toContain("task name");
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toContain("current step");
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toContain("pending actions");
|
||||
});
|
||||
|
||||
it("instructs to use memory_store with core category and importance 1.0", () => {
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toContain("memory_store");
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toContain("category 'core'");
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toContain("importance 1.0");
|
||||
});
|
||||
});
|
||||
|
||||
describe("DEFAULT_MEMORY_FLUSH_SYSTEM_PROMPT", () => {
|
||||
it("includes CRITICAL instruction about active tasks", () => {
|
||||
expect(DEFAULT_MEMORY_FLUSH_SYSTEM_PROMPT).toContain("CRITICAL");
|
||||
expect(DEFAULT_MEMORY_FLUSH_SYSTEM_PROMPT).toContain("active task");
|
||||
});
|
||||
|
||||
it("instructs to save task state with core category", () => {
|
||||
expect(DEFAULT_MEMORY_FLUSH_SYSTEM_PROMPT).toContain("memory_store");
|
||||
expect(DEFAULT_MEMORY_FLUSH_SYSTEM_PROMPT).toContain("category='core'");
|
||||
expect(DEFAULT_MEMORY_FLUSH_SYSTEM_PROMPT).toContain("importance=1.0");
|
||||
});
|
||||
|
||||
it("mentions task continuity after compaction", () => {
|
||||
expect(DEFAULT_MEMORY_FLUSH_SYSTEM_PROMPT).toContain("task continuity after compaction");
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveMemoryFlushSettings", () => {
|
||||
it("returns prompts containing task-related keywords by default", () => {
|
||||
const settings = resolveMemoryFlushSettings();
|
||||
expect(settings).not.toBeNull();
|
||||
expect(settings?.prompt).toContain("active task");
|
||||
expect(settings?.prompt).toContain("memory_store");
|
||||
expect(settings?.systemPrompt).toContain("CRITICAL");
|
||||
expect(settings?.systemPrompt).toContain("task continuity");
|
||||
});
|
||||
|
||||
it("preserves task checkpoint language alongside existing content", () => {
|
||||
const settings = resolveMemoryFlushSettings();
|
||||
expect(settings).not.toBeNull();
|
||||
// Original content still present
|
||||
expect(settings?.prompt).toContain("Pre-compaction memory flush");
|
||||
expect(settings?.prompt).toContain("durable memories");
|
||||
// New task checkpoint content also present
|
||||
expect(settings?.prompt).toContain("current step");
|
||||
expect(settings?.prompt).toContain("pending actions");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -36,6 +36,7 @@ import { appendUsageLine, formatResponseUsageLine } from "./agent-runner-utils.j
|
||||
import { createAudioAsVoiceBuffer, createBlockReplyPipeline } from "./block-reply-pipeline.js";
|
||||
import { resolveBlockStreamingCoalescing } from "./block-streaming.js";
|
||||
import { createFollowupRunner } from "./followup-runner.js";
|
||||
import { markNeedsPostCompactionRecovery } from "./post-compaction-recovery.js";
|
||||
import { enqueueFollowupRun, type FollowupRun, type QueueSettings } from "./queue.js";
|
||||
import { createReplyToModeFilterForChannel, resolveReplyToMode } from "./reply-threading.js";
|
||||
import { incrementRunCompactionCount, persistRunSessionUsage } from "./session-run-accounting.js";
|
||||
@@ -499,6 +500,16 @@ export async function runReplyAgent(params: {
|
||||
lastCallUsage: runResult.meta.agentMeta?.lastCallUsage,
|
||||
contextTokensUsed,
|
||||
});
|
||||
// P3: Mark session for post-compaction recovery on the next turn.
|
||||
// This path handles SDK auto-compaction (during the agent run itself).
|
||||
// The memory-flush path in agent-runner-memory.ts handles flush-triggered compaction.
|
||||
// These are mutually exclusive for a given compaction event; setting true is idempotent.
|
||||
await markNeedsPostCompactionRecovery({
|
||||
sessionEntry: activeSessionEntry,
|
||||
sessionStore: activeSessionStore,
|
||||
sessionKey,
|
||||
storePath,
|
||||
});
|
||||
if (verboseEnabled) {
|
||||
const suffix = typeof count === "number" ? ` (count ${count})` : "";
|
||||
finalPayloads = [{ text: `🧹 Auto-compaction complete${suffix}.` }, ...finalPayloads];
|
||||
|
||||
@@ -65,22 +65,23 @@ async function resolveContextReport(
|
||||
sessionKey: params.sessionKey,
|
||||
sessionId: params.sessionEntry?.sessionId,
|
||||
});
|
||||
const sandboxRuntime = resolveSandboxRuntimeStatus({
|
||||
cfg: params.cfg,
|
||||
sessionKey: params.ctx.SessionKey ?? params.sessionKey,
|
||||
});
|
||||
const skillsSnapshot = (() => {
|
||||
try {
|
||||
return buildWorkspaceSkillSnapshot(workspaceDir, {
|
||||
config: params.cfg,
|
||||
eligibility: { remote: getRemoteSkillEligibility() },
|
||||
snapshotVersion: getSkillsSnapshotVersion(workspaceDir),
|
||||
scopeToWorkspace: sandboxRuntime.sandboxed,
|
||||
});
|
||||
} catch {
|
||||
return { prompt: "", skills: [], resolvedSkills: [] };
|
||||
}
|
||||
})();
|
||||
const skillsPrompt = skillsSnapshot.prompt ?? "";
|
||||
const sandboxRuntime = resolveSandboxRuntimeStatus({
|
||||
cfg: params.cfg,
|
||||
sessionKey: params.ctx.SessionKey ?? params.sessionKey,
|
||||
});
|
||||
const tools = (() => {
|
||||
try {
|
||||
return createOpenClawCodingTools({
|
||||
|
||||
@@ -7,6 +7,7 @@ import type {
|
||||
import { logVerbose } from "../../globals.js";
|
||||
import { createInternalHookEvent, triggerInternalHook } from "../../hooks/internal-hooks.js";
|
||||
import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js";
|
||||
import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js";
|
||||
import { resolveSendPolicy } from "../../sessions/send-policy.js";
|
||||
import { shouldHandleTextCommands } from "../commands-registry.js";
|
||||
import { handleAllowlistCommand } from "./commands-allowlist.js";
|
||||
@@ -73,6 +74,26 @@ export async function handleCommands(params: HandleCommandsParams): Promise<Comm
|
||||
return { shouldContinue: false };
|
||||
}
|
||||
|
||||
// Notify plugins the old session ended before starting the new one
|
||||
if (resetRequested && params.command.isAuthorizedSender) {
|
||||
const hookRunner = getGlobalHookRunner();
|
||||
if (hookRunner?.hasHooks("session_end")) {
|
||||
const prevEntry = params.previousSessionEntry;
|
||||
const prevSessionId = prevEntry?.sessionId ?? "";
|
||||
await hookRunner.runSessionEnd(
|
||||
{
|
||||
sessionId: prevSessionId,
|
||||
messageCount: 0, // not tracked at this layer
|
||||
},
|
||||
{
|
||||
agentId: resolveAgentIdFromSessionKey(params.sessionKey),
|
||||
sessionId: prevSessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Trigger internal hook for reset/new commands
|
||||
if (resetRequested && params.command.isAuthorizedSender) {
|
||||
const commandAction = resetMatch?.[1] ?? "new";
|
||||
|
||||
@@ -23,7 +23,7 @@ const matchLevelDirective = (
|
||||
names: string[],
|
||||
): { start: number; end: number; rawLevel?: string } | null => {
|
||||
const namePattern = names.map(escapeRegExp).join("|");
|
||||
const match = body.match(new RegExp(`(?:^|\\s)\\/(?:${namePattern})(?=$|\\s|:)`, "i"));
|
||||
const match = body.match(new RegExp(`(?:^|\\s)[/.](?:${namePattern})(?=$|\\s|:)`, "i"));
|
||||
if (!match || match.index === undefined) {
|
||||
return null;
|
||||
}
|
||||
@@ -79,7 +79,7 @@ const extractSimpleDirective = (
|
||||
): { cleaned: string; hasDirective: boolean } => {
|
||||
const namePattern = names.map(escapeRegExp).join("|");
|
||||
const match = body.match(
|
||||
new RegExp(`(?:^|\\s)\\/(?:${namePattern})(?=$|\\s|:)(?:\\s*:\\s*)?`, "i"),
|
||||
new RegExp(`(?:^|\\s)[/.](?:${namePattern})(?=$|\\s|:)(?:\\s*:\\s*)?`, "i"),
|
||||
);
|
||||
const cleaned = match ? body.replace(match[0], " ").replace(/\s+/g, " ").trim() : body.trim();
|
||||
return {
|
||||
|
||||
@@ -172,7 +172,7 @@ export function extractExecDirective(body?: string): ExecDirectiveParse {
|
||||
invalidNode: false,
|
||||
};
|
||||
}
|
||||
const re = /(?:^|\s)\/exec(?=$|\s|:)/i;
|
||||
const re = /(?:^|\s)[/.]exec(?=$|\s|:)/i;
|
||||
const match = re.exec(body);
|
||||
if (!match) {
|
||||
return {
|
||||
@@ -185,8 +185,10 @@ export function extractExecDirective(body?: string): ExecDirectiveParse {
|
||||
invalidNode: false,
|
||||
};
|
||||
}
|
||||
const start = match.index + match[0].indexOf("/exec");
|
||||
const argsStart = start + "/exec".length;
|
||||
// Find the directive start (handle both /exec and .exec)
|
||||
const execMatch = match[0].match(/[/.]exec/i);
|
||||
const start = match.index + (execMatch ? match[0].indexOf(execMatch[0]) : 0);
|
||||
const argsStart = start + 5; // "/exec" or ".exec" is always 5 chars
|
||||
const parsed = parseExecDirectiveArgs(body.slice(argsStart));
|
||||
const cleanedRaw = `${body.slice(0, start)} ${body.slice(argsStart + parsed.consumed)}`;
|
||||
const cleaned = cleanedRaw.replace(/\s+/g, " ").trim();
|
||||
|
||||
@@ -262,6 +262,23 @@ export async function handleInlineActions(params: {
|
||||
sessionCtx.BodyForAgent = rewrittenBody;
|
||||
sessionCtx.BodyStripped = rewrittenBody;
|
||||
cleanedBody = rewrittenBody;
|
||||
|
||||
// Apply skill-level thinking/model overrides if configured
|
||||
if (skillInvocation.command.thinking) {
|
||||
directives = {
|
||||
...directives,
|
||||
hasThinkDirective: true,
|
||||
thinkLevel: skillInvocation.command.thinking,
|
||||
rawThinkLevel: skillInvocation.command.thinking,
|
||||
};
|
||||
}
|
||||
if (skillInvocation.command.model) {
|
||||
directives = {
|
||||
...directives,
|
||||
hasModelDirective: true,
|
||||
rawModelDirective: skillInvocation.command.model,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const sendInlineReply = async (reply?: ReplyPayload) => {
|
||||
|
||||
@@ -41,6 +41,10 @@ import { runReplyAgent } from "./agent-runner.js";
|
||||
import { applySessionHints } from "./body.js";
|
||||
import { buildGroupChatContext, buildGroupIntro } from "./groups.js";
|
||||
import { buildInboundMetaSystemPrompt, buildInboundUserContextPrefix } from "./inbound-meta.js";
|
||||
import {
|
||||
clearPostCompactionRecovery,
|
||||
prependPostCompactionRecovery,
|
||||
} from "./post-compaction-recovery.js";
|
||||
import { resolveQueueSettings } from "./queue.js";
|
||||
import { routeReply } from "./route-reply.js";
|
||||
import { BARE_SESSION_RESET_PROMPT } from "./session-reset-prompt.js";
|
||||
@@ -256,6 +260,18 @@ export async function runPreparedReply(
|
||||
isNewSession,
|
||||
prefixedBodyBase,
|
||||
});
|
||||
// P3: Prepend post-compaction recovery instructions if the previous turn
|
||||
// triggered auto-compaction. This ensures the agent recalls task state from
|
||||
// memory before responding to the user's next message.
|
||||
prefixedBodyBase = prependPostCompactionRecovery(prefixedBodyBase, sessionEntry);
|
||||
if (sessionEntry?.needsPostCompactionRecovery) {
|
||||
await clearPostCompactionRecovery({
|
||||
sessionEntry,
|
||||
sessionStore,
|
||||
sessionKey,
|
||||
storePath,
|
||||
});
|
||||
}
|
||||
prefixedBodyBase = appendUntrustedContext(prefixedBodyBase, sessionCtx.UntrustedContext);
|
||||
const threadStarterBody = ctx.ThreadStarterBody?.trim();
|
||||
const threadHistoryBody = ctx.ThreadHistoryBody?.trim();
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
resolveAgentSkillsFilter,
|
||||
} from "../../agents/agent-scope.js";
|
||||
import { resolveModelRefFromString } from "../../agents/model-selection.js";
|
||||
import { compactEmbeddedPiSession } from "../../agents/pi-embedded-runner.js";
|
||||
import { resolveAgentTimeoutMs } from "../../agents/timeout.js";
|
||||
import { DEFAULT_AGENT_WORKSPACE_DIR, ensureAgentWorkspace } from "../../agents/workspace.js";
|
||||
import { type OpenClawConfig, loadConfig } from "../../config/config.js";
|
||||
@@ -154,6 +155,7 @@ export async function getReplyFromConfig(
|
||||
sessionId,
|
||||
isNewSession,
|
||||
resetTriggered,
|
||||
compactTriggered,
|
||||
systemSent,
|
||||
abortedLastRun,
|
||||
storePath,
|
||||
@@ -293,6 +295,35 @@ export async function getReplyFromConfig(
|
||||
workspaceDir,
|
||||
});
|
||||
|
||||
// Handle compact trigger - force compaction without resetting session
|
||||
if (compactTriggered && sessionEntry.sessionFile) {
|
||||
try {
|
||||
const compactResult = await compactEmbeddedPiSession({
|
||||
sessionId: sessionEntry.sessionId,
|
||||
sessionFile: sessionEntry.sessionFile,
|
||||
config: cfg,
|
||||
workspaceDir,
|
||||
provider,
|
||||
model,
|
||||
});
|
||||
if (compactResult.compacted && compactResult.result) {
|
||||
const tokensBefore = compactResult.result.tokensBefore;
|
||||
const tokensAfter = compactResult.result.tokensAfter ?? 0;
|
||||
return {
|
||||
text: `✅ Context compacted successfully.\n\n**Before:** ${tokensBefore.toLocaleString()} tokens\n**After:** ${tokensAfter.toLocaleString()} tokens\n**Saved:** ${(tokensBefore - tokensAfter).toLocaleString()} tokens`,
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
text: `ℹ️ Nothing to compact. ${compactResult.reason ?? "Session is already compact."}`,
|
||||
};
|
||||
}
|
||||
} catch (err) {
|
||||
return {
|
||||
text: `❌ Compaction failed: ${String(err)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return runPreparedReply({
|
||||
ctx,
|
||||
sessionCtx,
|
||||
|
||||
@@ -12,12 +12,14 @@ export const DEFAULT_MEMORY_FLUSH_PROMPT = [
|
||||
"Pre-compaction memory flush.",
|
||||
"Store durable memories now (use memory/YYYY-MM-DD.md; create memory/ if needed).",
|
||||
"IMPORTANT: If the file already exists, APPEND new content only and do not overwrite existing entries.",
|
||||
"If there is an active task in progress, save its state: task name, current step, pending actions, and any critical variables. Use memory_store with category 'core' and importance 1.0 for active task state.",
|
||||
`If nothing to store, reply with ${SILENT_REPLY_TOKEN}.`,
|
||||
].join(" ");
|
||||
|
||||
export const DEFAULT_MEMORY_FLUSH_SYSTEM_PROMPT = [
|
||||
"Pre-compaction memory flush turn.",
|
||||
"The session is near auto-compaction; capture durable memories to disk.",
|
||||
"CRITICAL: If there is an active task being worked on, you MUST save its current state (task name, step, pending actions, key variables) to memory_store with category='core' and importance=1.0. This ensures task continuity after compaction.",
|
||||
`You may reply, but usually ${SILENT_REPLY_TOKEN} is correct.`,
|
||||
].join(" ");
|
||||
|
||||
|
||||
102
src/auto-reply/reply/post-compaction-recovery.test.ts
Normal file
102
src/auto-reply/reply/post-compaction-recovery.test.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
getPostCompactionRecoveryPrompt,
|
||||
POST_COMPACTION_RECOVERY_PROMPT,
|
||||
prependPostCompactionRecovery,
|
||||
} from "./post-compaction-recovery.js";
|
||||
|
||||
describe("post-compaction-recovery", () => {
|
||||
describe("POST_COMPACTION_RECOVERY_PROMPT", () => {
|
||||
it("is defined and non-empty", () => {
|
||||
expect(POST_COMPACTION_RECOVERY_PROMPT).toBeTruthy();
|
||||
expect(POST_COMPACTION_RECOVERY_PROMPT.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("stays under 200 tokens (rough estimate: <800 chars)", () => {
|
||||
// A rough heuristic: 1 token ≈ 4 chars. 200 tokens ≈ 800 chars.
|
||||
expect(POST_COMPACTION_RECOVERY_PROMPT.length).toBeLessThan(800);
|
||||
});
|
||||
|
||||
it("includes memory_recall instruction", () => {
|
||||
expect(POST_COMPACTION_RECOVERY_PROMPT).toContain("memory_recall");
|
||||
});
|
||||
|
||||
it("includes TASKS.md instruction", () => {
|
||||
expect(POST_COMPACTION_RECOVERY_PROMPT).toContain("TASKS.md");
|
||||
});
|
||||
|
||||
it("includes Context Reset notification template", () => {
|
||||
expect(POST_COMPACTION_RECOVERY_PROMPT).toContain("Context Reset");
|
||||
});
|
||||
});
|
||||
|
||||
describe("getPostCompactionRecoveryPrompt", () => {
|
||||
it("returns null when entry is undefined", () => {
|
||||
expect(getPostCompactionRecoveryPrompt(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null when needsPostCompactionRecovery is false", () => {
|
||||
const entry = {
|
||||
sessionId: "test",
|
||||
updatedAt: Date.now(),
|
||||
needsPostCompactionRecovery: false,
|
||||
};
|
||||
expect(getPostCompactionRecoveryPrompt(entry)).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null when needsPostCompactionRecovery is not set", () => {
|
||||
const entry = { sessionId: "test", updatedAt: Date.now() };
|
||||
expect(getPostCompactionRecoveryPrompt(entry)).toBeNull();
|
||||
});
|
||||
|
||||
it("returns the recovery prompt when needsPostCompactionRecovery is true", () => {
|
||||
const entry = {
|
||||
sessionId: "test",
|
||||
updatedAt: Date.now(),
|
||||
needsPostCompactionRecovery: true,
|
||||
};
|
||||
expect(getPostCompactionRecoveryPrompt(entry)).toBe(POST_COMPACTION_RECOVERY_PROMPT);
|
||||
});
|
||||
});
|
||||
|
||||
describe("prependPostCompactionRecovery", () => {
|
||||
it("returns original body when no recovery needed", () => {
|
||||
const body = "Hello, how are you?";
|
||||
expect(prependPostCompactionRecovery(body, undefined)).toBe(body);
|
||||
});
|
||||
|
||||
it("returns original body when flag is false", () => {
|
||||
const body = "Hello, how are you?";
|
||||
const entry = {
|
||||
sessionId: "test",
|
||||
updatedAt: Date.now(),
|
||||
needsPostCompactionRecovery: false,
|
||||
};
|
||||
expect(prependPostCompactionRecovery(body, entry)).toBe(body);
|
||||
});
|
||||
|
||||
it("prepends recovery prompt when flag is true", () => {
|
||||
const body = "Hello, how are you?";
|
||||
const entry = {
|
||||
sessionId: "test",
|
||||
updatedAt: Date.now(),
|
||||
needsPostCompactionRecovery: true,
|
||||
};
|
||||
const result = prependPostCompactionRecovery(body, entry);
|
||||
expect(result).toContain(POST_COMPACTION_RECOVERY_PROMPT);
|
||||
expect(result).toContain(body);
|
||||
expect(result.indexOf(POST_COMPACTION_RECOVERY_PROMPT)).toBeLessThan(result.indexOf(body));
|
||||
});
|
||||
|
||||
it("separates recovery prompt from body with double newline", () => {
|
||||
const body = "test message";
|
||||
const entry = {
|
||||
sessionId: "test",
|
||||
updatedAt: Date.now(),
|
||||
needsPostCompactionRecovery: true,
|
||||
};
|
||||
const result = prependPostCompactionRecovery(body, entry);
|
||||
expect(result).toBe(`${POST_COMPACTION_RECOVERY_PROMPT}\n\n${body}`);
|
||||
});
|
||||
});
|
||||
});
|
||||
103
src/auto-reply/reply/post-compaction-recovery.ts
Normal file
103
src/auto-reply/reply/post-compaction-recovery.ts
Normal file
@@ -0,0 +1,103 @@
|
||||
import type { SessionEntry } from "../../config/sessions.js";
|
||||
import { updateSessionStore } from "../../config/sessions.js";
|
||||
|
||||
/**
|
||||
* Post-compaction recovery prompt injected into the next user message after
|
||||
* auto-compaction completes. Instructs the agent to recall task state from
|
||||
* memory and notify the user about the context reset.
|
||||
*
|
||||
* Kept under 200 tokens to minimize context overhead.
|
||||
*/
|
||||
export const POST_COMPACTION_RECOVERY_PROMPT = [
|
||||
"[Post-compaction recovery — mandatory steps]",
|
||||
"Context was just compacted. Before responding, you MUST:",
|
||||
'1. Run memory_recall("active task") to check for saved task state.',
|
||||
"2. Read TASKS.md if it exists in your workspace.",
|
||||
"3. Compare recovered state against the compaction summary above.",
|
||||
'4. Notify the user: "🔄 Context Reset — last task: [X], resuming from step [Y]" (or summarize what you recall).',
|
||||
"Do NOT skip these steps. Proceed with the user's message after recovery.",
|
||||
].join("\n");
|
||||
|
||||
/**
|
||||
* Check whether the session needs post-compaction recovery and return the
|
||||
* recovery prompt if so. Returns `null` when no recovery is needed.
|
||||
*/
|
||||
export function getPostCompactionRecoveryPrompt(entry?: SessionEntry): string | null {
|
||||
if (!entry?.needsPostCompactionRecovery) {
|
||||
return null;
|
||||
}
|
||||
return POST_COMPACTION_RECOVERY_PROMPT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepend the post-compaction recovery prompt to the user's message body.
|
||||
* Returns the original body unchanged if no recovery is needed.
|
||||
*/
|
||||
export function prependPostCompactionRecovery(body: string, entry?: SessionEntry): string {
|
||||
const prompt = getPostCompactionRecoveryPrompt(entry);
|
||||
if (!prompt) {
|
||||
return body;
|
||||
}
|
||||
return `${prompt}\n\n${body}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set or clear the post-compaction recovery flag on a session.
|
||||
*/
|
||||
async function setPostCompactionRecovery(
|
||||
value: boolean,
|
||||
params: {
|
||||
sessionEntry?: SessionEntry;
|
||||
sessionStore?: Record<string, SessionEntry>;
|
||||
sessionKey?: string;
|
||||
storePath?: string;
|
||||
},
|
||||
): Promise<void> {
|
||||
const { sessionEntry, sessionStore, sessionKey, storePath } = params;
|
||||
if (!sessionStore || !sessionKey) {
|
||||
return;
|
||||
}
|
||||
const entry = sessionStore[sessionKey] ?? sessionEntry;
|
||||
if (!entry) {
|
||||
return;
|
||||
}
|
||||
sessionStore[sessionKey] = {
|
||||
...entry,
|
||||
needsPostCompactionRecovery: value,
|
||||
};
|
||||
if (storePath) {
|
||||
await updateSessionStore(storePath, (store) => {
|
||||
if (store[sessionKey]) {
|
||||
store[sessionKey] = {
|
||||
...store[sessionKey],
|
||||
needsPostCompactionRecovery: value,
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a session as needing post-compaction recovery on the next turn.
|
||||
*/
|
||||
export async function markNeedsPostCompactionRecovery(params: {
|
||||
sessionEntry?: SessionEntry;
|
||||
sessionStore?: Record<string, SessionEntry>;
|
||||
sessionKey?: string;
|
||||
storePath?: string;
|
||||
}): Promise<void> {
|
||||
return setPostCompactionRecovery(true, params);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the post-compaction recovery flag after recovery instructions have
|
||||
* been injected into the prompt.
|
||||
*/
|
||||
export async function clearPostCompactionRecovery(params: {
|
||||
sessionEntry?: SessionEntry;
|
||||
sessionStore?: Record<string, SessionEntry>;
|
||||
sessionKey?: string;
|
||||
storePath?: string;
|
||||
}): Promise<void> {
|
||||
return setPostCompactionRecovery(false, params);
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import crypto from "node:crypto";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import { resolveUserTimezone } from "../../agents/date-time.js";
|
||||
import { resolveSandboxRuntimeStatus } from "../../agents/sandbox/runtime-status.js";
|
||||
import { buildWorkspaceSkillSnapshot } from "../../agents/skills.js";
|
||||
import { ensureSkillsWatcher, getSkillsSnapshotVersion } from "../../agents/skills/refresh.js";
|
||||
import { type SessionEntry, updateSessionStore } from "../../config/sessions.js";
|
||||
@@ -149,6 +150,10 @@ export async function ensureSkillSnapshot(params: {
|
||||
skillFilter,
|
||||
} = params;
|
||||
|
||||
// Sandboxed agents should only see workspace skills — managed/bundled skill
|
||||
// paths are outside the sandbox root and would be blocked by path assertions.
|
||||
const sandboxed = sessionKey ? resolveSandboxRuntimeStatus({ cfg, sessionKey }).sandboxed : false;
|
||||
|
||||
let nextEntry = sessionEntry;
|
||||
let systemSent = sessionEntry?.systemSent ?? false;
|
||||
const remoteEligibility = getRemoteSkillEligibility();
|
||||
@@ -170,6 +175,7 @@ export async function ensureSkillSnapshot(params: {
|
||||
skillFilter,
|
||||
eligibility: { remote: remoteEligibility },
|
||||
snapshotVersion,
|
||||
scopeToWorkspace: sandboxed,
|
||||
})
|
||||
: current.skillsSnapshot;
|
||||
nextEntry = {
|
||||
@@ -194,6 +200,7 @@ export async function ensureSkillSnapshot(params: {
|
||||
skillFilter,
|
||||
eligibility: { remote: remoteEligibility },
|
||||
snapshotVersion,
|
||||
scopeToWorkspace: sandboxed,
|
||||
})
|
||||
: (nextEntry?.skillsSnapshot ??
|
||||
(isFirstTurnInSession
|
||||
@@ -203,6 +210,7 @@ export async function ensureSkillSnapshot(params: {
|
||||
skillFilter,
|
||||
eligibility: { remote: remoteEligibility },
|
||||
snapshotVersion,
|
||||
scopeToWorkspace: sandboxed,
|
||||
})));
|
||||
if (
|
||||
skillsSnapshot &&
|
||||
|
||||
@@ -44,6 +44,7 @@ export type SessionInitResult = {
|
||||
sessionId: string;
|
||||
isNewSession: boolean;
|
||||
resetTriggered: boolean;
|
||||
compactTriggered: boolean;
|
||||
systemSent: boolean;
|
||||
abortedLastRun: boolean;
|
||||
storePath: string;
|
||||
@@ -133,6 +134,7 @@ export async function initSessionState(params: {
|
||||
let systemSent = false;
|
||||
let abortedLastRun = false;
|
||||
let resetTriggered = false;
|
||||
let compactTriggered = false;
|
||||
|
||||
let persistedThinking: string | undefined;
|
||||
let persistedVerbose: string | undefined;
|
||||
@@ -198,6 +200,22 @@ export async function initSessionState(params: {
|
||||
}
|
||||
}
|
||||
|
||||
// Check for compact triggers (e.g., ".compact", "/compact")
|
||||
const compactTriggers = sessionCfg?.compactTriggers ?? [];
|
||||
if (!resetTriggered && resetAuthorized) {
|
||||
for (const trigger of compactTriggers) {
|
||||
if (!trigger) {
|
||||
continue;
|
||||
}
|
||||
const triggerLower = trigger.toLowerCase();
|
||||
if (trimmedBodyLower === triggerLower || strippedForResetLower === triggerLower) {
|
||||
compactTriggered = true;
|
||||
bodyStripped = "";
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sessionKey = resolveSessionKey(sessionScope, sessionCtxForState, mainKey);
|
||||
const entry = sessionStore[sessionKey];
|
||||
const previousSessionEntry = resetTriggered && entry ? { ...entry } : undefined;
|
||||
@@ -458,6 +476,7 @@ export async function initSessionState(params: {
|
||||
sessionId: sessionId ?? crypto.randomUUID(),
|
||||
isNewSession,
|
||||
resetTriggered,
|
||||
compactTriggered,
|
||||
systemSent,
|
||||
abortedLastRun,
|
||||
storePath,
|
||||
|
||||
@@ -100,8 +100,9 @@ const MAX_CONSOLE_MESSAGES = 500;
|
||||
const MAX_PAGE_ERRORS = 200;
|
||||
const MAX_NETWORK_REQUESTS = 500;
|
||||
|
||||
let cached: ConnectedBrowser | null = null;
|
||||
let connecting: Promise<ConnectedBrowser> | null = null;
|
||||
// Per-profile caching to allow parallel connections to different Chrome instances
|
||||
const cachedByUrl = new Map<string, ConnectedBrowser>();
|
||||
const connectingByUrl = new Map<string, Promise<ConnectedBrowser>>();
|
||||
|
||||
function normalizeCdpUrl(raw: string) {
|
||||
return raw.replace(/\/$/, "");
|
||||
@@ -315,11 +316,17 @@ function observeBrowser(browser: Browser) {
|
||||
|
||||
async function connectBrowser(cdpUrl: string): Promise<ConnectedBrowser> {
|
||||
const normalized = normalizeCdpUrl(cdpUrl);
|
||||
if (cached?.cdpUrl === normalized) {
|
||||
|
||||
// Check if we already have a cached connection for this specific URL
|
||||
const cached = cachedByUrl.get(normalized);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
if (connecting) {
|
||||
return await connecting;
|
||||
|
||||
// Check if there's already a connection in progress for this specific URL
|
||||
const existingConnecting = connectingByUrl.get(normalized);
|
||||
if (existingConnecting) {
|
||||
return await existingConnecting;
|
||||
}
|
||||
|
||||
const connectWithRetry = async (): Promise<ConnectedBrowser> => {
|
||||
@@ -332,12 +339,12 @@ async function connectBrowser(cdpUrl: string): Promise<ConnectedBrowser> {
|
||||
const headers = getHeadersWithAuth(endpoint);
|
||||
const browser = await chromium.connectOverCDP(endpoint, { timeout, headers });
|
||||
const onDisconnected = () => {
|
||||
if (cached?.browser === browser) {
|
||||
cached = null;
|
||||
if (cachedByUrl.get(normalized)?.browser === browser) {
|
||||
cachedByUrl.delete(normalized);
|
||||
}
|
||||
};
|
||||
const connected: ConnectedBrowser = { browser, cdpUrl: normalized, onDisconnected };
|
||||
cached = connected;
|
||||
cachedByUrl.set(normalized, connected);
|
||||
browser.on("disconnected", onDisconnected);
|
||||
observeBrowser(browser);
|
||||
return connected;
|
||||
@@ -354,11 +361,12 @@ async function connectBrowser(cdpUrl: string): Promise<ConnectedBrowser> {
|
||||
throw new Error(message);
|
||||
};
|
||||
|
||||
connecting = connectWithRetry().finally(() => {
|
||||
connecting = null;
|
||||
const connectingPromise = connectWithRetry().finally(() => {
|
||||
connectingByUrl.delete(normalized);
|
||||
});
|
||||
connectingByUrl.set(normalized, connectingPromise);
|
||||
|
||||
return await connecting;
|
||||
return await connectingPromise;
|
||||
}
|
||||
|
||||
async function getAllPages(browser: Browser): Promise<Page[]> {
|
||||
@@ -512,16 +520,16 @@ export function refLocator(page: Page, ref: string) {
|
||||
}
|
||||
|
||||
export async function closePlaywrightBrowserConnection(): Promise<void> {
|
||||
const cur = cached;
|
||||
cached = null;
|
||||
connecting = null;
|
||||
if (!cur) {
|
||||
return;
|
||||
// Close all cached browser connections
|
||||
const connections = Array.from(cachedByUrl.values());
|
||||
cachedByUrl.clear();
|
||||
connectingByUrl.clear();
|
||||
for (const c of connections) {
|
||||
if (c.onDisconnected && typeof c.browser.off === "function") {
|
||||
c.browser.off("disconnected", c.onDisconnected);
|
||||
}
|
||||
}
|
||||
if (cur.onDisconnected && typeof cur.browser.off === "function") {
|
||||
cur.browser.off("disconnected", cur.onDisconnected);
|
||||
}
|
||||
await cur.browser.close().catch(() => {});
|
||||
await Promise.all(connections.map((c) => c.browser.close().catch(() => {})));
|
||||
}
|
||||
|
||||
function normalizeCdpHttpBaseForJsonEndpoints(cdpUrl: string): string {
|
||||
@@ -649,31 +657,30 @@ export async function forceDisconnectPlaywrightForTarget(opts: {
|
||||
reason?: string;
|
||||
}): Promise<void> {
|
||||
const normalized = normalizeCdpUrl(opts.cdpUrl);
|
||||
if (cached?.cdpUrl !== normalized) {
|
||||
const cur = cachedByUrl.get(normalized);
|
||||
if (!cur) {
|
||||
return;
|
||||
}
|
||||
const cur = cached;
|
||||
cached = null;
|
||||
// Also clear `connecting` so the next call does a fresh connectOverCDP
|
||||
cachedByUrl.delete(normalized);
|
||||
// Also clear the connecting promise so the next call does a fresh connectOverCDP
|
||||
// rather than awaiting a stale promise.
|
||||
connecting = null;
|
||||
if (cur) {
|
||||
// Remove the "disconnected" listener to prevent the old browser's teardown
|
||||
// from racing with a fresh connection and nulling the new `cached`.
|
||||
if (cur.onDisconnected && typeof cur.browser.off === "function") {
|
||||
cur.browser.off("disconnected", cur.onDisconnected);
|
||||
}
|
||||
connectingByUrl.delete(normalized);
|
||||
|
||||
// Best-effort: kill any stuck JS to unblock the target's execution context before we
|
||||
// disconnect Playwright's CDP connection.
|
||||
const targetId = opts.targetId?.trim() || "";
|
||||
if (targetId) {
|
||||
await tryTerminateExecutionViaCdp({ cdpUrl: normalized, targetId }).catch(() => {});
|
||||
}
|
||||
|
||||
// Fire-and-forget: don't await because browser.close() may hang on the stuck CDP pipe.
|
||||
cur.browser.close().catch(() => {});
|
||||
// Remove the "disconnected" listener to prevent the old browser's teardown
|
||||
// from racing with a fresh connection and clearing the new cached entry.
|
||||
if (cur.onDisconnected && typeof cur.browser.off === "function") {
|
||||
cur.browser.off("disconnected", cur.onDisconnected);
|
||||
}
|
||||
|
||||
// Best-effort: kill any stuck JS to unblock the target's execution context before we
|
||||
// disconnect Playwright's CDP connection.
|
||||
const targetId = opts.targetId?.trim() || "";
|
||||
if (targetId) {
|
||||
await tryTerminateExecutionViaCdp({ cdpUrl: normalized, targetId }).catch(() => {});
|
||||
}
|
||||
|
||||
// Fire-and-forget: don't await because browser.close() may hang on the stuck CDP pipe.
|
||||
cur.browser.close().catch(() => {});
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { resolveCommitHash } from "../infra/git-commit.js";
|
||||
import { resolveCommitHash, resolveUpstreamCommitHash } from "../infra/git-commit.js";
|
||||
import { visibleWidth } from "../terminal/ansi.js";
|
||||
import { isRich, theme } from "../terminal/theme.js";
|
||||
import { pickTagline, type TaglineOptions } from "./tagline.js";
|
||||
@@ -6,6 +6,7 @@ import { pickTagline, type TaglineOptions } from "./tagline.js";
|
||||
type BannerOptions = TaglineOptions & {
|
||||
argv?: string[];
|
||||
commit?: string | null;
|
||||
upstreamCommit?: string | null;
|
||||
columns?: number;
|
||||
richTty?: boolean;
|
||||
};
|
||||
@@ -36,30 +37,33 @@ const hasVersionFlag = (argv: string[]) =>
|
||||
|
||||
export function formatCliBannerLine(version: string, options: BannerOptions = {}): string {
|
||||
const commit = options.commit ?? resolveCommitHash({ env: options.env });
|
||||
const upstreamCommit = options.upstreamCommit ?? resolveUpstreamCommitHash();
|
||||
const commitLabel = commit ?? "unknown";
|
||||
// Show upstream if different from current (indicates local commits ahead)
|
||||
const showUpstream = upstreamCommit && upstreamCommit !== commit;
|
||||
const commitDisplay = showUpstream ? `${commitLabel} ← ${upstreamCommit}` : commitLabel;
|
||||
const tagline = pickTagline(options);
|
||||
const rich = options.richTty ?? isRich();
|
||||
const title = "🦞 OpenClaw";
|
||||
const prefix = "🦞 ";
|
||||
const columns = options.columns ?? process.stdout.columns ?? 120;
|
||||
const plainFullLine = `${title} ${version} (${commitLabel}) — ${tagline}`;
|
||||
const plainFullLine = `${title} ${version} (${commitDisplay}) — ${tagline}`;
|
||||
const fitsOnOneLine = visibleWidth(plainFullLine) <= columns;
|
||||
if (rich) {
|
||||
const commitPart = showUpstream
|
||||
? `${theme.muted("(")}${commitLabel}${theme.muted(" ← ")}${theme.muted(upstreamCommit)}${theme.muted(")")}`
|
||||
: theme.muted(`(${commitLabel})`);
|
||||
if (fitsOnOneLine) {
|
||||
return `${theme.heading(title)} ${theme.info(version)} ${theme.muted(
|
||||
`(${commitLabel})`,
|
||||
)} ${theme.muted("—")} ${theme.accentDim(tagline)}`;
|
||||
return `${theme.heading(title)} ${theme.info(version)} ${commitPart} ${theme.muted("—")} ${theme.accentDim(tagline)}`;
|
||||
}
|
||||
const line1 = `${theme.heading(title)} ${theme.info(version)} ${theme.muted(
|
||||
`(${commitLabel})`,
|
||||
)}`;
|
||||
const line1 = `${theme.heading(title)} ${theme.info(version)} ${commitPart}`;
|
||||
const line2 = `${" ".repeat(prefix.length)}${theme.accentDim(tagline)}`;
|
||||
return `${line1}\n${line2}`;
|
||||
}
|
||||
if (fitsOnOneLine) {
|
||||
return plainFullLine;
|
||||
}
|
||||
const line1 = `${title} ${version} (${commitLabel})`;
|
||||
const line1 = `${title} ${version} (${commitDisplay})`;
|
||||
const line2 = `${" ".repeat(prefix.length)}${tagline}`;
|
||||
return `${line1}\n${line2}`;
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@ describe("logs cli", () => {
|
||||
expect(stderrWrites.join("")).toContain("Log cursor reset");
|
||||
});
|
||||
|
||||
it("wires --local-time through CLI parsing and emits local timestamps", async () => {
|
||||
it("emits local timestamps in plain mode", async () => {
|
||||
callGatewayFromCli.mockResolvedValueOnce({
|
||||
file: "/tmp/openclaw.log",
|
||||
lines: [
|
||||
@@ -77,15 +77,17 @@ describe("logs cli", () => {
|
||||
program.exitOverride();
|
||||
registerLogsCli(program);
|
||||
|
||||
await program.parseAsync(["logs", "--local-time", "--plain"], { from: "user" });
|
||||
await program.parseAsync(["logs", "--plain"], { from: "user" });
|
||||
|
||||
stdoutSpy.mockRestore();
|
||||
|
||||
const output = stdoutWrites.join("");
|
||||
expect(output).toContain("line one");
|
||||
const timestamp = output.match(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z?/u)?.[0];
|
||||
// Timestamps should be local ISO format (no trailing Z)
|
||||
const timestamp = output.match(
|
||||
/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}[+-]\d{2}:\d{2}/u,
|
||||
)?.[0];
|
||||
expect(timestamp).toBeTruthy();
|
||||
expect(timestamp?.endsWith("Z")).toBe(false);
|
||||
});
|
||||
|
||||
it("warns when the output pipe closes", async () => {
|
||||
@@ -119,35 +121,16 @@ describe("logs cli", () => {
|
||||
});
|
||||
|
||||
describe("formatLogTimestamp", () => {
|
||||
it("formats UTC timestamp in plain mode by default", () => {
|
||||
it("formats timestamp in local ISO format in plain mode", () => {
|
||||
const result = formatLogTimestamp("2025-01-01T12:00:00.000Z");
|
||||
expect(result).toBe("2025-01-01T12:00:00.000Z");
|
||||
});
|
||||
|
||||
it("formats UTC timestamp in pretty mode", () => {
|
||||
const result = formatLogTimestamp("2025-01-01T12:00:00.000Z", "pretty");
|
||||
expect(result).toBe("12:00:00");
|
||||
});
|
||||
|
||||
it("formats local time in plain mode when localTime is true", () => {
|
||||
const utcTime = "2025-01-01T12:00:00.000Z";
|
||||
const result = formatLogTimestamp(utcTime, "plain", true);
|
||||
// Should be local time with explicit timezone offset (not 'Z' suffix).
|
||||
// Should be local ISO time with timezone offset, no trailing Z
|
||||
expect(result).not.toContain("Z");
|
||||
expect(result).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}[+-]\d{2}:\d{2}$/);
|
||||
// The exact time depends on timezone, but should be different from UTC
|
||||
expect(result).not.toBe(utcTime);
|
||||
});
|
||||
|
||||
it("formats local time in pretty mode when localTime is true", () => {
|
||||
const utcTime = "2025-01-01T12:00:00.000Z";
|
||||
const result = formatLogTimestamp(utcTime, "pretty", true);
|
||||
// Should be HH:MM:SS format
|
||||
it("formats timestamp in local HH:MM:SS in pretty mode", () => {
|
||||
const result = formatLogTimestamp("2025-01-01T12:00:00.000Z", "pretty");
|
||||
expect(result).toMatch(/^\d{2}:\d{2}:\d{2}$/);
|
||||
// Should be different from UTC time (12:00:00) if not in UTC timezone
|
||||
const tzOffset = new Date(utcTime).getTimezoneOffset();
|
||||
if (tzOffset !== 0) {
|
||||
expect(result).not.toBe("12:00:00");
|
||||
}
|
||||
});
|
||||
|
||||
it("handles empty or invalid timestamps", () => {
|
||||
|
||||
@@ -2,7 +2,7 @@ import type { Command } from "commander";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { buildGatewayConnectionDetails } from "../gateway/call.js";
|
||||
import { parseLogLine } from "../logging/parse-log-line.js";
|
||||
import { formatLocalIsoWithOffset } from "../logging/timestamps.js";
|
||||
import { formatLocalIso } from "../logging/timestamp.js";
|
||||
import { formatDocsLink } from "../terminal/links.js";
|
||||
import { clearActiveProgressLine } from "../terminal/progress-line.js";
|
||||
import { createSafeStreamWriter } from "../terminal/stream-writer.js";
|
||||
@@ -27,7 +27,6 @@ type LogsCliOptions = {
|
||||
json?: boolean;
|
||||
plain?: boolean;
|
||||
color?: boolean;
|
||||
localTime?: boolean;
|
||||
url?: string;
|
||||
token?: string;
|
||||
timeout?: string;
|
||||
@@ -61,11 +60,7 @@ async function fetchLogs(
|
||||
return payload as LogsTailPayload;
|
||||
}
|
||||
|
||||
export function formatLogTimestamp(
|
||||
value?: string,
|
||||
mode: "pretty" | "plain" = "plain",
|
||||
localTime = false,
|
||||
) {
|
||||
export function formatLogTimestamp(value?: string, mode: "pretty" | "plain" = "plain") {
|
||||
if (!value) {
|
||||
return "";
|
||||
}
|
||||
@@ -73,17 +68,13 @@ export function formatLogTimestamp(
|
||||
if (Number.isNaN(parsed.getTime())) {
|
||||
return value;
|
||||
}
|
||||
|
||||
let timeString: string;
|
||||
if (localTime) {
|
||||
timeString = formatLocalIsoWithOffset(parsed);
|
||||
} else {
|
||||
timeString = parsed.toISOString();
|
||||
}
|
||||
if (mode === "pretty") {
|
||||
return timeString.slice(11, 19);
|
||||
const h = String(parsed.getHours()).padStart(2, "0");
|
||||
const m = String(parsed.getMinutes()).padStart(2, "0");
|
||||
const s = String(parsed.getSeconds()).padStart(2, "0");
|
||||
return `${h}:${m}:${s}`;
|
||||
}
|
||||
return timeString;
|
||||
return formatLocalIso(parsed);
|
||||
}
|
||||
|
||||
function formatLogLine(
|
||||
@@ -91,7 +82,6 @@ function formatLogLine(
|
||||
opts: {
|
||||
pretty: boolean;
|
||||
rich: boolean;
|
||||
localTime: boolean;
|
||||
},
|
||||
): string {
|
||||
const parsed = parseLogLine(raw);
|
||||
@@ -99,7 +89,7 @@ function formatLogLine(
|
||||
return raw;
|
||||
}
|
||||
const label = parsed.subsystem ?? parsed.module ?? "";
|
||||
const time = formatLogTimestamp(parsed.time, opts.pretty ? "pretty" : "plain", opts.localTime);
|
||||
const time = formatLogTimestamp(parsed.time, opts.pretty ? "pretty" : "plain");
|
||||
const level = parsed.level ?? "";
|
||||
const levelLabel = level.padEnd(5).trim();
|
||||
const message = parsed.message || parsed.raw;
|
||||
@@ -206,7 +196,6 @@ export function registerLogsCli(program: Command) {
|
||||
.option("--json", "Emit JSON log lines", false)
|
||||
.option("--plain", "Plain text output (no ANSI styling)", false)
|
||||
.option("--no-color", "Disable ANSI colors")
|
||||
.option("--local-time", "Display timestamps in local timezone", false)
|
||||
.addHelpText(
|
||||
"after",
|
||||
() =>
|
||||
@@ -223,7 +212,6 @@ export function registerLogsCli(program: Command) {
|
||||
const jsonMode = Boolean(opts.json);
|
||||
const pretty = !jsonMode && Boolean(process.stdout.isTTY) && !opts.plain;
|
||||
const rich = isRich() && opts.color !== false;
|
||||
const localTime = Boolean(opts.localTime);
|
||||
|
||||
while (true) {
|
||||
let payload: LogsTailPayload;
|
||||
@@ -295,7 +283,6 @@ export function registerLogsCli(program: Command) {
|
||||
formatLogLine(line, {
|
||||
pretty,
|
||||
rich,
|
||||
localTime,
|
||||
}),
|
||||
)
|
||||
) {
|
||||
|
||||
@@ -280,11 +280,14 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) {
|
||||
scan?: MemorySourceScan;
|
||||
}> = [];
|
||||
|
||||
const disabledAgentIds: string[] = [];
|
||||
for (const agentId of agentIds) {
|
||||
const managerPurpose = opts.index ? "default" : "status";
|
||||
await withManager<MemoryManager>({
|
||||
getManager: () => getMemorySearchManager({ cfg, agentId, purpose: managerPurpose }),
|
||||
onMissing: (error) => defaultRuntime.log(error ?? "Memory search disabled."),
|
||||
onMissing: () => {
|
||||
disabledAgentIds.push(agentId);
|
||||
},
|
||||
onCloseError: (err) =>
|
||||
defaultRuntime.error(`Memory manager close failed: ${formatErrorMessage(err)}`),
|
||||
close: async (manager) => {
|
||||
@@ -374,11 +377,31 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) {
|
||||
const accent = (text: string) => colorize(rich, theme.accent, text);
|
||||
const label = (text: string) => muted(`${text}:`);
|
||||
|
||||
const emptyAgentIds: string[] = [];
|
||||
for (const result of allResults) {
|
||||
const { agentId, status, embeddingProbe, indexError, scan } = result;
|
||||
const filesIndexed = status.files ?? 0;
|
||||
const chunksIndexed = status.chunks ?? 0;
|
||||
const totalFiles = scan?.totalFiles ?? null;
|
||||
|
||||
const hasDiagnostics =
|
||||
status.dirty ||
|
||||
Boolean(status.fallback) ||
|
||||
Boolean(status.vector?.loadError) ||
|
||||
(status.vector?.enabled === true && status.vector.available === false) ||
|
||||
Boolean(status.fts?.error);
|
||||
// Skip agents with no indexed content only when there are no relevant status diagnostics.
|
||||
const isEmpty =
|
||||
status.files === 0 &&
|
||||
status.chunks === 0 &&
|
||||
(totalFiles ?? 0) === 0 &&
|
||||
!indexError &&
|
||||
!hasDiagnostics;
|
||||
if (isEmpty) {
|
||||
emptyAgentIds.push(agentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
const indexedLabel =
|
||||
totalFiles === null
|
||||
? `${filesIndexed}/? files · ${chunksIndexed} chunks`
|
||||
@@ -510,6 +533,78 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) {
|
||||
defaultRuntime.log(lines.join("\n"));
|
||||
defaultRuntime.log("");
|
||||
}
|
||||
|
||||
// Show compact summary for agents with no indexed memory-search content
|
||||
if (emptyAgentIds.length > 0) {
|
||||
const agentList = emptyAgentIds.join(", ");
|
||||
defaultRuntime.log(
|
||||
muted(
|
||||
`Memory Search: ${emptyAgentIds.length} agent${emptyAgentIds.length > 1 ? "s" : ""} with no indexed files (${agentList})`,
|
||||
),
|
||||
);
|
||||
defaultRuntime.log("");
|
||||
}
|
||||
|
||||
// Show compact summary for agents with memory search disabled
|
||||
if (disabledAgentIds.length > 0 && emptyAgentIds.length === 0) {
|
||||
const agentList = disabledAgentIds.join(", ");
|
||||
defaultRuntime.log(
|
||||
muted(
|
||||
`Memory Search: disabled for ${disabledAgentIds.length} agent${disabledAgentIds.length > 1 ? "s" : ""} (${agentList})`,
|
||||
),
|
||||
);
|
||||
defaultRuntime.log("");
|
||||
}
|
||||
|
||||
// Detect configured memory plugins and show hints
|
||||
const memoryPlugins = detectMemoryPlugins(cfg);
|
||||
if (memoryPlugins.length > 0) {
|
||||
defaultRuntime.log(heading("Memory Plugins"));
|
||||
for (const plugin of memoryPlugins) {
|
||||
const statusText = plugin.enabled ? success("enabled") : muted("disabled");
|
||||
defaultRuntime.log(`${info(plugin.id)} ${statusText}`);
|
||||
if (plugin.hint) {
|
||||
defaultRuntime.log(muted(` → ${plugin.hint}`));
|
||||
}
|
||||
}
|
||||
defaultRuntime.log("");
|
||||
}
|
||||
}
|
||||
|
||||
type MemoryPluginInfo = {
|
||||
id: string;
|
||||
enabled: boolean;
|
||||
hint?: string;
|
||||
};
|
||||
|
||||
function detectMemoryPlugins(cfg: ReturnType<typeof loadConfig>): MemoryPluginInfo[] {
|
||||
const plugins: MemoryPluginInfo[] = [];
|
||||
const entries = cfg.plugins?.entries ?? {};
|
||||
const activeSlot = cfg.plugins?.slots?.memory;
|
||||
|
||||
// Check for memory-neo4j plugin
|
||||
if (entries["memory-neo4j"]) {
|
||||
const entry = entries["memory-neo4j"];
|
||||
const enabled = entry.enabled !== false && activeSlot !== "none";
|
||||
plugins.push({
|
||||
id: "memory-neo4j",
|
||||
enabled,
|
||||
hint: enabled ? "Run `openclaw memory neo4j stats` for detailed statistics" : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
// Check for memory-lancedb plugin
|
||||
if (entries["memory-lancedb"]) {
|
||||
const entry = entries["memory-lancedb"];
|
||||
const enabled = entry.enabled !== false && activeSlot !== "none";
|
||||
plugins.push({
|
||||
id: "memory-lancedb",
|
||||
enabled,
|
||||
hint: enabled ? "Run `openclaw memory lancedb stats` for detailed statistics" : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
return plugins;
|
||||
}
|
||||
|
||||
export function registerMemoryCli(program: Command) {
|
||||
|
||||
@@ -101,6 +101,16 @@ describe("shouldSkipPluginCommandRegistration", () => {
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps plugin registration for plugin-extensible builtins like memory", () => {
|
||||
expect(
|
||||
shouldSkipPluginCommandRegistration({
|
||||
argv: ["node", "openclaw", "memory", "neo4j", "sleep"],
|
||||
primary: "memory",
|
||||
hasBuiltinPrimary: true,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldEnsureCliPath", () => {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user