diff --git a/CHANGELOG.md b/CHANGELOG.md index 394607a..762dc10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## 2026-08-23 +### Added + +- **whatsapp: chats and messages now survive a restart (v8 / 2.2.0)** — closes the gap left open by v7. WhatsApp delivers a history sync at *login*, not on reconnect, so with a purely in-memory store every process restart left the connector blind until new messages happened to arrive — and the process had restarted 23 times in two weeks. The store is now mirrored to `store/`, next to `auth/` and bind-mounted the same way, so it outlives both a restart and a container recreate. + - **No SQLite, deliberately.** `node:sqlite` needs Node 22 (and is only unflagged from Node 24); the runtime image ships Debian trixie's `nodejs` = 20.19.2. `better-sqlite3` is a native module the slim image has no toolchain to build. So: `store/messages.jsonl`, an append-only log (one `appendFileSync` per message, O(1)), plus `store/meta.json`, a debounced snapshot of chats/contacts (5s, since they churn in bursts during a history sync). Both written via write-tmp-then-`rename`, which is atomic — a crash mid-write cannot truncate the store. + - The log is compacted on load and every 500 appends: the capped in-memory Maps are re-serialised, so the file cannot creep upward across restarts. + - **Fixes a pre-existing duplication bug in the process.** `pushMessage` appended unconditionally, so every history re-sync re-added messages already held. It now returns false on a known message id (a ≤500-element scan per message) and `ingestMessage` skips both the transcript and the log — without it, persistence would have multiplied the duplicates once per restart instead of merely showing them. + - `MAX_MSGS_PER_CHAT` 200 → 500, worth more now that it is not thrown away at every restart. `logout` deletes `store/` along with `auth/`, so re-linking a different phone cannot inherit the previous account's history. + - ⚠️ Data at rest: message text is now written to disk in the user's bind-mounted home. Nothing was persisted before this change beyond the session keys. + - Verified on `skald-runtime:v4` with a seeded store — 751 log lines containing 50 exact duplicates, a 700-message chat, and a deliberately torn trailing line → loaded as 701 messages (duplicates collapsed, torn line skipped, no crash), compacted to 501 lines, the 700-message chat trimmed to its most recent 500, `meta.json` round-tripped, and a second run reloading 501 → 501 unchanged ✅ + ### Fixed - **whatsapp: history sync silently disabled, and Signal sessions corrupted by reconnects (v7 / 2.1.0)** — diagnosed from the live server, where two users (two separate accounts, separate `auth/` dirs) share one log file. Four distinct defects, plus a dependency upgrade. diff --git a/connectors/connectors.json b/connectors/connectors.json index ef54c9f..d6929a9 100644 --- a/connectors/connectors.json +++ b/connectors/connectors.json @@ -794,14 +794,14 @@ "type": "qr" }, "folder": "whatsapp", - "version": 7, - "version_string": "2.1.0", + "version": 8, + "version_string": "2.2.0", "version_release_date": "2026-08-23", "files": [ { "path": "connector.json", - "sha256": "87bac98a524c604ca016a80b26200ae16aaee3926baecfe99e1c336833823307", - "size": 1485 + "sha256": "7efc38c81915ecef2ecdd5bc16b4e3fbccc9ef735a5ae6262317b823a64d6f31", + "size": 1609 }, { "path": "icon_lg.png", @@ -815,8 +815,8 @@ }, { "path": "index.js", - "sha256": "fa1f1c31c1f3b6b32df3de0a1175e1cb51a0417dce7e3aa52fa8655900dd2967", - "size": 29495 + "sha256": "291a450714859e9f579bd2594bc6a5120daf17a9c77cf6c7cb1e59c64d5aa4f4", + "size": 33564 }, { "path": "package.json", diff --git a/connectors/whatsapp/connector.json b/connectors/whatsapp/connector.json index 8a057ac..677683d 100644 --- a/connectors/whatsapp/connector.json +++ b/connectors/whatsapp/connector.json @@ -23,7 +23,8 @@ "Skald installs dependencies automatically (npm install).", "Open the connector in Skald and scan the QR code with your WhatsApp phone.", "The session is persisted and survives restarts.", - "The phone lists this device as \"Mac OS Chrome\": that browser identity is what makes WhatsApp deliver the full history sync." + "The phone lists this device as \"Mac OS Chrome\": that browser identity is what makes WhatsApp deliver the full history sync.", + "Chats and messages are mirrored to ./store/ so they survive a restart; `logout` deletes them along with the session." ], "docs": [ { @@ -45,7 +46,7 @@ "homepage": "https://github.com/WhiskeySockets/Baileys", "icon_small": "icon_sm.png", "icon_large": "icon_lg.png", - "version": 7, - "version_string": "2.1.0", + "version": 8, + "version_string": "2.2.0", "version_release_date": "2026-08-23" } diff --git a/connectors/whatsapp/fragment.json b/connectors/whatsapp/fragment.json index e741d46..0a80045 100644 --- a/connectors/whatsapp/fragment.json +++ b/connectors/whatsapp/fragment.json @@ -20,7 +20,7 @@ "type": "qr" }, "folder": "whatsapp", - "version": 7, - "version_string": "2.1.0", + "version": 8, + "version_string": "2.2.0", "version_release_date": "2026-08-23" } diff --git a/connectors/whatsapp/index.js b/connectors/whatsapp/index.js index eb99fa1..45fde2f 100644 --- a/connectors/whatsapp/index.js +++ b/connectors/whatsapp/index.js @@ -125,22 +125,126 @@ const RECONNECT_BASE_MS = 1500; const RECONNECT_MAX_MS = 60_000; let backoffMs = RECONNECT_BASE_MS; -// ── Lightweight in-memory store ─────────────────────────────────────────────── +// ── Store ───────────────────────────────────────────────────────────────────── // Baileys keeps no chat/contact store of its own; we build a minimal one from the -// history-sync event and live upserts. It lives for the process lifetime — enough -// for "what's going on now", not a full archive. +// history-sync event and live upserts. It is mirrored to `store/` (bind-mounted next +// to `auth/`, so it survives a container recreate) because WhatsApp only delivers a +// history sync at *login* — without a mirror every process restart left the connector +// blind until new messages happened to arrive. +// +// Deliberately not SQLite: `node:sqlite` needs Node 22 (the runtime image is 20.19.2) +// and `better-sqlite3` is a native module the slim image cannot build. So: an +// append-only JSONL log for messages (O(1) per message) plus a small debounced +// snapshot for chats/contacts. const chats = new Map(); // jid -> { id, name, unread, conversationTimestamp } const contacts = new Map(); // jid -> { id, name } const messages = new Map(); // jid -> [ { id, fromMe, ts, text, author } ] (capped) -const MAX_MSGS_PER_CHAT = 200; +const MAX_MSGS_PER_CHAT = 500; +const STORE_DIR = path.join(__dirname, 'store'); +const MSG_LOG = path.join(STORE_DIR, 'messages.jsonl'); +const META_FILE = path.join(STORE_DIR, 'meta.json'); +const COMPACT_EVERY = 500; // appends between rewrites of the log +const META_DEBOUNCE_MS = 5000; + +let appendsSinceCompact = 0; +let metaTimer = null; + +// Returns false when the message was already known: history sync re-delivers the same +// ids on every login, and without this both the log and the transcript would grow a +// duplicate per restart. function pushMessage(jid, m) { - if (!jid) return; + if (!jid) return false; let arr = messages.get(jid); if (!arr) { arr = []; messages.set(jid, arr); } + if (m.id && arr.some((x) => x.id === m.id)) return false; arr.push(m); if (arr.length > MAX_MSGS_PER_CHAT) arr.splice(0, arr.length - MAX_MSGS_PER_CHAT); + return true; +} + +function writeFileAtomic(file, data) { + const tmp = `${file}.tmp`; + fs.writeFileSync(tmp, data); + fs.renameSync(tmp, file); // rename is atomic: a crash mid-write cannot truncate the store +} + +function loadStore() { + try { + const meta = JSON.parse(fs.readFileSync(META_FILE, 'utf8')); + for (const c of meta.chats || []) if (c?.id) chats.set(c.id, c); + for (const c of meta.contacts || []) if (c?.id) contacts.set(c.id, c); + } catch { /* first run, or a snapshot we cannot parse — start empty */ } + + let restored = 0; + try { + for (const line of fs.readFileSync(MSG_LOG, 'utf8').split('\n')) { + if (!line) continue; + try { + const { jid, m } = JSON.parse(line); + if (pushMessage(jid, m)) restored++; + } catch { /* skip a torn trailing line */ } + } + } catch { /* no log yet */ } + + if (restored || chats.size || contacts.size) { + log(`store loaded: ${chats.size} chats, ${contacts.size} contacts, ${restored} messages`); + } + // Rewrite immediately: replays the per-chat cap and drops duplicates, so the log + // cannot creep upward across restarts. + if (restored) compactMessageLog(); +} + +function compactMessageLog() { + try { + fs.mkdirSync(STORE_DIR, { recursive: true }); + const out = []; + for (const [jid, arr] of messages) for (const m of arr) out.push(JSON.stringify({ jid, m })); + writeFileAtomic(MSG_LOG, out.length ? out.join('\n') + '\n' : ''); + appendsSinceCompact = 0; + } catch (e) { + log(`store compaction failed: ${e.message}`); + } +} + +function appendMessage(jid, m) { + try { + fs.mkdirSync(STORE_DIR, { recursive: true }); + fs.appendFileSync(MSG_LOG, JSON.stringify({ jid, m }) + '\n'); + if (++appendsSinceCompact >= COMPACT_EVERY) compactMessageLog(); + } catch (e) { + log(`store append failed: ${e.message}`); + } +} + +function saveMeta() { + try { + fs.mkdirSync(STORE_DIR, { recursive: true }); + writeFileAtomic(META_FILE, JSON.stringify({ + chats: [...chats.values()], + contacts: [...contacts.values()], + })); + } catch (e) { + log(`store meta save failed: ${e.message}`); + } +} + +// chats/contacts churn in bursts during a history sync; one write per burst is enough. +function scheduleMetaSave() { + if (metaTimer) return; + metaTimer = setTimeout(() => { metaTimer = null; saveMeta(); }, META_DEBOUNCE_MS); +} + +function flushStore() { + if (metaTimer) { clearTimeout(metaTimer); metaTimer = null; } + saveMeta(); +} + +function clearStore() { + try { fs.rmSync(STORE_DIR, { recursive: true, force: true }); } catch { /* nothing to clear */ } + appendsSinceCompact = 0; + if (metaTimer) { clearTimeout(metaTimer); metaTimer = null; } } function contactName(jid) { @@ -307,6 +411,7 @@ async function startSock() { contacts.set(c.id, { id: c.id, name: c.name || c.notify || c.verifiedName || null }); } for (const m of hm || []) ingestMessage(m, false); + scheduleMetaSave(); log(`history sync (type ${syncType ?? '?'}${progress != null ? `, ${progress}%` : ''}): ` + `+${(hc || []).length} chats, +${(hcs || []).length} contacts, +${(hm || []).length} messages ` + `→ ${chats.size} chats / ${contacts.size} contacts known`); @@ -319,10 +424,12 @@ async function startSock() { unread: c.unreadCount || 0, conversationTimestamp: Number(c.conversationTimestamp) || 0, }); + scheduleMetaSave(); }); s.ev.on('contacts.upsert', (cs) => { if (myGen !== generation) return; for (const c of cs) contacts.set(c.id, { id: c.id, name: c.name || c.notify || c.verifiedName || null }); + scheduleMetaSave(); }); s.ev.on('contacts.update', (cs) => { if (myGen !== generation) return; @@ -330,6 +437,7 @@ async function startSock() { const prev = contacts.get(c.id) || { id: c.id }; contacts.set(c.id, { ...prev, name: c.name || c.notify || prev.name || null }); } + scheduleMetaSave(); }); s.ev.on('messages.upsert', ({ messages: ms, type }) => { @@ -363,19 +471,24 @@ function ingestMessage(m, live) { // and are pure noise in a transcript. if (!text) return; - pushMessage(jid, { + const stored = { id: m.key?.id, fromMe: !!m.key?.fromMe, ts: Number(m.messageTimestamp) || 0, text, author: m.key?.participant || (m.key?.fromMe ? meJid : jid), - }); + }; + if (!pushMessage(jid, stored)) return; // already known — do not log it twice + appendMessage(jid, stored); + if (live && !chats.has(jid)) { chats.set(jid, { id: jid, name: m.pushName || null, unread: 0, conversationTimestamp: Number(m.messageTimestamp) || 0 }); + scheduleMetaSave(); } else if (live) { const ch = chats.get(jid); ch.conversationTimestamp = Number(m.messageTimestamp) || ch.conversationTimestamp; if (m.pushName && !ch.name) ch.name = m.pushName; + scheduleMetaSave(); } } catch { /* one malformed frame must not stop the stream */ } } @@ -435,6 +548,7 @@ async function toolLogout() { if (reconnectTimer) { clearTimeout(reconnectTimer); reconnectTimer = null; } try { fs.rmSync(AUTH_DIR, { recursive: true, force: true }); } catch { /* nothing to clear */ } chats.clear(); contacts.clear(); messages.clear(); + clearStore(); curQr = null; state = 'connecting'; starting = false; meJid = null; backoffMs = RECONNECT_BASE_MS; scheduleReconnect(500); @@ -634,6 +748,7 @@ async function handleRequest(msg) { async function main() { log('Starting WhatsApp MCP server (Baileys)'); fs.mkdirSync(MEDIA_DIR, { recursive: true }); + loadStore(); if (await loadBaileys()) { startSock().catch((e) => log(`initial startSock failed: ${e.message}`)); @@ -658,6 +773,7 @@ function shutdown(code) { generation++; teardownSock(sock); sock = null; + flushStore(); process.exit(code); }