From 79b51b1c9a6c096ec6bba2d477a12a7c9470d659 Mon Sep 17 00:00:00 2001 From: null Date: Sun, 7 Jun 2026 20:07:27 -0500 Subject: [PATCH] fix(bank-sync): transaction matching, services, and worker updates --- HISTORY.md | 2 + .../data/TransactionMatchingSection.jsx | 12 ++- db/database.js | 15 +++ routes/transactions.js | 2 +- services/bankSyncService.js | 102 +++++++++++++----- services/bankSyncWorker.js | 6 +- services/billMerchantRuleService.js | 2 + services/matchSuggestionService.js | 1 + services/simplefinService.js | 8 +- services/transactionService.js | 2 +- tests/bankSyncService.test.js | 68 ++++++++++++ 11 files changed, 188 insertions(+), 32 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 92084d7..4feed0a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -3,6 +3,8 @@ ### ✨ Added +- **SimpleFIN pending transactions** — Bank sync now requests pending (not-yet-settled) transactions, which SimpleFIN excludes by default. The fetch URL gains `pending=1`, and `normalizeTransaction` records a `pending` flag (migration v1.01 adds `transactions.pending` + a partial index). Pending charges are imported and shown with a sky-blue "Pending" badge on the Data page transaction list, but are deliberately **excluded from auto-matching** — both the merchant-rule and score-based match passes skip `pending = 1` rows so an unsettled charge can never create a phantom payment that later vanishes. `insertTransactionIfNew` is replaced by an idempotent `upsertTransaction`: when a pending charge later settles under the same id, the existing row is updated in place (gains its `posted_date`, refreshed amount, `pending → 0`) rather than duplicated — and only then becomes eligible for matching. To handle banks that re-post a pending charge under a *new* id, each sync prunes pending rows for an account that are no longer present in the feed (tracked via a per-account seen-id set), so stale pending rows can't accumulate. The sync result and worker logs now report settled and pending-cleared counts. Debug logging traces each pending insert/settle/prune step. + - **Autopay trust indicator** — Autopay is no longer treated as a binary flag. The tracker row's AP badge now shows a confidence score derived from the last 12 months of payment history: "✓ 12/12 successful" on a healthy bill, or "⚠ 11/12" in amber when even one payment was recorded as an autopay failure. A clock nudge appears when autopay hasn't been manually confirmed in over 90 days (or never). Hovering the badge shows a tooltip with full detail: success rate, last failure date and notes, and verification status. Inside BillModal, an autopay trust panel appears below the Autodraft Status select for existing bills: it surfaces the same confidence rate, shows a staleness warning if verification is overdue, and provides a "Mark verified" button that calls the new `POST /api/bills/:id/verify-autopay` endpoint and updates the timestamp in place. New migration v0.99 adds `bills.autopay_verified_at` (timestamp of last user confirmation) and `payments.autopay_failure` (flag marking a payment that was made because autopay silently failed). The PaymentModal edit dialog now shows an "Autopay failed — paid manually" checkbox on autopay-enabled bills or when the payment method is set to autopay, enabling retroactive flagging of past failures. - **Trend sparkline on amount column** — Each tracker row now renders a 44×16 px inline SVG polyline to the right of the expected amount showing the last 6 months of actual payment totals, oldest-to-newest, normalized to the row's min/max range. No chart library is required — the sparkline is computed server-side in `trackerService.fetchSparklines` (a single batched `GROUP BY bill_id, month_str` query for all bill IDs at once) and rendered as a `` on the client. The chart appears only when two or more months of data exist. diff --git a/client/components/data/TransactionMatchingSection.jsx b/client/components/data/TransactionMatchingSection.jsx index e470fb6..3639d82 100644 --- a/client/components/data/TransactionMatchingSection.jsx +++ b/client/components/data/TransactionMatchingSection.jsx @@ -2,7 +2,7 @@ import React, { useState, useEffect, useMemo } from 'react'; import { toast } from 'sonner'; import { Sparkles, CheckCircle2, Loader2, RefreshCw, Link2, Link2Off, - XCircle, Eye, EyeOff, Search, Plus, + XCircle, Eye, EyeOff, Search, Plus, Clock, } from 'lucide-react'; import { api } from '@/api'; import { cn } from '@/lib/utils'; @@ -670,7 +670,15 @@ export default function TransactionMatchingSection({ refreshKey, simplefinConn,
- +
+ + {tx.pending ? ( + + + Pending + + ) : null} +
{tx.matched_bill_name ? ( {tx.matched_bill_name} ) : tx.advisory_filter?.confidence === 'high' ? ( diff --git a/db/database.js b/db/database.js index 453b3f2..3b575b3 100644 --- a/db/database.js +++ b/db/database.js @@ -3344,6 +3344,21 @@ function runMigrations() { console.log('[v1.00] calendar feed token table ensured'); } }, + { + version: 'v1.01', + description: 'transactions: pending flag for SimpleFIN pending transactions', + run() { + const cols = db.prepare('PRAGMA table_info(transactions)').all().map(c => c.name); + if (!cols.includes('pending')) { + db.exec('ALTER TABLE transactions ADD COLUMN pending INTEGER NOT NULL DEFAULT 0'); + } + // Partial index speeds the per-account orphan prune that clears pending rows + // which never posted (e.g. a pending charge that re-posted under a new id). + db.exec(`CREATE INDEX IF NOT EXISTS idx_transactions_pending + ON transactions(account_id) WHERE pending = 1`); + console.log('[v1.01] transactions.pending flag + partial index added'); + } + }, ]; // ── users: notification columns ─────────────────────────────────────────── diff --git a/routes/transactions.js b/routes/transactions.js index 6b85ca1..91bae2c 100644 --- a/routes/transactions.js +++ b/routes/transactions.js @@ -355,7 +355,7 @@ router.get('/', (req, res) => { t.id, t.user_id, t.data_source_id, t.account_id, t.provider_transaction_id, t.source_type, t.transaction_type, t.posted_date, t.transacted_at, t.amount, t.currency, t.description, t.payee, t.memo, t.category, t.matched_bill_id, - t.match_status, t.ignored, t.created_at, t.updated_at, + t.match_status, t.ignored, t.pending, t.created_at, t.updated_at, ds.type AS data_source_type, ds.provider AS data_source_provider, ds.name AS data_source_name, ds.status AS data_source_status, fa.name AS account_name, fa.org_name AS account_org_name, diff --git a/services/bankSyncService.js b/services/bankSyncService.js index 1779b39..80882ed 100644 --- a/services/bankSyncService.js +++ b/services/bankSyncService.js @@ -56,27 +56,56 @@ function upsertAccount(db, accountRow) { return { id: result.lastInsertRowid, monitored: 1 }; } -// Insert a transaction, ignoring duplicates (unique index on data_source_id + provider_transaction_id). -function insertTransactionIfNew(db, txRow) { - try { - db.prepare(` - INSERT INTO transactions - (user_id, data_source_id, account_id, provider_transaction_id, - source_type, posted_date, transacted_at, amount, currency, - description, payee, memo, match_status, ignored, raw_data) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `).run( - txRow.user_id, txRow.data_source_id, txRow.account_id, txRow.provider_transaction_id, - txRow.source_type, txRow.posted_date, txRow.transacted_at, txRow.amount, txRow.currency, - txRow.description, txRow.payee, txRow.memo, txRow.match_status, txRow.ignored, txRow.raw_data, - ); - return 'inserted'; - } catch (err) { - if (err.code === 'SQLITE_CONSTRAINT_UNIQUE' || (err.message || '').includes('UNIQUE')) { - return 'skipped'; +// Insert a transaction, or update one we previously stored as PENDING. A pending charge +// can change amount before it settles, and eventually flips pending → posted (gaining a +// real posted_date). A transaction we already recorded as posted is final and left alone; +// a row the user has matched or ignored is never touched. +// Returns: 'inserted' | 'posted' (pending→settled) | 'updated' (pending refreshed) | 'skipped'. +function upsertTransaction(db, txRow) { + const existing = db.prepare(` + SELECT id, pending, match_status FROM transactions + WHERE data_source_id = ? AND provider_transaction_id = ? + `).get(txRow.data_source_id, txRow.provider_transaction_id); + + if (!existing) { + try { + db.prepare(` + INSERT INTO transactions + (user_id, data_source_id, account_id, provider_transaction_id, + source_type, posted_date, transacted_at, amount, currency, + description, payee, memo, match_status, ignored, pending, raw_data) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + txRow.user_id, txRow.data_source_id, txRow.account_id, txRow.provider_transaction_id, + txRow.source_type, txRow.posted_date, txRow.transacted_at, txRow.amount, txRow.currency, + txRow.description, txRow.payee, txRow.memo, txRow.match_status, txRow.ignored, txRow.pending, txRow.raw_data, + ); + return 'inserted'; + } catch (err) { + if (err.code === 'SQLITE_CONSTRAINT_UNIQUE' || (err.message || '').includes('UNIQUE')) { + return 'skipped'; + } + throw err; } - throw err; } + + // Only refresh rows still pending in our DB and not yet acted on by the user. + if (existing.pending === 1 && existing.match_status === 'unmatched') { + db.prepare(` + UPDATE transactions + SET posted_date = ?, transacted_at = ?, amount = ?, currency = ?, + description = ?, payee = ?, memo = ?, pending = ?, raw_data = ?, + updated_at = datetime('now') + WHERE id = ? + `).run( + txRow.posted_date, txRow.transacted_at, txRow.amount, txRow.currency, + txRow.description, txRow.payee, txRow.memo, txRow.pending, txRow.raw_data, + existing.id, + ); + return txRow.pending === 0 ? 'posted' : 'updated'; + } + + return 'skipped'; } async function runSync(db, userId, dataSource, { days, debug = false } = {}) { @@ -100,9 +129,19 @@ async function runSync(db, userId, dataSource, { days, debug = false } = {}) { console.warn(`[bankSync] errlist for source ${dataSource.id}: ${raw._errlistSummary}`); } - let accountsUpserted = 0; - let transactionsNew = 0; - let transactionsSkip = 0; + let accountsUpserted = 0; + let transactionsNew = 0; + let transactionsSkip = 0; + let transactionsPosted = 0; // pending → settled this cycle + let pendingCleared = 0; // stale pending rows pruned (re-posted under new id / dropped) + + // Remove pending rows we hold for an account that are no longer in the feed — a pending + // charge that re-posted under a new id, or was dropped by the bank before settling. + const pruneOrphanPending = db.prepare(` + DELETE FROM transactions + WHERE account_id = ? AND user_id = ? AND pending = 1 AND match_status = 'unmatched' + AND provider_transaction_id NOT IN (SELECT value FROM json_each(?)) + `); for (const rawAccount of accounts) { const accountRow = normalizeAccount(rawAccount, dataSource.id, userId); @@ -114,19 +153,32 @@ async function runSync(db, userId, dataSource, { days, debug = false } = {}) { if (localAccount.monitored === 0) continue; + const seenTxIds = []; for (const rawTx of txList) { const txRow = normalizeTransaction( rawTx, localAccount.id, dataSource.id, userId, rawAccount.id, rawAccount.currency, ); - const outcome = insertTransactionIfNew(db, txRow); + seenTxIds.push(txRow.provider_transaction_id); + const outcome = upsertTransaction(db, txRow); if (outcome === 'inserted') { transactionsNew += 1; - if (debug) console.log(`[bankSync:debug] tx ${txRow.provider_transaction_id}: inserted (${rawTx.description || rawTx.payee || '—'}, ${txRow.amount}¢)`); + if (debug) console.log(`[bankSync:debug] tx ${txRow.provider_transaction_id}: inserted${txRow.pending ? ' (pending)' : ''} (${rawTx.description || rawTx.payee || '—'}, ${txRow.amount}¢)`); + } else if (outcome === 'posted') { + transactionsPosted += 1; + if (debug) console.log(`[bankSync:debug] tx ${txRow.provider_transaction_id}: pending → posted`); + } else if (outcome === 'updated') { + if (debug) console.log(`[bankSync:debug] tx ${txRow.provider_transaction_id}: pending amount refreshed`); } else { transactionsSkip += 1; if (debug) console.log(`[bankSync:debug] tx ${txRow.provider_transaction_id}: duplicate — skipped`); } } + + const orphans = pruneOrphanPending.run(localAccount.id, userId, JSON.stringify(seenTxIds)); + if (orphans.changes > 0) { + pendingCleared += orphans.changes; + if (debug) console.log(`[bankSync:debug] Account "${rawAccount.name}": pruned ${orphans.changes} stale pending row(s)`); + } } // Store any errlist warnings alongside a successful sync so users can see them @@ -149,7 +201,7 @@ async function runSync(db, userId, dataSource, { days, debug = false } = {}) { if (debug) console.log(`[bankSync:debug] Source #${dataSource.id}: auto-matched ${autoMatched} transaction(s)`); - return { accountsUpserted, transactionsNew, transactionsSkip, autoMatched, matched_bills: matchedBills || [], late_attributions: lateAttributions || [], errlist: raw._errlistSummary || null }; + return { accountsUpserted, transactionsNew, transactionsSkip, transactionsPosted, pendingCleared, autoMatched, matched_bills: matchedBills || [], late_attributions: lateAttributions || [], errlist: raw._errlistSummary || null }; } // ─── Public API ─────────────────────────────────────────────────────────────── diff --git a/services/bankSyncWorker.js b/services/bankSyncWorker.js index 43c52a1..eedab97 100644 --- a/services/bankSyncWorker.js +++ b/services/bankSyncWorker.js @@ -79,7 +79,11 @@ async function runCycle() { try { const result = await syncDataSource(db, source.user_id, source.id, { debug }); synced++; - console.log(`[bankSync] Source #${source.id}: OK — ${result.accountsUpserted} account(s), ${result.transactionsNew} new, ${result.transactionsSkip} skipped${result.errlist ? ` [partial: ${result.errlist}]` : ''}`); + const extra = [ + result.transactionsPosted ? `${result.transactionsPosted} settled` : null, + result.pendingCleared ? `${result.pendingCleared} pending cleared` : null, + ].filter(Boolean).join(', '); + console.log(`[bankSync] Source #${source.id}: OK — ${result.accountsUpserted} account(s), ${result.transactionsNew} new, ${result.transactionsSkip} skipped${extra ? `, ${extra}` : ''}${result.errlist ? ` [partial: ${result.errlist}]` : ''}`); } catch (err) { failed++; console.error(`[bankSync] Source #${source.id}: FAILED — ${err.message}`); diff --git a/services/billMerchantRuleService.js b/services/billMerchantRuleService.js index a5dd208..507af78 100644 --- a/services/billMerchantRuleService.js +++ b/services/billMerchantRuleService.js @@ -112,6 +112,7 @@ function applyMerchantRules(db, userId) { AND t.match_status = 'unmatched' AND t.ignored = 0 AND t.amount < 0 + AND t.pending = 0 AND (t.account_id IS NULL OR fa.id IS NULL OR fa.monitored = 1) `).all(userId); } catch (err) { @@ -258,6 +259,7 @@ function syncBillPaymentsFromSimplefin(db, userId, billId) { AND t.match_status = 'unmatched' AND t.ignored = 0 AND t.amount < 0 + AND t.pending = 0 AND (t.account_id IS NULL OR fa.id IS NULL OR fa.monitored = 1) `).all(userId); } catch (err) { diff --git a/services/matchSuggestionService.js b/services/matchSuggestionService.js index 7657b1f..1b58c7b 100644 --- a/services/matchSuggestionService.js +++ b/services/matchSuggestionService.js @@ -185,6 +185,7 @@ function loadCandidateTransactions(db, userId, transactionId = null) { 't.user_id = ?', 't.ignored = 0', "t.match_status = 'unmatched'", + 't.pending = 0', // don't match/pay against unsettled charges — they can change or vanish '(t.account_id IS NULL OR fa.id IS NULL OR fa.monitored = 1)', ]; if (transactionId) { diff --git a/services/simplefinService.js b/services/simplefinService.js index 31411be..fa82fa6 100644 --- a/services/simplefinService.js +++ b/services/simplefinService.js @@ -96,7 +96,8 @@ async function fetchAccountsAndTransactions(accessUrl, sinceEpoch) { const basicAuth = Buffer.from(`${url.username}:${url.password}`).toString('base64'); const baseUrl = `${url.protocol}//${url.host}${url.pathname.replace(/\/?$/, '')}`; - const endpoint = `${baseUrl}/accounts?start-date=${Math.floor(sinceEpoch)}&version=2`; + // pending=1 asks SimpleFIN to include not-yet-settled transactions (excluded by default). + const endpoint = `${baseUrl}/accounts?start-date=${Math.floor(sinceEpoch)}&pending=1&version=2`; let lastErr; for (let attempt = 0; attempt < FETCH_RETRY_ATTEMPTS; attempt++) { @@ -167,7 +168,9 @@ function normalizeAccount(rawAccount, dataSourceId, userId) { // survives disconnect/reconnect (data_source_id is intentionally omitted). function normalizeTransaction(rawTx, localAccountId, dataSourceId, userId, accountId, accountCurrency) { const amount = Math.round(parseFloat(rawTx.amount) * 100); - const postedDate = rawTx.posted + // Pending transactions report posted = 0 (or omit it) until they settle. + const isPending = rawTx.pending === true || rawTx.pending === 1; + const postedDate = (rawTx.posted && !isPending) ? new Date(rawTx.posted * 1000).toISOString().slice(0, 10) : null; const transactedAt = rawTx['transacted_at'] @@ -192,6 +195,7 @@ function normalizeTransaction(rawTx, localAccountId, dataSourceId, userId, accou memo: rawTx.memo ? String(rawTx.memo).slice(0, 500) : null, match_status: 'unmatched', ignored: 0, + pending: isPending ? 1 : 0, raw_data: sanitizeRawData(rawTx), }; } diff --git a/services/transactionService.js b/services/transactionService.js index 93b18d2..ccc4768 100644 --- a/services/transactionService.js +++ b/services/transactionService.js @@ -77,7 +77,7 @@ function getTransactionForUser(db, userId, id) { t.id, t.user_id, t.data_source_id, t.account_id, t.provider_transaction_id, t.source_type, t.transaction_type, t.posted_date, t.transacted_at, t.amount, t.currency, t.description, t.payee, t.memo, t.category, t.matched_bill_id, - t.match_status, t.ignored, t.created_at, t.updated_at, + t.match_status, t.ignored, t.pending, t.created_at, t.updated_at, ds.type AS data_source_type, ds.provider AS data_source_provider, ds.name AS data_source_name, ds.status AS data_source_status, fa.name AS account_name, fa.org_name AS account_org_name, diff --git a/tests/bankSyncService.test.js b/tests/bankSyncService.test.js index 7946729..50b2195 100644 --- a/tests/bankSyncService.test.js +++ b/tests/bankSyncService.test.js @@ -94,3 +94,71 @@ test('SimpleFIN sync skips storing transactions for unmonitored accounts', async const trackedTransactions = db.prepare('SELECT COUNT(*) AS count FROM transactions WHERE account_id = ?').get(trackedAccount.id).count; assert.equal(trackedTransactions, 1); }); + +test('SimpleFIN pending transactions: insert, settle in place, then prune orphans', async () => { + const db = getDb(); + const userId = createUser(db, 'pending-lifecycle'); + const dataSourceId = createSource(db, userId); + const accountId = createAccount(db, userId, dataSourceId, 'pending-acct', true); + + const originalFetch = global.fetch; + const mockAccounts = (transactions) => { + global.fetch = async () => ({ + ok: true, status: 200, + json: async () => ({ + accounts: [{ id: 'pending-acct', name: 'Pending Acct', currency: 'USD', balance: '500.00', transactions }], + }), + }); + }; + + try { + // 1. First sync — one pending charge (posted=0, pending:true) + mockAccounts([ + { id: 'tx-1', amount: '-42.00', posted: 0, pending: true, description: 'Coffee (pending)' }, + ]); + let result = await syncDataSource(db, userId, dataSourceId); + assert.equal(result.transactionsNew, 1); + + let row = db.prepare('SELECT pending, posted_date FROM transactions WHERE provider_transaction_id = ?') + .get('simplefin:pending-acct:tx-1'); + assert.equal(row.pending, 1, 'stored as pending'); + assert.equal(row.posted_date, null, 'pending has no posted_date'); + + // 2. Second sync — same id now settled (posted timestamp, pending gone) + mockAccounts([ + { id: 'tx-1', amount: '-42.50', posted: 1772323200, description: 'Coffee' }, + ]); + result = await syncDataSource(db, userId, dataSourceId); + assert.equal(result.transactionsNew, 0, 'no new row — updated in place'); + assert.equal(result.transactionsPosted, 1, 'counted as settled'); + + const rows = db.prepare('SELECT pending, posted_date, amount FROM transactions WHERE account_id = ?').all(accountId); + assert.equal(rows.length, 1, 'no duplicate row created on settle'); + assert.equal(rows[0].pending, 0, 'flipped to posted'); + assert.equal(rows[0].posted_date, '2026-03-01', 'gained posted_date'); + assert.equal(rows[0].amount, -4250, 'amount refreshed to settled value'); + + // 3. Add a fresh pending charge, then a sync where it vanishes (re-posted under a new id) + mockAccounts([ + { id: 'tx-2', amount: '-9.99', posted: 0, pending: true, description: 'Snack (pending)' }, + ]); + await syncDataSource(db, userId, dataSourceId); + assert.equal( + db.prepare("SELECT COUNT(*) AS n FROM transactions WHERE account_id = ? AND pending = 1").get(accountId).n, + 1, 'pending tx-2 stored', + ); + + // tx-2 disappears from the feed (settled under a new id); orphan prune should clear it + mockAccounts([ + { id: 'tx-3', amount: '-9.99', posted: 1772323200, description: 'Snack' }, + ]); + result = await syncDataSource(db, userId, dataSourceId); + assert.equal(result.pendingCleared, 1, 'orphaned pending row pruned'); + assert.equal( + db.prepare("SELECT COUNT(*) AS n FROM transactions WHERE account_id = ? AND pending = 1").get(accountId).n, + 0, 'no stale pending rows remain', + ); + } finally { + global.fetch = originalFetch; + } +});