fix(bank-sync): transaction matching, services, and worker updates
This commit is contained in:
null
parent
31be51e77f
commit
79b51b1c9a
|
|
@ -3,6 +3,8 @@
|
|||
|
||||
### ✨ Added
|
||||
|
||||
- **SimpleFIN pending transactions** — Bank sync now requests pending (not-yet-settled) transactions, which SimpleFIN excludes by default. The fetch URL gains `pending=1`, and `normalizeTransaction` records a `pending` flag (migration v1.01 adds `transactions.pending` + a partial index). Pending charges are imported and shown with a sky-blue "Pending" badge on the Data page transaction list, but are deliberately **excluded from auto-matching** — both the merchant-rule and score-based match passes skip `pending = 1` rows so an unsettled charge can never create a phantom payment that later vanishes. `insertTransactionIfNew` is replaced by an idempotent `upsertTransaction`: when a pending charge later settles under the same id, the existing row is updated in place (gains its `posted_date`, refreshed amount, `pending → 0`) rather than duplicated — and only then becomes eligible for matching. To handle banks that re-post a pending charge under a *new* id, each sync prunes pending rows for an account that are no longer present in the feed (tracked via a per-account seen-id set), so stale pending rows can't accumulate. The sync result and worker logs now report settled and pending-cleared counts. Debug logging traces each pending insert/settle/prune step.
|
||||
|
||||
- **Autopay trust indicator** — Autopay is no longer treated as a binary flag. The tracker row's AP badge now shows a confidence score derived from the last 12 months of payment history: "✓ 12/12 successful" on a healthy bill, or "⚠ 11/12" in amber when even one payment was recorded as an autopay failure. A clock nudge appears when autopay hasn't been manually confirmed in over 90 days (or never). Hovering the badge shows a tooltip with full detail: success rate, last failure date and notes, and verification status. Inside BillModal, an autopay trust panel appears below the Autodraft Status select for existing bills: it surfaces the same confidence rate, shows a staleness warning if verification is overdue, and provides a "Mark verified" button that calls the new `POST /api/bills/:id/verify-autopay` endpoint and updates the timestamp in place. New migration v0.99 adds `bills.autopay_verified_at` (timestamp of last user confirmation) and `payments.autopay_failure` (flag marking a payment that was made because autopay silently failed). The PaymentModal edit dialog now shows an "Autopay failed — paid manually" checkbox on autopay-enabled bills or when the payment method is set to autopay, enabling retroactive flagging of past failures.
|
||||
|
||||
- **Trend sparkline on amount column** — Each tracker row now renders a 44×16 px inline SVG polyline to the right of the expected amount showing the last 6 months of actual payment totals, oldest-to-newest, normalized to the row's min/max range. No chart library is required — the sparkline is computed server-side in `trackerService.fetchSparklines` (a single batched `GROUP BY bill_id, month_str` query for all bill IDs at once) and rendered as a `<polyline>` on the client. The chart appears only when two or more months of data exist.
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import React, { useState, useEffect, useMemo } from 'react';
|
|||
import { toast } from 'sonner';
|
||||
import {
|
||||
Sparkles, CheckCircle2, Loader2, RefreshCw, Link2, Link2Off,
|
||||
XCircle, Eye, EyeOff, Search, Plus,
|
||||
XCircle, Eye, EyeOff, Search, Plus, Clock,
|
||||
} from 'lucide-react';
|
||||
import { api } from '@/api';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
|
@ -670,7 +670,15 @@ export default function TransactionMatchingSection({ refreshKey, simplefinConn,
|
|||
</td>
|
||||
<td className="px-4 py-3">
|
||||
<div className="flex flex-col gap-1.5">
|
||||
<div className="flex items-center gap-1.5">
|
||||
<TransactionStatusBadge tx={tx} />
|
||||
{tx.pending ? (
|
||||
<span className="inline-flex items-center gap-1 rounded-full border border-sky-500/30 bg-sky-500/10 px-2 py-0.5 text-[10px] font-semibold uppercase text-sky-600 dark:text-sky-400">
|
||||
<Clock className="h-2.5 w-2.5" />
|
||||
Pending
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
{tx.matched_bill_name ? (
|
||||
<span className="truncate text-xs text-foreground">{tx.matched_bill_name}</span>
|
||||
) : tx.advisory_filter?.confidence === 'high' ? (
|
||||
|
|
|
|||
|
|
@ -3344,6 +3344,21 @@ function runMigrations() {
|
|||
console.log('[v1.00] calendar feed token table ensured');
|
||||
}
|
||||
},
|
||||
{
|
||||
version: 'v1.01',
|
||||
description: 'transactions: pending flag for SimpleFIN pending transactions',
|
||||
run() {
|
||||
const cols = db.prepare('PRAGMA table_info(transactions)').all().map(c => c.name);
|
||||
if (!cols.includes('pending')) {
|
||||
db.exec('ALTER TABLE transactions ADD COLUMN pending INTEGER NOT NULL DEFAULT 0');
|
||||
}
|
||||
// Partial index speeds the per-account orphan prune that clears pending rows
|
||||
// which never posted (e.g. a pending charge that re-posted under a new id).
|
||||
db.exec(`CREATE INDEX IF NOT EXISTS idx_transactions_pending
|
||||
ON transactions(account_id) WHERE pending = 1`);
|
||||
console.log('[v1.01] transactions.pending flag + partial index added');
|
||||
}
|
||||
},
|
||||
];
|
||||
|
||||
// ── users: notification columns ───────────────────────────────────────────
|
||||
|
|
|
|||
|
|
@ -355,7 +355,7 @@ router.get('/', (req, res) => {
|
|||
t.id, t.user_id, t.data_source_id, t.account_id, t.provider_transaction_id,
|
||||
t.source_type, t.transaction_type, t.posted_date, t.transacted_at, t.amount,
|
||||
t.currency, t.description, t.payee, t.memo, t.category, t.matched_bill_id,
|
||||
t.match_status, t.ignored, t.created_at, t.updated_at,
|
||||
t.match_status, t.ignored, t.pending, t.created_at, t.updated_at,
|
||||
ds.type AS data_source_type, ds.provider AS data_source_provider,
|
||||
ds.name AS data_source_name, ds.status AS data_source_status,
|
||||
fa.name AS account_name, fa.org_name AS account_org_name,
|
||||
|
|
|
|||
|
|
@ -56,19 +56,29 @@ function upsertAccount(db, accountRow) {
|
|||
return { id: result.lastInsertRowid, monitored: 1 };
|
||||
}
|
||||
|
||||
// Insert a transaction, ignoring duplicates (unique index on data_source_id + provider_transaction_id).
|
||||
function insertTransactionIfNew(db, txRow) {
|
||||
// Insert a transaction, or update one we previously stored as PENDING. A pending charge
|
||||
// can change amount before it settles, and eventually flips pending → posted (gaining a
|
||||
// real posted_date). A transaction we already recorded as posted is final and left alone;
|
||||
// a row the user has matched or ignored is never touched.
|
||||
// Returns: 'inserted' | 'posted' (pending→settled) | 'updated' (pending refreshed) | 'skipped'.
|
||||
function upsertTransaction(db, txRow) {
|
||||
const existing = db.prepare(`
|
||||
SELECT id, pending, match_status FROM transactions
|
||||
WHERE data_source_id = ? AND provider_transaction_id = ?
|
||||
`).get(txRow.data_source_id, txRow.provider_transaction_id);
|
||||
|
||||
if (!existing) {
|
||||
try {
|
||||
db.prepare(`
|
||||
INSERT INTO transactions
|
||||
(user_id, data_source_id, account_id, provider_transaction_id,
|
||||
source_type, posted_date, transacted_at, amount, currency,
|
||||
description, payee, memo, match_status, ignored, raw_data)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
description, payee, memo, match_status, ignored, pending, raw_data)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
txRow.user_id, txRow.data_source_id, txRow.account_id, txRow.provider_transaction_id,
|
||||
txRow.source_type, txRow.posted_date, txRow.transacted_at, txRow.amount, txRow.currency,
|
||||
txRow.description, txRow.payee, txRow.memo, txRow.match_status, txRow.ignored, txRow.raw_data,
|
||||
txRow.description, txRow.payee, txRow.memo, txRow.match_status, txRow.ignored, txRow.pending, txRow.raw_data,
|
||||
);
|
||||
return 'inserted';
|
||||
} catch (err) {
|
||||
|
|
@ -77,6 +87,25 @@ function insertTransactionIfNew(db, txRow) {
|
|||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Only refresh rows still pending in our DB and not yet acted on by the user.
|
||||
if (existing.pending === 1 && existing.match_status === 'unmatched') {
|
||||
db.prepare(`
|
||||
UPDATE transactions
|
||||
SET posted_date = ?, transacted_at = ?, amount = ?, currency = ?,
|
||||
description = ?, payee = ?, memo = ?, pending = ?, raw_data = ?,
|
||||
updated_at = datetime('now')
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
txRow.posted_date, txRow.transacted_at, txRow.amount, txRow.currency,
|
||||
txRow.description, txRow.payee, txRow.memo, txRow.pending, txRow.raw_data,
|
||||
existing.id,
|
||||
);
|
||||
return txRow.pending === 0 ? 'posted' : 'updated';
|
||||
}
|
||||
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
async function runSync(db, userId, dataSource, { days, debug = false } = {}) {
|
||||
|
|
@ -103,6 +132,16 @@ async function runSync(db, userId, dataSource, { days, debug = false } = {}) {
|
|||
let accountsUpserted = 0;
|
||||
let transactionsNew = 0;
|
||||
let transactionsSkip = 0;
|
||||
let transactionsPosted = 0; // pending → settled this cycle
|
||||
let pendingCleared = 0; // stale pending rows pruned (re-posted under new id / dropped)
|
||||
|
||||
// Remove pending rows we hold for an account that are no longer in the feed — a pending
|
||||
// charge that re-posted under a new id, or was dropped by the bank before settling.
|
||||
const pruneOrphanPending = db.prepare(`
|
||||
DELETE FROM transactions
|
||||
WHERE account_id = ? AND user_id = ? AND pending = 1 AND match_status = 'unmatched'
|
||||
AND provider_transaction_id NOT IN (SELECT value FROM json_each(?))
|
||||
`);
|
||||
|
||||
for (const rawAccount of accounts) {
|
||||
const accountRow = normalizeAccount(rawAccount, dataSource.id, userId);
|
||||
|
|
@ -114,19 +153,32 @@ async function runSync(db, userId, dataSource, { days, debug = false } = {}) {
|
|||
|
||||
if (localAccount.monitored === 0) continue;
|
||||
|
||||
const seenTxIds = [];
|
||||
for (const rawTx of txList) {
|
||||
const txRow = normalizeTransaction(
|
||||
rawTx, localAccount.id, dataSource.id, userId, rawAccount.id, rawAccount.currency,
|
||||
);
|
||||
const outcome = insertTransactionIfNew(db, txRow);
|
||||
seenTxIds.push(txRow.provider_transaction_id);
|
||||
const outcome = upsertTransaction(db, txRow);
|
||||
if (outcome === 'inserted') {
|
||||
transactionsNew += 1;
|
||||
if (debug) console.log(`[bankSync:debug] tx ${txRow.provider_transaction_id}: inserted (${rawTx.description || rawTx.payee || '—'}, ${txRow.amount}¢)`);
|
||||
if (debug) console.log(`[bankSync:debug] tx ${txRow.provider_transaction_id}: inserted${txRow.pending ? ' (pending)' : ''} (${rawTx.description || rawTx.payee || '—'}, ${txRow.amount}¢)`);
|
||||
} else if (outcome === 'posted') {
|
||||
transactionsPosted += 1;
|
||||
if (debug) console.log(`[bankSync:debug] tx ${txRow.provider_transaction_id}: pending → posted`);
|
||||
} else if (outcome === 'updated') {
|
||||
if (debug) console.log(`[bankSync:debug] tx ${txRow.provider_transaction_id}: pending amount refreshed`);
|
||||
} else {
|
||||
transactionsSkip += 1;
|
||||
if (debug) console.log(`[bankSync:debug] tx ${txRow.provider_transaction_id}: duplicate — skipped`);
|
||||
}
|
||||
}
|
||||
|
||||
const orphans = pruneOrphanPending.run(localAccount.id, userId, JSON.stringify(seenTxIds));
|
||||
if (orphans.changes > 0) {
|
||||
pendingCleared += orphans.changes;
|
||||
if (debug) console.log(`[bankSync:debug] Account "${rawAccount.name}": pruned ${orphans.changes} stale pending row(s)`);
|
||||
}
|
||||
}
|
||||
|
||||
// Store any errlist warnings alongside a successful sync so users can see them
|
||||
|
|
@ -149,7 +201,7 @@ async function runSync(db, userId, dataSource, { days, debug = false } = {}) {
|
|||
|
||||
if (debug) console.log(`[bankSync:debug] Source #${dataSource.id}: auto-matched ${autoMatched} transaction(s)`);
|
||||
|
||||
return { accountsUpserted, transactionsNew, transactionsSkip, autoMatched, matched_bills: matchedBills || [], late_attributions: lateAttributions || [], errlist: raw._errlistSummary || null };
|
||||
return { accountsUpserted, transactionsNew, transactionsSkip, transactionsPosted, pendingCleared, autoMatched, matched_bills: matchedBills || [], late_attributions: lateAttributions || [], errlist: raw._errlistSummary || null };
|
||||
}
|
||||
|
||||
// ─── Public API ───────────────────────────────────────────────────────────────
|
||||
|
|
|
|||
|
|
@ -79,7 +79,11 @@ async function runCycle() {
|
|||
try {
|
||||
const result = await syncDataSource(db, source.user_id, source.id, { debug });
|
||||
synced++;
|
||||
console.log(`[bankSync] Source #${source.id}: OK — ${result.accountsUpserted} account(s), ${result.transactionsNew} new, ${result.transactionsSkip} skipped${result.errlist ? ` [partial: ${result.errlist}]` : ''}`);
|
||||
const extra = [
|
||||
result.transactionsPosted ? `${result.transactionsPosted} settled` : null,
|
||||
result.pendingCleared ? `${result.pendingCleared} pending cleared` : null,
|
||||
].filter(Boolean).join(', ');
|
||||
console.log(`[bankSync] Source #${source.id}: OK — ${result.accountsUpserted} account(s), ${result.transactionsNew} new, ${result.transactionsSkip} skipped${extra ? `, ${extra}` : ''}${result.errlist ? ` [partial: ${result.errlist}]` : ''}`);
|
||||
} catch (err) {
|
||||
failed++;
|
||||
console.error(`[bankSync] Source #${source.id}: FAILED — ${err.message}`);
|
||||
|
|
|
|||
|
|
@ -112,6 +112,7 @@ function applyMerchantRules(db, userId) {
|
|||
AND t.match_status = 'unmatched'
|
||||
AND t.ignored = 0
|
||||
AND t.amount < 0
|
||||
AND t.pending = 0
|
||||
AND (t.account_id IS NULL OR fa.id IS NULL OR fa.monitored = 1)
|
||||
`).all(userId);
|
||||
} catch (err) {
|
||||
|
|
@ -258,6 +259,7 @@ function syncBillPaymentsFromSimplefin(db, userId, billId) {
|
|||
AND t.match_status = 'unmatched'
|
||||
AND t.ignored = 0
|
||||
AND t.amount < 0
|
||||
AND t.pending = 0
|
||||
AND (t.account_id IS NULL OR fa.id IS NULL OR fa.monitored = 1)
|
||||
`).all(userId);
|
||||
} catch (err) {
|
||||
|
|
|
|||
|
|
@ -185,6 +185,7 @@ function loadCandidateTransactions(db, userId, transactionId = null) {
|
|||
't.user_id = ?',
|
||||
't.ignored = 0',
|
||||
"t.match_status = 'unmatched'",
|
||||
't.pending = 0', // don't match/pay against unsettled charges — they can change or vanish
|
||||
'(t.account_id IS NULL OR fa.id IS NULL OR fa.monitored = 1)',
|
||||
];
|
||||
if (transactionId) {
|
||||
|
|
|
|||
|
|
@ -96,7 +96,8 @@ async function fetchAccountsAndTransactions(accessUrl, sinceEpoch) {
|
|||
|
||||
const basicAuth = Buffer.from(`${url.username}:${url.password}`).toString('base64');
|
||||
const baseUrl = `${url.protocol}//${url.host}${url.pathname.replace(/\/?$/, '')}`;
|
||||
const endpoint = `${baseUrl}/accounts?start-date=${Math.floor(sinceEpoch)}&version=2`;
|
||||
// pending=1 asks SimpleFIN to include not-yet-settled transactions (excluded by default).
|
||||
const endpoint = `${baseUrl}/accounts?start-date=${Math.floor(sinceEpoch)}&pending=1&version=2`;
|
||||
|
||||
let lastErr;
|
||||
for (let attempt = 0; attempt < FETCH_RETRY_ATTEMPTS; attempt++) {
|
||||
|
|
@ -167,7 +168,9 @@ function normalizeAccount(rawAccount, dataSourceId, userId) {
|
|||
// survives disconnect/reconnect (data_source_id is intentionally omitted).
|
||||
function normalizeTransaction(rawTx, localAccountId, dataSourceId, userId, accountId, accountCurrency) {
|
||||
const amount = Math.round(parseFloat(rawTx.amount) * 100);
|
||||
const postedDate = rawTx.posted
|
||||
// Pending transactions report posted = 0 (or omit it) until they settle.
|
||||
const isPending = rawTx.pending === true || rawTx.pending === 1;
|
||||
const postedDate = (rawTx.posted && !isPending)
|
||||
? new Date(rawTx.posted * 1000).toISOString().slice(0, 10)
|
||||
: null;
|
||||
const transactedAt = rawTx['transacted_at']
|
||||
|
|
@ -192,6 +195,7 @@ function normalizeTransaction(rawTx, localAccountId, dataSourceId, userId, accou
|
|||
memo: rawTx.memo ? String(rawTx.memo).slice(0, 500) : null,
|
||||
match_status: 'unmatched',
|
||||
ignored: 0,
|
||||
pending: isPending ? 1 : 0,
|
||||
raw_data: sanitizeRawData(rawTx),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ function getTransactionForUser(db, userId, id) {
|
|||
t.id, t.user_id, t.data_source_id, t.account_id, t.provider_transaction_id,
|
||||
t.source_type, t.transaction_type, t.posted_date, t.transacted_at, t.amount,
|
||||
t.currency, t.description, t.payee, t.memo, t.category, t.matched_bill_id,
|
||||
t.match_status, t.ignored, t.created_at, t.updated_at,
|
||||
t.match_status, t.ignored, t.pending, t.created_at, t.updated_at,
|
||||
ds.type AS data_source_type, ds.provider AS data_source_provider,
|
||||
ds.name AS data_source_name, ds.status AS data_source_status,
|
||||
fa.name AS account_name, fa.org_name AS account_org_name,
|
||||
|
|
|
|||
|
|
@ -94,3 +94,71 @@ test('SimpleFIN sync skips storing transactions for unmonitored accounts', async
|
|||
const trackedTransactions = db.prepare('SELECT COUNT(*) AS count FROM transactions WHERE account_id = ?').get(trackedAccount.id).count;
|
||||
assert.equal(trackedTransactions, 1);
|
||||
});
|
||||
|
||||
test('SimpleFIN pending transactions: insert, settle in place, then prune orphans', async () => {
|
||||
const db = getDb();
|
||||
const userId = createUser(db, 'pending-lifecycle');
|
||||
const dataSourceId = createSource(db, userId);
|
||||
const accountId = createAccount(db, userId, dataSourceId, 'pending-acct', true);
|
||||
|
||||
const originalFetch = global.fetch;
|
||||
const mockAccounts = (transactions) => {
|
||||
global.fetch = async () => ({
|
||||
ok: true, status: 200,
|
||||
json: async () => ({
|
||||
accounts: [{ id: 'pending-acct', name: 'Pending Acct', currency: 'USD', balance: '500.00', transactions }],
|
||||
}),
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
// 1. First sync — one pending charge (posted=0, pending:true)
|
||||
mockAccounts([
|
||||
{ id: 'tx-1', amount: '-42.00', posted: 0, pending: true, description: 'Coffee (pending)' },
|
||||
]);
|
||||
let result = await syncDataSource(db, userId, dataSourceId);
|
||||
assert.equal(result.transactionsNew, 1);
|
||||
|
||||
let row = db.prepare('SELECT pending, posted_date FROM transactions WHERE provider_transaction_id = ?')
|
||||
.get('simplefin:pending-acct:tx-1');
|
||||
assert.equal(row.pending, 1, 'stored as pending');
|
||||
assert.equal(row.posted_date, null, 'pending has no posted_date');
|
||||
|
||||
// 2. Second sync — same id now settled (posted timestamp, pending gone)
|
||||
mockAccounts([
|
||||
{ id: 'tx-1', amount: '-42.50', posted: 1772323200, description: 'Coffee' },
|
||||
]);
|
||||
result = await syncDataSource(db, userId, dataSourceId);
|
||||
assert.equal(result.transactionsNew, 0, 'no new row — updated in place');
|
||||
assert.equal(result.transactionsPosted, 1, 'counted as settled');
|
||||
|
||||
const rows = db.prepare('SELECT pending, posted_date, amount FROM transactions WHERE account_id = ?').all(accountId);
|
||||
assert.equal(rows.length, 1, 'no duplicate row created on settle');
|
||||
assert.equal(rows[0].pending, 0, 'flipped to posted');
|
||||
assert.equal(rows[0].posted_date, '2026-03-01', 'gained posted_date');
|
||||
assert.equal(rows[0].amount, -4250, 'amount refreshed to settled value');
|
||||
|
||||
// 3. Add a fresh pending charge, then a sync where it vanishes (re-posted under a new id)
|
||||
mockAccounts([
|
||||
{ id: 'tx-2', amount: '-9.99', posted: 0, pending: true, description: 'Snack (pending)' },
|
||||
]);
|
||||
await syncDataSource(db, userId, dataSourceId);
|
||||
assert.equal(
|
||||
db.prepare("SELECT COUNT(*) AS n FROM transactions WHERE account_id = ? AND pending = 1").get(accountId).n,
|
||||
1, 'pending tx-2 stored',
|
||||
);
|
||||
|
||||
// tx-2 disappears from the feed (settled under a new id); orphan prune should clear it
|
||||
mockAccounts([
|
||||
{ id: 'tx-3', amount: '-9.99', posted: 1772323200, description: 'Snack' },
|
||||
]);
|
||||
result = await syncDataSource(db, userId, dataSourceId);
|
||||
assert.equal(result.pendingCleared, 1, 'orphaned pending row pruned');
|
||||
assert.equal(
|
||||
db.prepare("SELECT COUNT(*) AS n FROM transactions WHERE account_id = ? AND pending = 1").get(accountId).n,
|
||||
0, 'no stale pending rows remain',
|
||||
);
|
||||
} finally {
|
||||
global.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue