From 060c8dc2f4800a5a6ed888ff70721bc37a4f6805 Mon Sep 17 00:00:00 2001 From: null Date: Sat, 16 May 2026 21:36:04 -0500 Subject: [PATCH] chore: version bump to 0.28.01 and update HISTORY format --- .gitignore | 1 + HISTORY.md | 14 +- client/api.js | 15 + client/components/BillModal.jsx | 180 +++++- client/pages/DataPage.jsx | 803 ++++++++++++++++++++++---- db/database.js | 81 +++ db/schema.sql | 16 + docs/Engineering_Reference_Manual.md | 49 +- package.json | 2 +- routes/bills.js | 59 ++ routes/import.js | 2 +- routes/matches.js | 34 ++ routes/payments.js | 16 + routes/profile.js | 14 +- routes/transactions.js | 101 ++-- server.js | 1 + services/matchSuggestionService.js | 334 +++++++++++ services/paymentValidation.js | 2 +- services/transactionMatchService.js | 313 ++++++++++ services/userDbImportService.js | 2 +- tests/transactionMatchService.test.js | 383 ++++++++++++ 21 files changed, 2240 insertions(+), 182 deletions(-) create mode 100644 routes/matches.js create mode 100644 services/matchSuggestionService.js create mode 100644 services/transactionMatchService.js create mode 100644 tests/transactionMatchService.test.js diff --git a/.gitignore b/.gitignore index 4832523..ac7a930 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ backups/ *.log simplefin-bank-sync-issue.md project-wide-data-input-and-sync-issue.md +docs/SIMPLEFIN_CONSUMER_GUARDRAILS.md diff --git a/HISTORY.md b/HISTORY.md index a3f50c9..be84e97 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,19 +1,27 @@ # Bill Tracker β€” Changelog -## v0.28.1 +## v0.28.01 -### Added +### πŸ† Major Features + +- **Transaction foundation and CSV import** β€” New `data_sources`, `financial_accounts`, and `transactions` tables provide the shared data layer for manual, file-import, and provider-sync workflows. CSV transaction import on the Data page offers upload β†’ preview β†’ column mapping β†’ commit with SHA-256 dedupe and import-history logging. + +### πŸš€ Features - **Manual bill payment history** β€” The Bills edit/detail modal now shows a payment history ledger with paid date, amount, method, notes, and payment source. - **Bills-side payment management** β€” Users can add, edit, soft-delete, and restore manual payments from the bill modal using the existing payment APIs. - **Payment source metadata** β€” The existing `payments` table now carries `payment_source` and `transaction_id` metadata so manual records can become the canonical base for later import and sync work without adding a new payment table. - **Transaction CSV import** β€” The Data page now includes a transaction CSV importer with preview, column mapping, commit counts, duplicate skipping, and import-history logging into the shared `transactions` table. -### Changed +### 🌟 Enhancements - **Canonical payment ledger** β€” Payment responses now include source metadata while preserving existing REAL-dollar payment amounts, partial-payment status derivation, and Tracker payment behavior. - **Import controls** β€” `DATA_IMPORT_ENABLED=false` now disables import preview/apply/commit endpoints, and CSV import is available through both `/api/import/csv/*` and `/api/imports/csv/*`. +### Release Image + +![Doing my part](/img/doingmypart.jpg) + ## v0.28.0 ### πŸ† Major Features diff --git a/client/api.js b/client/api.js index b96e7a7..ee062e1 100644 --- a/client/api.js +++ b/client/api.js @@ -168,6 +168,7 @@ export const api = { duplicateBill: (id, data) => post(`/bills/${id}/duplicate`, data), togglePaid: (id, data) => post(`/bills/${id}/toggle-paid`, data), billPayments: (id, p, l) => get(`/bills/${id}/payments?page=${p||1}&limit=${l||20}`), + billTransactions: (id) => get(`/bills/${id}/transactions`), billMonthlyState: (id, y, m) => get(`/bills/${id}/monthly-state?year=${y}&month=${m}`), saveBillMonthlyState: (id, data) => put(`/bills/${id}/monthly-state`, data), billHistoryRanges: (id) => get(`/bills/${id}/history-ranges`), @@ -287,6 +288,20 @@ export const api = { commitCsvTransactionImport: (data) => post('/import/csv/commit', data), importHistory: () => get('/import/history'), + // Transactions + transactions: (params = {}) => get(`/transactions${queryString(params)}`), + createManualTransaction: (data) => post('/transactions/manual', data), + updateTransaction: (id, data) => put(`/transactions/${id}`, data), + deleteTransaction: (id) => del(`/transactions/${id}`), + matchTransaction: (id, billId) => post(`/transactions/${id}/match`, { billId }), + unmatchTransaction: (id) => post(`/transactions/${id}/unmatch`), + ignoreTransaction: (id) => post(`/transactions/${id}/ignore`), + unignoreTransaction: (id) => post(`/transactions/${id}/unignore`), + + // Match suggestions + matchSuggestions: (params = {}) => get(`/matches/suggestions${queryString(params)}`), + rejectMatchSuggestion: (id) => post(`/matches/${encodeURIComponent(id)}/reject`), + // User SQLite import previewUserDbImport: async (file) => { const res = await fetch('/api/import/user-db/preview', { diff --git a/client/components/BillModal.jsx b/client/components/BillModal.jsx index 6382e35..a275078 100644 --- a/client/components/BillModal.jsx +++ b/client/components/BillModal.jsx @@ -1,5 +1,5 @@ import { useEffect, useState } from 'react'; -import { ChevronDown, Copy, Pencil, Plus, Trash2 } from 'lucide-react'; +import { ChevronDown, Copy, Link2, Link2Off, Pencil, Plus, Trash2 } from 'lucide-react'; import { toast } from 'sonner'; import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; @@ -45,6 +45,28 @@ const PAYMENT_METHODS = [ const DEBT_KEYWORDS = ['credit', 'loan', 'mortgage', 'housing', 'debt']; const SNOWBALL_KEYWORDS = ['credit', 'loan', 'debt']; +function fmtTransactionAmount(amount, currency = 'USD') { + const cents = Number(amount || 0); + const value = Math.abs(cents) / 100; + const sign = cents < 0 ? '-' : '+'; + return `${sign}${new Intl.NumberFormat(undefined, { + style: 'currency', + currency: currency || 'USD', + }).format(value)}`; +} + +function transactionDate(tx) { + return tx?.posted_date || String(tx?.transacted_at || '').slice(0, 10) || null; +} + +function transactionTitle(tx) { + return tx?.payee || tx?.description || tx?.memo || 'Transaction'; +} + +function isTransactionLinkedPayment(payment) { + return payment?.payment_source === 'transaction_match' || payment?.transaction_id != null; +} + function isDebtCat(categories, catId) { if (!catId || catId === CAT_NONE) return false; const cat = categories.find(c => String(c.id) === catId); @@ -93,6 +115,9 @@ export default function BillModal({ bill, initialBill, categories, onClose, onSa const [errors, setErrors] = useState({}); const [payments, setPayments] = useState([]); const [paymentsLoading, setPaymentsLoading] = useState(false); + const [linkedTransactions, setLinkedTransactions] = useState([]); + const [linkedTransactionsLoading, setLinkedTransactionsLoading] = useState(false); + const [transactionBusyId, setTransactionBusyId] = useState(null); const [paymentBusy, setPaymentBusy] = useState(false); const [paymentFormOpen, setPaymentFormOpen] = useState(false); const [editingPayment, setEditingPayment] = useState(null); @@ -120,8 +145,22 @@ export default function BillModal({ bill, initialBill, categories, onClose, onSa } } + async function loadLinkedTransactions() { + if (isNew || !bill?.id) return; + setLinkedTransactionsLoading(true); + try { + const data = await api.billTransactions(bill.id); + setLinkedTransactions(data.transactions || []); + } catch (err) { + toast.error(err.message || 'Failed to load linked transactions.'); + } finally { + setLinkedTransactionsLoading(false); + } + } + useEffect(() => { loadPayments(); + loadLinkedTransactions(); }, [bill?.id]); const validateName = (val) => { @@ -318,6 +357,21 @@ export default function BillModal({ bill, initialBill, categories, onClose, onSa } } + async function handleUnmatchTransaction(transaction) { + if (!transaction?.id) return; + setTransactionBusyId(transaction.id); + try { + await api.unmatchTransaction(transaction.id); + toast.success('Transaction unmatched'); + await Promise.all([loadPayments(), loadLinkedTransactions()]); + onSave?.(); + } catch (err) { + toast.error(err.message || 'Transaction could not be unmatched.'); + } finally { + setTransactionBusyId(null); + } + } + async function handleSubmit(e) { e.preventDefault(); @@ -825,35 +879,111 @@ export default function BillModal({ bill, initialBill, categories, onClose, onSa ) : (
- {payments.map(payment => ( -
-
-
-

{fmt(payment.amount)}

- - {payment.payment_source || 'manual'} - + {payments.map(payment => { + const linkedPayment = isTransactionLinkedPayment(payment); + return ( +
+
+
+

{fmt(payment.amount)}

+ + {payment.payment_source || 'manual'} + +
+

+ {fmtDate(payment.paid_date)} Β· {payment.method || 'manual'} +

+ {payment.notes && ( +

{payment.notes}

+ )} +
+
+ {linkedPayment ? ( + + + Matched + + ) : ( + <> + + + + )}
-

- {fmtDate(payment.paid_date)} Β· {payment.method || 'manual'} -

- {payment.notes && ( -

{payment.notes}

- )}
-
- - -
-
- ))} + ); + })}
)} +
+
+
+

Linked transactions

+

{linkedTransactions.length} confirmed matches

+
+ + + Matched + +
+ + {linkedTransactionsLoading ? ( +
+ Loading linked transactions... +
+ ) : linkedTransactions.length === 0 ? ( +
+ + No transactions linked to this bill yet. +
+ ) : ( +
+ {linkedTransactions.map(transaction => ( +
+
+
+

{transactionTitle(transaction)}

+ + {transaction.source_label || transaction.source_type_label || 'Transaction'} + +
+

+ {transactionDate(transaction) ? fmtDate(transactionDate(transaction)) : 'No date'} Β· {transaction.description || transaction.memo || 'No description'} +

+ {transaction.account_name && ( +

{transaction.account_name}

+ )} +
+
+

+ {fmtTransactionAmount(transaction.amount, transaction.currency)} +

+ +
+
+ ))} +
+ )} +
+ {paymentFormOpen && (
diff --git a/client/pages/DataPage.jsx b/client/pages/DataPage.jsx index a0f7bcd..db69f1c 100644 --- a/client/pages/DataPage.jsx +++ b/client/pages/DataPage.jsx @@ -4,7 +4,8 @@ import { Upload, FileSpreadsheet, Database, Download, CheckCircle2, XCircle, AlertTriangle, Loader2, RefreshCw, Clock, ChevronDown, ChevronUp, SkipForward, Plus, CheckCheck, Sparkles, - List, Building2, ChevronLeft, FileText, + List, Building2, ChevronLeft, FileText, Link2, Link2Off, + EyeOff, Eye, Search, } from 'lucide-react'; import { api } from '@/api'; import { cn } from '@/lib/utils'; @@ -22,6 +23,14 @@ import { AlertDialogTitle, AlertDialogTrigger, } from '@/components/ui/alert-dialog'; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog'; // ─── User export availability flag ─────────────────────────────────────────── // Flip to true when GET /api/export/user-db and GET /api/export/user-excel exist. @@ -328,6 +337,384 @@ function CountPill({ label, value }) { ); } +// ─── Section 2: Transaction Matching ───────────────────────────────────────── + +const TRANSACTION_FILTERS = [ + { id: 'open', label: 'Open', params: { ignored: 'false' } }, + { id: 'unmatched', label: 'Unmatched', params: { match_status: 'unmatched', ignored: 'false' } }, + { id: 'matched', label: 'Matched', params: { match_status: 'matched', ignored: 'false' } }, + { id: 'ignored', label: 'Ignored', params: { match_status: 'ignored', ignored: 'true' } }, + { id: 'all', label: 'All', params: { ignored: 'all' } }, +]; + +function transactionStatus(tx) { + if (tx?.ignored) return 'ignored'; + return tx?.match_status || 'unmatched'; +} + +function TransactionStatusBadge({ tx }) { + const status = transactionStatus(tx); + const styles = { + matched: 'border-emerald-500/30 bg-emerald-500/10 text-emerald-600', + ignored: 'border-muted-foreground/30 bg-muted/40 text-muted-foreground', + unmatched: 'border-amber-500/30 bg-amber-500/10 text-amber-600 dark:text-amber-400', + }; + + return ( + + {status} + + ); +} + +function formatTransactionAmount(amount, currency = 'USD') { + const value = Math.abs(Number(amount || 0)) / 100; + const sign = Number(amount || 0) < 0 ? '-' : '+'; + return `${sign}${new Intl.NumberFormat(undefined, { + style: 'currency', + currency: currency || 'USD', + }).format(value)}`; +} + +function transactionDate(tx) { + return tx?.posted_date || String(tx?.transacted_at || '').slice(0, 10) || 'β€”'; +} + +function transactionTitle(tx) { + return tx?.payee || tx?.description || tx?.memo || 'Transaction'; +} + +function MatchBillDialog({ open, onOpenChange, transaction, bills, onConfirm, loading }) { + const [query, setQuery] = useState(''); + const [selectedBillId, setSelectedBillId] = useState(''); + + useEffect(() => { + if (open) { + setQuery(''); + setSelectedBillId(transaction?.matched_bill_id ? String(transaction.matched_bill_id) : ''); + } + }, [open, transaction?.id, transaction?.matched_bill_id]); + + const filteredBills = useMemo(() => { + const q = query.trim().toLowerCase(); + if (!q) return bills.slice(0, 40); + return bills + .filter(bill => String(bill.name || '').toLowerCase().includes(q)) + .slice(0, 40); + }, [bills, query]); + + const selectedBill = bills.find(bill => String(bill.id) === String(selectedBillId)); + + return ( + + + + Match Transaction + + Choose the bill this transaction paid. Nothing changes until you confirm. + + + + {transaction && ( +
+
+
+

{transactionTitle(transaction)}

+

+ {transactionDate(transaction)} Β· {transaction.source_label || transaction.source_type_label || 'Transaction'} +

+
+

+ {formatTransactionAmount(transaction.amount, transaction.currency)} +

+
+ {transaction.description && transaction.description !== transactionTitle(transaction) && ( +

{transaction.description}

+ )} +
+ )} + +
+ + +
+ {filteredBills.length === 0 ? ( +

No bills found.

+ ) : ( +
+ {filteredBills.map(bill => ( + + ))} +
+ )} +
+
+ + + + + + + + ); +} + +function TransactionMatchingSection({ refreshKey }) { + const [transactions, setTransactions] = useState([]); + const [bills, setBills] = useState([]); + const [filter, setFilter] = useState('open'); + const [loading, setLoading] = useState(true); + const [billsLoading, setBillsLoading] = useState(true); + const [actionId, setActionId] = useState(null); + const [matchOpen, setMatchOpen] = useState(false); + const [matchTransaction, setMatchTransaction] = useState(null); + + const currentFilter = TRANSACTION_FILTERS.find(item => item.id === filter) || TRANSACTION_FILTERS[0]; + + const loadTransactions = async () => { + setLoading(true); + try { + const data = await api.transactions({ limit: 100, ...currentFilter.params }); + setTransactions(data || []); + } catch (err) { + toast.error(err.message || 'Failed to load transactions.'); + setTransactions([]); + } finally { + setLoading(false); + } + }; + + const loadBills = async () => { + setBillsLoading(true); + try { + const data = await api.bills(); + setBills(data || []); + } catch { + setBills([]); + } finally { + setBillsLoading(false); + } + }; + + useEffect(() => { loadBills(); }, []); + useEffect(() => { loadTransactions(); }, [filter, refreshKey]); + + const openMatchDialog = (tx) => { + setMatchTransaction(tx); + setMatchOpen(true); + if (!bills.length && !billsLoading) loadBills(); + }; + + const runTransactionAction = async (tx, action) => { + setActionId(`${action}:${tx.id}`); + try { + if (action === 'unmatch') { + await api.unmatchTransaction(tx.id); + toast.success('Transaction unmatched.'); + } else if (action === 'ignore') { + await api.ignoreTransaction(tx.id); + toast.success('Transaction ignored.'); + } else if (action === 'unignore') { + await api.unignoreTransaction(tx.id); + toast.success('Transaction restored.'); + } + await loadTransactions(); + } catch (err) { + toast.error(err.message || 'Transaction action failed.'); + } finally { + setActionId(null); + } + }; + + const confirmMatch = async (billId) => { + if (!matchTransaction) return; + setActionId(`match:${matchTransaction.id}`); + try { + await api.matchTransaction(matchTransaction.id, billId); + toast.success('Transaction matched to bill.'); + setMatchOpen(false); + setMatchTransaction(null); + await loadTransactions(); + } catch (err) { + toast.error(err.message || 'Transaction match failed.'); + } finally { + setActionId(null); + } + }; + + return ( + +
+
+
+ {TRANSACTION_FILTERS.map(item => ( + + ))} +
+ +
+ +
+ {loading ? ( +
Loading transactions…
+ ) : transactions.length === 0 ? ( +
+ No transactions found for this filter. +
+ ) : ( + + + + + + + + + + + + {transactions.map(tx => { + const status = transactionStatus(tx); + const busy = actionId?.endsWith(`:${tx.id}`); + return ( + + + + + + + + ); + })} + +
DateTransactionMatchAmountActions
+ {transactionDate(tx)} + +
+

{transactionTitle(tx)}

+

+ {[tx.description, tx.account_name, tx.source_label].filter(Boolean).join(' Β· ') || 'β€”'} +

+
+ 		+
+ + {tx.matched_bill_name ? ( + {tx.matched_bill_name} + ) : ( + No bill linked + )} +
+ 		+ {formatTransactionAmount(tx.amount, tx.currency)} + +
+ {status === 'ignored' ? ( + + ) : ( + <> + {status === 'matched' ? ( + + ) : ( + + )} + + + )} +
+
+ )} +
+
+ + + + ); +} + // ─── Section 1: Import Transaction CSV ─────────────────────────────────────── const CSV_MAPPING_FIELDS = [ @@ -356,34 +743,207 @@ function canCommitCsvMapping(mapping) { return !!mapping?.posted_date && !!(mapping.amount || mapping.debit_amount || mapping.credit_amount); } -function CsvMappingSelect({ field, label, headers, mapping, onChange }) { +const CSV_IMPORT_STEPS = ['Upload', 'Preview', 'Map', 'Commit', 'Results']; + +function csvImportStepIndex(preview, mapping, commitState) { + if (commitState.status === 'done') return 4; + if (commitState.status === 'loading') return 3; + if (preview.status === 'ready') return canCommitCsvMapping(mapping) ? 3 : 2; + if (preview.status === 'loading' || preview.status === 'error') return 1; + return 0; +} + +function CsvImportStepper({ activeIndex }) { + return ( +
+ {CSV_IMPORT_STEPS.map((step, index) => { + const complete = index < activeIndex; + const active = index === activeIndex; + return ( +
+ + {complete ? : index + 1} + + {step} +
+ ); + })} +
+ ); +} + +function csvFieldRequirement(field, mapping) { + if (field === 'posted_date') return 'Required'; + if (['amount', 'debit_amount', 'credit_amount'].includes(field)) { + return canCommitCsvMapping({ ...mapping, posted_date: mapping?.posted_date || '__date__' }) + ? 'Amount source' + : 'One required'; + } + return 'Optional'; +} + +function csvFieldSamples(preview, header) { + if (!header) return []; + const values = []; + for (const row of preview?.sampleRows || []) { + const value = String(row?.[header] || '').trim(); + if (value && !values.includes(value)) values.push(value); + if (values.length >= 3) break; + } + return values; +} + +function CsvMappingRow({ field, label, preview, mapping, onChange, disabled = false }) { + const headers = preview?.headers || []; + const suggested = preview?.suggestedMapping?.[field] || ''; const current = mapping[field] || ''; const used = new Set(Object.entries(mapping) .filter(([key, value]) => key !== field && value) .map(([, value]) => value)); + const requirement = csvFieldRequirement(field, mapping); + const missingRequired = (requirement === 'Required' || requirement === 'One required') && !current; + const samples = csvFieldSamples(preview, current); + const suggestedAvailable = suggested && suggested !== current && !used.has(suggested); return ( -
-
- -
- - + + +
+
+ +
+ + +
@@ -563,61 +1146,55 @@ export function ImportTransactionCsvSection({ onHistoryRefresh }) { {preview.status === 'ready' && preview.data && (
-
- - - -
+
+
+
+

CSV Preview

+

{file?.name || 'Transaction CSV'}

+
+
+ + + +
+
- {preview.data.errors?.length > 0 && ( -
-

Review mapping

-
    - {preview.data.errors.map((issue, i) => ( -
  • - - {issue.message || JSON.stringify(issue)} -
    • - ))} -
-
- )} - -
-
-

Column mapping

- - Posted date and one amount mapping are required. - -
-
- {mappingFields.map(field => ( - - ))} -
- {!canCommitCsvMapping(mapping) && ( -

- Map a posted date column and either amount, debit amount, or credit amount before importing. -

+ {preview.data.errors?.length > 0 && ( +
+

Review mapping

+
    + {preview.data.errors.map((issue, i) => ( +
  • + + {issue.message || JSON.stringify(issue)} +
    • + ))} +
+
)} -
-
-

Sample rows

+ +
-

- Duplicate rows are skipped using a CSV transaction ID when available, otherwise a stable row hash. -

+
+

+ {canCommitCsvMapping(mapping) ? 'Ready to commit' : 'Mapping incomplete'} +

+

+ Duplicates are skipped using a CSV transaction ID when available, otherwise a stable row hash. +

+
{commitState.status === 'done' ? (
{skippedRows.length > 0 && (
-

Skipped duplicates

-
    +

    Skipped duplicates ({skippedRows.length})

    +
      {skippedRows.map(row => ( -
    • Row {row.row}: {row.provider_transaction_id}
      • +
    • + Row {row.row}: {row.provider_transaction_id} +
      • ))}
)} {failedRows.length > 0 && (
-

Failed rows

-
    - {failedRows.map(row => ( -
  • Row {row.row}: {row.message}
    • +

    Failed rows ({failedRows.length})

    +
      + {failedRows.map((row, index) => ( +
    • +

      Row {row.row}: {row.message}

      + {row.details?.length > 0 && ( +
        + {row.details.map((detail, detailIndex) => ( +
      • {formatCsvRowDetail(detail)}
        • + ))} +
      + )} +
      • ))}
@@ -2324,6 +2912,7 @@ function SeedDemoDataSection({ onSeeded }) { export default function DataPage() { const [history, setHistory] = useState(null); const [historyLoading, setHistoryLoading] = useState(true); + const [transactionRefreshKey, setTransactionRefreshKey] = useState(0); const loadHistory = async () => { setHistoryLoading(true); @@ -2339,6 +2928,11 @@ export default function DataPage() { useEffect(() => { loadHistory(); }, []); + const handleTransactionImportComplete = () => { + loadHistory(); + setTransactionRefreshKey(key => key + 1); + }; + return (
@@ -2354,7 +2948,8 @@ export default function DataPage() {
- + +
diff --git a/db/database.js b/db/database.js index d0d13f3..d6a6809 100644 --- a/db/database.js +++ b/db/database.js @@ -984,6 +984,43 @@ function reconcileLegacyMigrations() { ensureTransactionFoundationSchema(db); console.log('[migration] transaction foundation tables ensured'); } + }, + { + version: 'v0.61', + description: 'payments: one active payment per linked transaction', + check: function() { + return !!db.prepare("SELECT name FROM sqlite_master WHERE type='index' AND name='idx_payments_transaction_active'").get(); + }, + run: function() { + db.exec(` + CREATE UNIQUE INDEX IF NOT EXISTS idx_payments_transaction_active + ON payments(transaction_id) + WHERE transaction_id IS NOT NULL AND deleted_at IS NULL + `); + console.log('[migration] payments: transaction active unique index ensured'); + } + }, + { + version: 'v0.62', + description: 'matches: rejected transaction match suggestions', + check: function() { + return !!db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='match_suggestion_rejections'").get(); + }, + run: function() { + db.exec(` + CREATE TABLE IF NOT EXISTS match_suggestion_rejections ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + transaction_id INTEGER NOT NULL REFERENCES transactions(id) ON DELETE CASCADE, + bill_id INTEGER NOT NULL REFERENCES bills(id) ON DELETE CASCADE, + rejected_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(user_id, transaction_id, bill_id) + ); + CREATE INDEX IF NOT EXISTS idx_match_suggestion_rejections_user + ON match_suggestion_rejections(user_id, transaction_id, bill_id); + `); + console.log('[migration] match suggestion rejections table ensured'); + } } ]; @@ -1699,6 +1736,39 @@ function runMigrations() { ensureTransactionFoundationSchema(db); console.log('[migration] transaction foundation tables ensured'); } + }, + { + version: 'v0.61', + description: 'payments: one active payment per linked transaction', + dependsOn: ['v0.60'], + run: function() { + db.exec(` + CREATE UNIQUE INDEX IF NOT EXISTS idx_payments_transaction_active + ON payments(transaction_id) + WHERE transaction_id IS NOT NULL AND deleted_at IS NULL + `); + console.log('[migration] payments: transaction active unique index ensured'); + } + }, + { + version: 'v0.62', + description: 'matches: rejected transaction match suggestions', + dependsOn: ['v0.61'], + run: function() { + db.exec(` + CREATE TABLE IF NOT EXISTS match_suggestion_rejections ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + transaction_id INTEGER NOT NULL REFERENCES transactions(id) ON DELETE CASCADE, + bill_id INTEGER NOT NULL REFERENCES bills(id) ON DELETE CASCADE, + rejected_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(user_id, transaction_id, bill_id) + ); + CREATE INDEX IF NOT EXISTS idx_match_suggestion_rejections_user + ON match_suggestion_rejections(user_id, transaction_id, bill_id); + `); + console.log('[migration] match suggestion rejections table ensured'); + } } ]; @@ -2111,6 +2181,17 @@ const ROLLBACK_SQL_MAP = { 'DROP TABLE IF EXISTS data_sources', ] }, + 'v0.61': { + description: 'payments: one active payment per linked transaction', + sql: ['DROP INDEX IF EXISTS idx_payments_transaction_active'] + }, + 'v0.62': { + description: 'matches: rejected transaction match suggestions', + sql: [ + 'DROP INDEX IF EXISTS idx_match_suggestion_rejections_user', + 'DROP TABLE IF EXISTS match_suggestion_rejections', + ] + }, 'v0.51': { description: 'bills: snowball_exempt column', sql: ['ALTER TABLE bills DROP COLUMN snowball_exempt'] diff --git a/db/schema.sql b/db/schema.sql index 122ad26..834c65a 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -50,6 +50,7 @@ CREATE TABLE IF NOT EXISTS payments ( balance_delta REAL, payment_source TEXT NOT NULL DEFAULT 'manual', transaction_id INTEGER, + deleted_at TEXT, created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')) ); @@ -178,6 +179,9 @@ CREATE INDEX IF NOT EXISTS idx_notifications_lookup ON notifications(bill_id, us CREATE INDEX IF NOT EXISTS idx_bills_active ON bills(active); CREATE INDEX IF NOT EXISTS idx_payments_bill_id ON payments(bill_id); CREATE INDEX IF NOT EXISTS idx_payments_paid_date ON payments(paid_date); +CREATE UNIQUE INDEX IF NOT EXISTS idx_payments_transaction_active + ON payments(transaction_id) + WHERE transaction_id IS NOT NULL AND deleted_at IS NULL; CREATE INDEX IF NOT EXISTS idx_sessions_user_id ON sessions(user_id); CREATE INDEX IF NOT EXISTS idx_sessions_expires ON sessions(expires_at); CREATE INDEX IF NOT EXISTS idx_data_sources_user_type ON data_sources(user_id, type, status); @@ -193,6 +197,18 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_transactions_provider_dedupe ON transactions (data_source_id, provider_transaction_id) WHERE provider_transaction_id IS NOT NULL; +CREATE TABLE IF NOT EXISTS match_suggestion_rejections ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + transaction_id INTEGER NOT NULL REFERENCES transactions(id) ON DELETE CASCADE, + bill_id INTEGER NOT NULL REFERENCES bills(id) ON DELETE CASCADE, + rejected_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(user_id, transaction_id, bill_id) +); + +CREATE INDEX IF NOT EXISTS idx_match_suggestion_rejections_user + ON match_suggestion_rejections(user_id, transaction_id, bill_id); + CREATE TABLE IF NOT EXISTS monthly_bill_state ( id INTEGER PRIMARY KEY AUTOINCREMENT, bill_id INTEGER NOT NULL REFERENCES bills(id) ON DELETE CASCADE, diff --git a/docs/Engineering_Reference_Manual.md b/docs/Engineering_Reference_Manual.md index 5d72927..85b62ef 100644 --- a/docs/Engineering_Reference_Manual.md +++ b/docs/Engineering_Reference_Manual.md @@ -1,8 +1,8 @@ # Engineering Reference Manual β€” Bill Tracker **Status:** Current code reference -**Last Updated:** 2026-05-10 -**Version:** 0.23.2 +**Last Updated:** 2026-05-16 +**Version:** 0.28.1 **Primary stack:** Node.js + Express, React + Vite, Tailwind CSS + shadcn/ui, Sonner, SQLite via `better-sqlite3` This manual reflects the current application code in `server.js`, `routes/`, `services/`, `middleware/`, `db/`, `client/`, `package.json`, `Dockerfile`, and `docker-compose.yml`. It is written as a current-state reference, not a changelog. @@ -34,7 +34,7 @@ Runtime flow: - `server.js` β€” Express entry point and route mounting. - `routes/` β€” HTTP API handlers. -- `services/` β€” auth, OIDC, backup, cleanup, notification, import, status, audit business logic. +- `services/` β€” auth, OIDC, backup, cleanup, notification, import, status, audit, transaction, CSV import business logic. - `middleware/` β€” auth guards, CSRF, rate limits, security headers, error formatting. - `db/schema.sql` β€” base SQLite schema. - `db/database.js` β€” DB connection, migrations, defaults, settings, rollback support. @@ -197,15 +197,34 @@ Settings are stored in `settings`; run results are stored as JSON. - Sanitizes categories, bills, payments, monthly state, and starting amounts. - Preview stores an import session; apply maps export IDs to current user-owned IDs. -### `services/statusService.js` +### `services/transactionService.js` -Shared tracker/calendar logic: +Transaction data source and transaction row helpers: -- `resolveDueDate(bill, year, month)` clamps due day to month length. -- `resolveBucket(bill)` uses bucket or due-day threshold. -- `getCycleRange(year, month)` returns first/last day of month. -- `calculateStatus(...)` returns paid/autodraft/upcoming/due/overdue-style status. -- `buildTrackerRow(...)` returns row data for the monthly tracker. +- `ensureManualDataSource(db, userId)` creates/retrieves a user-specific manual data source (`type='manual', provider='manual', name='Manual Entry'`). +- `decorateDataSource(row)` removes `encrypted_secret`, adds `source_label` and `source_type_label`. +- `decorateTransaction(row)` adds `source_label`, `source_type_label`, and embedded `data_source` object with safe fields. +- `getSourceTypeLabel(type)` returns labels: `manual` β†’ Manual, `file_import` β†’ File import, `provider_sync` β†’ Provider sync. +- `sourceLabel(source)` constructs human-readable source labels for manual entries or provider names. + +### `services/csvTransactionImportService.js` + +CSV import workflow for transactions: + +- Parses CSV with quoted-field support and quote doubling. +- `previewCsvTransactions(userId, buffer, options)` returns headers, sample rows, suggested field mapping, errors, and creates a 24-hour TTL import session (max 25k rows). +- `suggestMapping(headers)` auto-detects field mappings from header names against `posted_date`, `transacted_at`, `amount`, `description`, `payee`, `memo`, `category`, `account`, `transaction_type`, `currency`, etc. +- `commitCsvTransactions(userId, importSessionId, mapping)` imports rows into the `transactions` table with `source_type='file_import'`, auto-creates a CSV data source and financial accounts per unique account name. +- Stable deduplication via SHA-256 hash: `csv:id:` prefix for explicit transaction IDs, `csv:hash:` prefix from date+amount+description+payee+account. +- Imports record to `import_history` with counts and details. +- `FIELD_LABELS` maps field keys to user-friendly labels for validation messages. + +### `services/paymentValidation.js` + +Payment validation helpers for source tracking and matching: + +- Validates `payment_source` values (`manual`, `file_import`, `provider_sync`). +- Supports transaction linking via `transaction_id` when available. ### `services/auditService.js` @@ -797,6 +816,13 @@ SQLite uses WAL mode and foreign keys. Base schema is in `db/schema.sql`; `db/da - `is_seeded INTEGER NOT NULL DEFAULT 0` - `cycle_type TEXT NOT NULL DEFAULT 'monthly'` - `cycle_day TEXT` +- `current_balance REAL` +- `minimum_payment REAL` +- `snowball_order INTEGER` +- `snowball_include INTEGER NOT NULL DEFAULT 0` +- `snowball_exempt INTEGER NOT NULL DEFAULT 0` +- `auto_mark_paid INTEGER NOT NULL DEFAULT 0` +- `deleted_at TEXT` #### `payments` @@ -806,6 +832,9 @@ SQLite uses WAL mode and foreign keys. Base schema is in `db/schema.sql`; `db/da - `paid_date TEXT NOT NULL` - `method TEXT` - `notes TEXT` +- `balance_delta REAL` +- `payment_source TEXT NOT NULL DEFAULT 'manual'` +- `transaction_id INTEGER` - `created_at TEXT DEFAULT datetime('now')` - `updated_at TEXT DEFAULT datetime('now')` - `deleted_at TEXT` diff --git a/package.json b/package.json index b840f4e..7148406 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bill-tracker", - "version": "0.28.1", + "version": "0.28.01", "description": "Monthly bill tracking system", "main": "server.js", "scripts": { diff --git a/routes/bills.js b/routes/bills.js index fb57994..378324c 100644 --- a/routes/bills.js +++ b/routes/bills.js @@ -13,6 +13,7 @@ const { const { amortizationSchedule, debtAprSnapshot } = require('../services/aprService'); const { standardizeError } = require('../middleware/errorFormatter'); const { validatePaymentInput } = require('../services/paymentValidation'); +const { decorateTransaction } = require('../services/transactionService'); // ── GET /api/bills ──────────────────────────────────────────────────────────── router.get('/', (req, res) => { @@ -395,6 +396,64 @@ router.get('/:id/payments', (req, res) => { }); }); +// ── GET /api/bills/:id/transactions ────────────────────────────────────────── +router.get('/:id/transactions', (req, res) => { + const db = getDb(); + const billId = parseInt(req.params.id, 10); + if (!Number.isInteger(billId)) { + return res.status(400).json(standardizeError('bill_id must be an integer', 'VALIDATION_ERROR', 'bill_id')); + } + + const bill = db.prepare('SELECT id, name FROM bills WHERE id = ? AND user_id = ? AND deleted_at IS NULL').get(billId, req.user.id); + if (!bill) return res.status(404).json(standardizeError('Bill not found', 'NOT_FOUND', 'bill_id')); + + const rows = db.prepare(` + SELECT + t.id, t.user_id, t.data_source_id, t.account_id, t.provider_transaction_id, + t.source_type, t.transaction_type, t.posted_date, t.transacted_at, t.amount, + t.currency, t.description, t.payee, t.memo, t.category, t.matched_bill_id, + t.match_status, t.ignored, t.created_at, t.updated_at, + ds.type AS data_source_type, ds.provider AS data_source_provider, + ds.name AS data_source_name, ds.status AS data_source_status, + fa.name AS account_name, fa.org_name AS account_org_name, + fa.account_type AS account_type, + b.name AS matched_bill_name, + p.id AS linked_payment_id, + p.amount AS linked_payment_amount, + p.paid_date AS linked_payment_date, + p.payment_source AS linked_payment_source, + p.method AS linked_payment_method + FROM transactions t + LEFT JOIN data_sources ds ON ds.id = t.data_source_id AND ds.user_id = t.user_id + LEFT JOIN financial_accounts fa ON fa.id = t.account_id AND fa.user_id = t.user_id + LEFT JOIN bills b ON b.id = t.matched_bill_id AND b.user_id = t.user_id AND b.deleted_at IS NULL + LEFT JOIN payments p ON p.transaction_id = t.id AND p.bill_id = ? AND p.deleted_at IS NULL + WHERE t.user_id = ? + AND t.matched_bill_id = ? + AND t.match_status = 'matched' + AND t.ignored = 0 + ORDER BY COALESCE(t.posted_date, substr(t.transacted_at, 1, 10), t.created_at) DESC, t.id DESC + `).all(billId, req.user.id, billId); + + const transactions = rows.map(row => decorateTransaction({ + ...row, + linked_payment: row.linked_payment_id ? { + id: row.linked_payment_id, + amount: row.linked_payment_amount, + paid_date: row.linked_payment_date, + payment_source: row.linked_payment_source, + method: row.linked_payment_method, + } : null, + })); + + res.json({ + bill_id: billId, + bill_name: bill.name, + total: transactions.length, + transactions, + }); +}); + // ── POST /api/bills/:id/toggle-paid β€” toggle Paid/Unpaid status ────────────── router.post('/:id/toggle-paid', (req, res) => { const db = getDb(); diff --git a/routes/import.js b/routes/import.js index 47dc387..a8bc863 100644 --- a/routes/import.js +++ b/routes/import.js @@ -240,7 +240,7 @@ router.post('/csv/commit', requireDataImportEnabled, express.json({ limit: '1mb' // ─── GET /api/import/history ────────────────────────────────────────────────── // Returns the authenticated user's import history (last 100 imports). -router.get('/history', (req, res) => { +router.get('/history', requireDataImportEnabled, (req, res) => { try { const history = getImportHistory(req.user.id); res.json({ history }); diff --git a/routes/matches.js b/routes/matches.js new file mode 100644 index 0000000..180a8c6 --- /dev/null +++ b/routes/matches.js @@ -0,0 +1,34 @@ +const router = require('express').Router(); +const { standardizeError } = require('../middleware/errorFormatter'); +const { + listMatchSuggestions, + rejectMatchSuggestion, +} = require('../services/matchSuggestionService'); + +function sendMatchError(res, err, fallbackMessage = 'Match operation failed') { + if (err.status) { + return res.status(err.status).json(standardizeError(err.message, err.code || 'MATCH_ERROR', err.field)); + } + console.error('[matches] service error:', err.stack || err.message); + return res.status(500).json(standardizeError(fallbackMessage, 'MATCH_ERROR')); +} + +// GET /api/matches/suggestions +router.get('/suggestions', (req, res) => { + try { + res.json(listMatchSuggestions(req.user.id, req.query)); + } catch (err) { + return sendMatchError(res, err, 'Match suggestions failed'); + } +}); + +// POST /api/matches/:id/reject +router.post('/:id/reject', (req, res) => { + try { + res.json(rejectMatchSuggestion(req.user.id, req.params.id)); + } catch (err) { + return sendMatchError(res, err, 'Rejecting match suggestion failed'); + } +}); + +module.exports = router; diff --git a/routes/payments.js b/routes/payments.js index f88f306..ef7ce3d 100644 --- a/routes/payments.js +++ b/routes/payments.js @@ -7,6 +7,19 @@ const { validatePaymentInput } = require('../services/paymentValidation'); const { getCycleRange, resolveDueDate } = require('../services/statusService'); const LIVE = 'deleted_at IS NULL'; // filter for non-deleted payments +const TRANSACTION_MATCH_SOURCE = 'transaction_match'; + +function isTransactionLinkedPayment(payment) { + return payment?.payment_source === TRANSACTION_MATCH_SOURCE || payment?.transaction_id != null; +} + +function rejectTransactionLinkedPayment(res) { + return res.status(409).json(standardizeError( + 'Transaction-linked payments must be changed through transaction match controls', + 'TRANSACTION_PAYMENT_LOCKED', + 'transaction_id', + )); +} function parseYearMonth(body) { const year = parseInt(body.year, 10); @@ -349,6 +362,7 @@ router.put('/:id', (req, res) => { const db = getDb(); const existing = db.prepare(`SELECT p.* FROM payments p JOIN bills b ON b.id = p.bill_id WHERE p.id = ? AND p.${LIVE} AND b.user_id = ? AND b.deleted_at IS NULL`).get(req.params.id, req.user.id); if (!existing) return res.status(404).json(standardizeError('Payment not found', 'NOT_FOUND', 'id')); + if (isTransactionLinkedPayment(existing)) return rejectTransactionLinkedPayment(res); const { amount, paid_date, method, notes, payment_source } = req.body; const validation = validatePaymentInput( @@ -405,6 +419,7 @@ router.delete('/:id', (req, res) => { const db = getDb(); const payment = db.prepare(`SELECT p.* FROM payments p JOIN bills b ON b.id = p.bill_id WHERE p.id = ? AND p.${LIVE} AND b.user_id = ? AND b.deleted_at IS NULL`).get(req.params.id, req.user.id); if (!payment) return res.status(404).json(standardizeError('Payment not found', 'NOT_FOUND', 'id')); + if (isTransactionLinkedPayment(payment)) return rejectTransactionLinkedPayment(res); // Reverse any balance delta that was stored when this payment was created if (payment.balance_delta != null) { @@ -424,6 +439,7 @@ router.post('/:id/restore', (req, res) => { const db = getDb(); const payment = db.prepare('SELECT p.* FROM payments p JOIN bills b ON b.id = p.bill_id WHERE p.id = ? AND p.deleted_at IS NOT NULL AND b.user_id = ? AND b.deleted_at IS NULL').get(req.params.id, req.user.id); if (!payment) return res.status(404).json(standardizeError('Deleted payment not found', 'NOT_FOUND', 'id')); + if (isTransactionLinkedPayment(payment)) return rejectTransactionLinkedPayment(res); // Re-apply the balance delta (undo the reversal done on delete) if (payment.balance_delta != null) { diff --git a/routes/profile.js b/routes/profile.js index 931bfda..18f10ce 100644 --- a/routes/profile.js +++ b/routes/profile.js @@ -9,10 +9,22 @@ const { getDb, getSetting } = require('../db/database'); const { hashPassword, invalidateOtherSessions, rotateSessionId, COOKIE_NAME, cookieOpts } = require('../services/authService'); const { getImportHistory } = require('../services/spreadsheetImportService'); const { logAudit } = require('../services/auditService'); +const { standardizeError } = require('../middleware/errorFormatter'); // All profile routes require authentication β€” enforced in server.js. // req.user is always the signed-in user; user_id is never accepted from the body. +function dataImportEnabled() { + return String(process.env.DATA_IMPORT_ENABLED ?? 'true').toLowerCase() !== 'false'; +} + +function requireDataImportEnabled(req, res, next) { + if (!dataImportEnabled()) { + return res.status(403).json(standardizeError('Data import is disabled by DATA_IMPORT_ENABLED=false', 'FORBIDDEN')); + } + next(); +} + // ── GET /api/profile ────────────────────────────────────────────────────────── // Returns safe profile data for the signed-in user. // Never returns password_hash, session tokens, or secrets. @@ -281,7 +293,7 @@ router.get('/exports', (req, res) => { // ── GET /api/profile/import-history ────────────────────────────────────────── // Returns the signed-in user's import history. // Delegates to the same service as GET /api/import/history. -router.get('/import-history', (req, res) => { +router.get('/import-history', requireDataImportEnabled, (req, res) => { try { const history = getImportHistory(req.user.id); res.json({ history }); diff --git a/routes/transactions.js b/routes/transactions.js index 161ac22..02635ff 100644 --- a/routes/transactions.js +++ b/routes/transactions.js @@ -6,9 +6,16 @@ const { ensureManualDataSource, getTransactionForUser, } = require('../services/transactionService'); +const { + ignoreTransaction, + matchTransactionToBill, + unignoreTransaction, + unmatchTransaction, +} = require('../services/transactionMatchService'); const MATCH_STATUSES = new Set(['unmatched', 'matched', 'ignored']); const SOURCE_TYPES = new Set(['manual', 'file_import', 'provider_sync']); +const MATCH_CONTROL_FIELDS = ['matched_bill_id', 'match_status', 'ignored']; const TEXT_FIELDS = { transaction_type: 64, currency: 16, @@ -243,6 +250,24 @@ function selectedTransaction(db, userId, id) { return decorateTransaction(getTransactionForUser(db, userId, id)); } +function rejectDirectMatchState(body = {}) { + const field = MATCH_CONTROL_FIELDS.find(name => hasOwn(body, name)); + if (!field) return null; + return standardizeError( + 'Use the transaction match, unmatch, ignore, or unignore endpoint to change match state', + 'VALIDATION_ERROR', + field, + ); +} + +function sendTransactionServiceError(res, err, fallbackMessage = 'Transaction operation failed') { + if (err.status) { + return res.status(err.status).json(standardizeError(err.message, err.code || 'TRANSACTION_ERROR', err.field)); + } + console.error('[transactions] service error:', err.stack || err.message); + return res.status(500).json(standardizeError(fallbackMessage, 'TRANSACTION_ERROR')); +} + // GET /api/transactions router.get('/', (req, res) => { const db = getDb(); @@ -346,6 +371,9 @@ router.get('/', (req, res) => { // POST /api/transactions/manual router.post('/manual', (req, res) => { const db = getDb(); + const directMatchState = rejectDirectMatchState(req.body); + if (directMatchState) return res.status(400).json(directMatchState); + const validation = normalizeTransactionFields(db, req.user.id, req.body); if (validation.error) return res.status(validation.status || 400).json(validation.error); const tx = validation.normalized; @@ -384,6 +412,9 @@ router.post('/manual', (req, res) => { // PUT /api/transactions/:id router.put('/:id', (req, res) => { const db = getDb(); + const directMatchState = rejectDirectMatchState(req.body); + if (directMatchState) return res.status(400).json(directMatchState); + const id = parseInteger(req.params.id, 'id'); if (id.error) return res.status(400).json(id.error); @@ -442,55 +473,55 @@ router.delete('/:id', (req, res) => { if (!existing) return res.status(404).json(standardizeError('Transaction not found', 'NOT_FOUND', 'id')); db.transaction(() => { - db.prepare(` - UPDATE payments - SET transaction_id = NULL, updated_at = datetime('now') - WHERE transaction_id = ? - AND bill_id IN (SELECT id FROM bills WHERE user_id = ?) - `).run(id.value, req.user.id); + unmatchTransaction(req.user.id, id.value); db.prepare('DELETE FROM transactions WHERE id = ? AND user_id = ?').run(id.value, req.user.id); })(); res.json({ success: true, deleted: true, id: id.value }); }); +// POST /api/transactions/:id/match +router.post('/:id/match', (req, res) => { + try { + const result = matchTransactionToBill( + req.user.id, + req.params.id, + req.body?.billId ?? req.body?.bill_id, + ); + res.json(result); + } catch (err) { + return sendTransactionServiceError(res, err, 'Transaction match failed'); + } +}); + +// POST /api/transactions/:id/unmatch +router.post('/:id/unmatch', (req, res) => { + try { + const result = unmatchTransaction(req.user.id, req.params.id); + res.json(result); + } catch (err) { + return sendTransactionServiceError(res, err, 'Transaction unmatch failed'); + } +}); + // POST /api/transactions/:id/ignore router.post('/:id/ignore', (req, res) => { - const db = getDb(); - const id = parseInteger(req.params.id, 'id'); - if (id.error) return res.status(400).json(id.error); - if (!getTransactionForUser(db, req.user.id, id.value)) { - return res.status(404).json(standardizeError('Transaction not found', 'NOT_FOUND', 'id')); + try { + const result = ignoreTransaction(req.user.id, req.params.id); + res.json(result.transaction); + } catch (err) { + return sendTransactionServiceError(res, err, 'Transaction ignore failed'); } - - db.prepare(` - UPDATE transactions - SET ignored = 1, match_status = 'ignored', matched_bill_id = NULL, updated_at = datetime('now') - WHERE id = ? AND user_id = ? - `).run(id.value, req.user.id); - - res.json(selectedTransaction(db, req.user.id, id.value)); }); // POST /api/transactions/:id/unignore router.post('/:id/unignore', (req, res) => { - const db = getDb(); - const id = parseInteger(req.params.id, 'id'); - if (id.error) return res.status(400).json(id.error); - if (!getTransactionForUser(db, req.user.id, id.value)) { - return res.status(404).json(standardizeError('Transaction not found', 'NOT_FOUND', 'id')); + try { + const result = unignoreTransaction(req.user.id, req.params.id); + res.json(result.transaction); + } catch (err) { + return sendTransactionServiceError(res, err, 'Transaction unignore failed'); } - - db.prepare(` - UPDATE transactions - SET ignored = 0, - match_status = 'unmatched', - matched_bill_id = NULL, - updated_at = datetime('now') - WHERE id = ? AND user_id = ? - `).run(id.value, req.user.id); - - res.json(selectedTransaction(db, req.user.id, id.value)); }); module.exports = router; diff --git a/server.js b/server.js index 4b81d95..5ce643e 100644 --- a/server.js +++ b/server.js @@ -85,6 +85,7 @@ app.use('/api/bills', csrfMiddleware, requireAuth, requireUser, require( app.use('/api/payments', csrfMiddleware, requireAuth, requireUser, require('./routes/payments')); app.use('/api/data-sources', csrfMiddleware, requireAuth, requireUser, require('./routes/dataSources')); app.use('/api/transactions', csrfMiddleware, requireAuth, requireUser, require('./routes/transactions')); +app.use('/api/matches', csrfMiddleware, requireAuth, requireUser, require('./routes/matches')); app.use('/api/categories', csrfMiddleware, requireAuth, requireUser, require('./routes/categories')); app.use('/api/settings', csrfMiddleware, requireAuth, requireUser, require('./routes/settings')); app.use('/api/user', csrfMiddleware, requireAuth, requireUser, require('./routes/user')); diff --git a/services/matchSuggestionService.js b/services/matchSuggestionService.js new file mode 100644 index 0000000..d2c056d --- /dev/null +++ b/services/matchSuggestionService.js @@ -0,0 +1,334 @@ +'use strict'; + +const { getDb } = require('../db/database'); +const { getCycleRange, resolveDueDate } = require('./statusService'); +const { decorateTransaction } = require('./transactionService'); + +function suggestionError(status, message, code, field = null) { + const err = new Error(message); + err.status = status; + err.code = code; + err.field = field; + return err; +} + +function normalizeId(value, field) { + const id = typeof value === 'number' ? value : Number(value); + if (!Number.isSafeInteger(id) || id <= 0) { + throw suggestionError(400, `${field} must be a positive integer`, 'VALIDATION_ERROR', field); + } + return id; +} + +function suggestionId(transactionId, billId) { + return `${transactionId}:${billId}`; +} + +function parseSuggestionId(id) { + const match = /^(\d+):(\d+)$/.exec(String(id || '').trim()); + if (!match) { + throw suggestionError(400, 'Suggestion id must be transactionId:billId', 'VALIDATION_ERROR', 'id'); + } + return { + transactionId: normalizeId(match[1], 'transaction_id'), + billId: normalizeId(match[2], 'bill_id'), + }; +} + +function textKey(value) { + return String(value || '') + .toLowerCase() + .replace(/[^a-z0-9]+/g, ' ') + .trim(); +} + +function transactionDate(transaction) { + const date = transaction.posted_date || String(transaction.transacted_at || '').slice(0, 10); + return /^\d{4}-\d{2}-\d{2}$/.test(date) ? date : null; +} + +function dateParts(date) { + const [year, month] = String(date).split('-').map(Number); + return { year, month }; +} + +function diffDays(a, b) { + const left = new Date(`${a}T00:00:00Z`).getTime(); + const right = new Date(`${b}T00:00:00Z`).getTime(); + if (!Number.isFinite(left) || !Number.isFinite(right)) return null; + return Math.abs(Math.round((left - right) / 86400000)); +} + +function amountDollars(transaction) { + const cents = Number(transaction.amount); + return Number.isFinite(cents) ? Math.abs(cents) / 100 : 0; +} + +function addAmountScore(score, reasons, transaction, bill) { + const txAmount = amountDollars(transaction); + const expected = Number(bill.expected_amount) || 0; + if (txAmount <= 0 || expected <= 0) return score; + + const delta = Math.abs(txAmount - expected); + const pct = delta / expected; + if (delta <= 0.01) { + reasons.push('amount matches'); + return score + 40; + } + if (delta <= 1) { + reasons.push('amount within $1'); + return score + 32; + } + if (delta <= 5 || pct <= 0.05) { + reasons.push('amount close to bill'); + return score + 22; + } + if (pct <= 0.15) { + reasons.push('amount within 15%'); + return score + 12; + } + return score; +} + +function addDateScore(score, reasons, transaction, bill) { + const postedDate = transactionDate(transaction); + if (!postedDate) return score; + + const { year, month } = dateParts(postedDate); + const dueDate = resolveDueDate(bill, year, month); + if (!dueDate) return score; + + const distance = diffDays(postedDate, dueDate); + if (distance === null) return score; + if (distance <= 1) { + reasons.push('date within 1 day'); + return score + 25; + } + if (distance <= 3) { + reasons.push(`date within ${distance} days`); + return score + 20; + } + if (distance <= 7) { + reasons.push('date within 7 days'); + return score + 12; + } + return score; +} + +function addNameScore(score, reasons, transaction, bill) { + const billName = textKey(bill.name); + if (!billName) return score; + + const payee = textKey(transaction.payee); + const description = textKey(transaction.description); + const memo = textKey(transaction.memo); + + if (payee && (payee.includes(billName) || billName.includes(payee))) { + reasons.push('payee contains bill name'); + score += 22; + } + if (description && (description.includes(billName) || billName.includes(description))) { + reasons.push('description contains bill name'); + score += 18; + } + if (memo && (memo.includes(billName) || billName.includes(memo))) { + reasons.push('memo contains bill name'); + score += 8; + } + return score; +} + +function addPriorMatchScore(score, reasons, transaction, bill, priorMatchKeys) { + const payee = textKey(transaction.payee); + const description = textKey(transaction.description); + if ( + (payee && priorMatchKeys.has(`${bill.id}:payee:${payee}`)) || + (description && priorMatchKeys.has(`${bill.id}:description:${description}`)) + ) { + reasons.push('prior match for this bill'); + return score + 12; + } + return score; +} + +function hasPaymentInTransactionCycle(db, bill, transaction) { + const postedDate = transactionDate(transaction); + if (!postedDate) return false; + const { year, month } = dateParts(postedDate); + const range = getCycleRange(year, month, bill); + if (!range) return false; + + return !!db.prepare(` + SELECT 1 + FROM payments + WHERE bill_id = ? + AND paid_date BETWEEN ? AND ? + AND deleted_at IS NULL + LIMIT 1 + `).get(bill.id, range.start, range.end); +} + +function loadCandidateTransactions(db, userId, transactionId = null) { + const params = [userId]; + const where = [ + 't.user_id = ?', + 't.ignored = 0', + "t.match_status = 'unmatched'", + ]; + if (transactionId) { + where.push('t.id = ?'); + params.push(transactionId); + } + + return db.prepare(` + SELECT + t.id, t.user_id, t.data_source_id, t.account_id, t.provider_transaction_id, + t.source_type, t.transaction_type, t.posted_date, t.transacted_at, t.amount, + t.currency, t.description, t.payee, t.memo, t.category, t.matched_bill_id, + t.match_status, t.ignored, t.created_at, t.updated_at, + ds.type AS data_source_type, ds.provider AS data_source_provider, + ds.name AS data_source_name, ds.status AS data_source_status, + fa.name AS account_name, fa.org_name AS account_org_name, + fa.account_type AS account_type, + b.name AS matched_bill_name + FROM transactions t + LEFT JOIN data_sources ds ON ds.id = t.data_source_id AND ds.user_id = t.user_id + LEFT JOIN financial_accounts fa ON fa.id = t.account_id AND fa.user_id = t.user_id + LEFT JOIN bills b ON b.id = t.matched_bill_id AND b.user_id = t.user_id AND b.deleted_at IS NULL + WHERE ${where.join(' AND ')} + ORDER BY COALESCE(t.posted_date, substr(t.transacted_at, 1, 10), t.created_at) DESC, t.id DESC + LIMIT 100 + `).all(...params).map(decorateTransaction); +} + +function loadBills(db, userId) { + return db.prepare(` + SELECT b.*, c.name AS category_name + FROM bills b + LEFT JOIN categories c ON c.id = b.category_id AND c.deleted_at IS NULL + WHERE b.user_id = ? + AND b.deleted_at IS NULL + AND b.active = 1 + ORDER BY b.name COLLATE NOCASE ASC + `).all(userId); +} + +function loadRejections(db, userId) { + const rows = db.prepare(` + SELECT transaction_id, bill_id + FROM match_suggestion_rejections + WHERE user_id = ? + `).all(userId); + return new Set(rows.map(row => suggestionId(row.transaction_id, row.bill_id))); +} + +function loadPriorMatchKeys(db, userId) { + const rows = db.prepare(` + SELECT matched_bill_id, payee, description + FROM transactions + WHERE user_id = ? + AND matched_bill_id IS NOT NULL + AND match_status = 'matched' + AND ignored = 0 + `).all(userId); + const keys = new Set(); + for (const row of rows) { + const payee = textKey(row.payee); + const description = textKey(row.description); + if (payee) keys.add(`${row.matched_bill_id}:payee:${payee}`); + if (description) keys.add(`${row.matched_bill_id}:description:${description}`); + } + return keys; +} + +function scoreSuggestion(transaction, bill, priorMatchKeys) { + const reasons = []; + let score = 0; + score = addAmountScore(score, reasons, transaction, bill); + score = addDateScore(score, reasons, transaction, bill); + score = addNameScore(score, reasons, transaction, bill); + score = addPriorMatchScore(score, reasons, transaction, bill, priorMatchKeys); + return { score: Math.min(score, 100), reasons }; +} + +function listMatchSuggestions(userId, options = {}) { + const db = getDb(); + const rawTransactionId = options.transactionId ?? options.transaction_id; + const transactionId = rawTransactionId + ? normalizeId(rawTransactionId, 'transaction_id') + : null; + const limit = Math.max(1, Math.min(Number.parseInt(options.limit || '50', 10) || 50, 100)); + + const transactions = loadCandidateTransactions(db, userId, transactionId); + const bills = loadBills(db, userId); + const rejections = loadRejections(db, userId); + const priorMatchKeys = loadPriorMatchKeys(db, userId); + const suggestions = []; + + for (const transaction of transactions) { + for (const bill of bills) { + const id = suggestionId(transaction.id, bill.id); + if (rejections.has(id)) continue; + if (hasPaymentInTransactionCycle(db, bill, transaction)) continue; + + const scored = scoreSuggestion(transaction, bill, priorMatchKeys); + if (scored.score < 20) continue; + + suggestions.push({ + id, + transactionId: transaction.id, + billId: bill.id, + score: scored.score, + reasons: scored.reasons, + transaction, + bill: { + id: bill.id, + name: bill.name, + expected_amount: bill.expected_amount, + due_day: bill.due_day, + category_name: bill.category_name || null, + }, + }); + } + } + + return suggestions + .sort((a, b) => b.score - a.score || a.bill.name.localeCompare(b.bill.name)) + .slice(0, limit); +} + +function rejectMatchSuggestion(userId, id) { + const db = getDb(); + const parsed = parseSuggestionId(id); + + const transaction = db.prepare('SELECT id FROM transactions WHERE id = ? AND user_id = ?').get(parsed.transactionId, userId); + if (!transaction) { + throw suggestionError(404, 'Transaction not found', 'NOT_FOUND', 'transaction_id'); + } + const bill = db.prepare('SELECT id FROM bills WHERE id = ? AND user_id = ? AND deleted_at IS NULL').get(parsed.billId, userId); + if (!bill) { + throw suggestionError(404, 'Bill not found', 'NOT_FOUND', 'bill_id'); + } + + db.prepare(` + INSERT INTO match_suggestion_rejections (user_id, transaction_id, bill_id, rejected_at) + VALUES (?, ?, ?, datetime('now')) + ON CONFLICT(user_id, transaction_id, bill_id) DO UPDATE SET + rejected_at = excluded.rejected_at + `).run(userId, parsed.transactionId, parsed.billId); + + return { + success: true, + id: suggestionId(parsed.transactionId, parsed.billId), + transactionId: parsed.transactionId, + billId: parsed.billId, + rejected: true, + }; +} + +module.exports = { + listMatchSuggestions, + parseSuggestionId, + rejectMatchSuggestion, + suggestionId, +}; diff --git a/services/paymentValidation.js b/services/paymentValidation.js index a9519dd..839cd9f 100644 --- a/services/paymentValidation.js +++ b/services/paymentValidation.js @@ -39,7 +39,7 @@ function validatePositiveAmount(value, field = 'amount') { return { value: amount }; } -const PAYMENT_SOURCES = ['manual', 'file_import', 'provider_sync']; +const PAYMENT_SOURCES = ['manual', 'file_import', 'provider_sync', 'transaction_match']; function validatePaymentSource(value, field = 'payment_source') { if (typeof value !== 'string') { diff --git a/services/transactionMatchService.js b/services/transactionMatchService.js new file mode 100644 index 0000000..103bef4 --- /dev/null +++ b/services/transactionMatchService.js @@ -0,0 +1,313 @@ +'use strict'; + +const { getDb } = require('../db/database'); +const { computeBalanceDelta } = require('./billsService'); +const { + decorateTransaction, + getTransactionForUser, +} = require('./transactionService'); + +const MATCH_PAYMENT_SOURCE = 'transaction_match'; +const MATCH_PAYMENT_METHOD = 'transaction_match'; + +function matchError(status, message, code, field = null) { + const err = new Error(message); + err.status = status; + err.code = code; + err.field = field; + return err; +} + +function normalizeId(value, field) { + const id = typeof value === 'number' ? value : Number(value); + if (!Number.isSafeInteger(id) || id <= 0) { + throw matchError(400, `${field} must be a positive integer`, 'VALIDATION_ERROR', field); + } + return id; +} + +function getOwnedTransaction(db, userId, transactionId) { + const id = normalizeId(transactionId, 'transaction_id'); + const transaction = getTransactionForUser(db, userId, id); + if (!transaction) { + throw matchError(404, 'Transaction not found', 'NOT_FOUND', 'transaction_id'); + } + return transaction; +} + +function getOwnedBill(db, userId, billId) { + const id = normalizeId(billId, 'bill_id'); + const bill = db.prepare(` + SELECT * + FROM bills + WHERE id = ? AND user_id = ? AND deleted_at IS NULL + `).get(id, userId); + if (!bill) { + throw matchError(404, 'Bill not found', 'NOT_FOUND', 'bill_id'); + } + return bill; +} + +function paymentDateForTransaction(transaction) { + const date = transaction.posted_date || String(transaction.transacted_at || '').slice(0, 10); + if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) { + throw matchError( + 400, + 'Transaction must have a posted date before it can be matched to a bill', + 'VALIDATION_ERROR', + 'posted_date', + ); + } + return date; +} + +function paymentAmountForTransaction(transaction) { + const cents = Number(transaction.amount); + if (!Number.isSafeInteger(cents) || cents === 0) { + throw matchError( + 400, + 'Transaction amount must be a non-zero integer number of cents', + 'VALIDATION_ERROR', + 'amount', + ); + } + return Math.round(Math.abs(cents)) / 100; +} + +function getActivePaymentForTransaction(db, userId, transactionId) { + return db.prepare(` + SELECT p.* + FROM payments p + JOIN bills b ON b.id = p.bill_id + WHERE p.transaction_id = ? + AND p.deleted_at IS NULL + AND b.user_id = ? + AND b.deleted_at IS NULL + ORDER BY p.id ASC + LIMIT 1 + `).get(transactionId, userId); +} + +function getPaymentForResponse(db, userId, paymentId) { + if (!paymentId) return null; + return db.prepare(` + SELECT p.* + FROM payments p + JOIN bills b ON b.id = p.bill_id + WHERE p.id = ? + AND b.user_id = ? + `).get(paymentId, userId) || null; +} + +function restorePaymentBalance(db, payment) { + if (!payment || payment.balance_delta == null) return; + const bill = db.prepare('SELECT id, current_balance FROM bills WHERE id = ?').get(payment.bill_id); + if (bill?.current_balance == null) return; + + const restored = Math.max(0, Math.round((Number(bill.current_balance) - Number(payment.balance_delta)) * 100) / 100); + db.prepare("UPDATE bills SET current_balance = ?, updated_at = datetime('now') WHERE id = ?") + .run(restored, bill.id); +} + +function applyPaymentBalance(db, bill, amount) { + const freshBill = db.prepare('SELECT * FROM bills WHERE id = ?').get(bill.id) || bill; + const balCalc = computeBalanceDelta(freshBill, amount); + if (balCalc) { + db.prepare("UPDATE bills SET current_balance = ?, updated_at = datetime('now') WHERE id = ?") + .run(balCalc.new_balance, bill.id); + } + return balCalc?.balance_delta ?? null; +} + +function buildMatchPaymentNotes(transaction, bill) { + const label = transaction.payee || transaction.description || `transaction ${transaction.id}`; + return `Matched transaction to ${bill.name}: ${label}`.slice(0, 500); +} + +function createOrUpdateMatchPayment(db, userId, transaction, bill) { + const amount = paymentAmountForTransaction(transaction); + const paidDate = paymentDateForTransaction(transaction); + const notes = buildMatchPaymentNotes(transaction, bill); + const existingPayment = getActivePaymentForTransaction(db, userId, transaction.id); + + if (existingPayment && existingPayment.payment_source !== MATCH_PAYMENT_SOURCE) { + throw matchError( + 409, + 'Transaction is already linked to a non-matching payment. Unlink that payment before matching this transaction.', + 'TRANSACTION_PAYMENT_ALREADY_LINKED', + 'transaction_id', + ); + } + + if (existingPayment) { + restorePaymentBalance(db, existingPayment); + const balanceDelta = applyPaymentBalance(db, bill, amount); + db.prepare(` + UPDATE payments + SET bill_id = ?, + amount = ?, + paid_date = ?, + method = ?, + notes = ?, + balance_delta = ?, + payment_source = ?, + updated_at = datetime('now') + WHERE id = ? + `).run( + bill.id, + amount, + paidDate, + MATCH_PAYMENT_METHOD, + notes, + balanceDelta, + MATCH_PAYMENT_SOURCE, + existingPayment.id, + ); + return existingPayment.id; + } + + const balanceDelta = applyPaymentBalance(db, bill, amount); + const result = db.prepare(` + INSERT INTO payments + (bill_id, amount, paid_date, method, notes, balance_delta, payment_source, transaction_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `).run( + bill.id, + amount, + paidDate, + MATCH_PAYMENT_METHOD, + notes, + balanceDelta, + MATCH_PAYMENT_SOURCE, + transaction.id, + ); + return result.lastInsertRowid; +} + +function unlinkPaymentForTransaction(db, userId, transactionId) { + const existingPayment = getActivePaymentForTransaction(db, userId, transactionId); + if (!existingPayment) return null; + + if (existingPayment.payment_source === MATCH_PAYMENT_SOURCE) { + restorePaymentBalance(db, existingPayment); + db.prepare(` + UPDATE payments + SET deleted_at = datetime('now'), updated_at = datetime('now') + WHERE id = ? + `).run(existingPayment.id); + return { ...existingPayment, deleted: true }; + } + + db.prepare(` + UPDATE payments + SET transaction_id = NULL, updated_at = datetime('now') + WHERE id = ? + `).run(existingPayment.id); + return { ...existingPayment, unlinked: true }; +} + +function responseForTransaction(db, userId, transactionId, paymentId = null, extra = {}) { + return { + success: true, + transaction: decorateTransaction(getTransactionForUser(db, userId, transactionId)), + payment: getPaymentForResponse(db, userId, paymentId), + ...extra, + }; +} + +function matchTransactionToBill(userId, transactionId, billId) { + const db = getDb(); + const tx = db.transaction(() => { + const transaction = getOwnedTransaction(db, userId, transactionId); + if (transaction.ignored || transaction.match_status === 'ignored') { + throw matchError(400, 'Ignored transactions must be unignored before matching', 'TRANSACTION_IGNORED', 'transaction_id'); + } + + const bill = getOwnedBill(db, userId, billId); + const paymentId = createOrUpdateMatchPayment(db, userId, transaction, bill); + + db.prepare(` + UPDATE transactions + SET matched_bill_id = ?, + match_status = 'matched', + ignored = 0, + updated_at = datetime('now') + WHERE id = ? AND user_id = ? + `).run(bill.id, transaction.id, userId); + + return responseForTransaction(db, userId, transaction.id, paymentId); + }); + + return tx(); +} + +function unmatchTransaction(userId, transactionId) { + const db = getDb(); + const tx = db.transaction(() => { + const transaction = getOwnedTransaction(db, userId, transactionId); + const removedPayment = unlinkPaymentForTransaction(db, userId, transaction.id); + + db.prepare(` + UPDATE transactions + SET matched_bill_id = NULL, + match_status = 'unmatched', + ignored = 0, + updated_at = datetime('now') + WHERE id = ? AND user_id = ? + `).run(transaction.id, userId); + + return responseForTransaction(db, userId, transaction.id, null, { removed_payment: removedPayment }); + }); + + return tx(); +} + +function ignoreTransaction(userId, transactionId) { + const db = getDb(); + const tx = db.transaction(() => { + const transaction = getOwnedTransaction(db, userId, transactionId); + const removedPayment = unlinkPaymentForTransaction(db, userId, transaction.id); + + db.prepare(` + UPDATE transactions + SET ignored = 1, + match_status = 'ignored', + matched_bill_id = NULL, + updated_at = datetime('now') + WHERE id = ? AND user_id = ? + `).run(transaction.id, userId); + + return responseForTransaction(db, userId, transaction.id, null, { removed_payment: removedPayment }); + }); + + return tx(); +} + +function unignoreTransaction(userId, transactionId) { + const db = getDb(); + const tx = db.transaction(() => { + const transaction = getOwnedTransaction(db, userId, transactionId); + + db.prepare(` + UPDATE transactions + SET ignored = 0, + match_status = 'unmatched', + matched_bill_id = NULL, + updated_at = datetime('now') + WHERE id = ? AND user_id = ? + `).run(transaction.id, userId); + + return responseForTransaction(db, userId, transaction.id); + }); + + return tx(); +} + +module.exports = { + MATCH_PAYMENT_METHOD, + MATCH_PAYMENT_SOURCE, + ignoreTransaction, + matchTransactionToBill, + unignoreTransaction, + unmatchTransaction, +}; diff --git a/services/userDbImportService.js b/services/userDbImportService.js index 77ae547..a6c3ced 100644 --- a/services/userDbImportService.js +++ b/services/userDbImportService.js @@ -12,7 +12,7 @@ const SESSION_TTL_HOURS = 24; const REQUIRED_TABLES = ['export_metadata', 'categories', 'bills', 'payments', 'monthly_bill_state']; const VALID_BILLING_CYCLES = new Set(['monthly', 'quarterly', 'annually', 'irregular']); const VALID_AUTODRAFT = new Set(['none', 'pending', 'assumed_paid', 'confirmed']); -const VALID_PAYMENT_SOURCES = new Set(['manual', 'file_import', 'provider_sync']); +const VALID_PAYMENT_SOURCES = new Set(['manual', 'file_import', 'provider_sync', 'transaction_match']); function importError(status, message, code, details = []) { const err = new Error(message); diff --git a/tests/transactionMatchService.test.js b/tests/transactionMatchService.test.js new file mode 100644 index 0000000..7595457 --- /dev/null +++ b/tests/transactionMatchService.test.js @@ -0,0 +1,383 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +const dbPath = path.join(os.tmpdir(), `bill-tracker-transaction-match-test-${process.pid}.sqlite`); +process.env.DB_PATH = dbPath; + +const { getDb, closeDb } = require('../db/database'); +const { ensureManualDataSource } = require('../services/transactionService'); +const { getTracker } = require('../services/trackerService'); +const { + listMatchSuggestions, + rejectMatchSuggestion, + suggestionId, +} = require('../services/matchSuggestionService'); +const { + ignoreTransaction, + matchTransactionToBill, + unignoreTransaction, + unmatchTransaction, +} = require('../services/transactionMatchService'); + +function createUser(db, suffix) { + return db.prepare(` + INSERT INTO users (username, password_hash, role, active, email, created_at, updated_at) + VALUES (?, 'x', 'user', 1, ?, datetime('now'), datetime('now')) + `).run(`match-user-${suffix}`, `match-user-${suffix}@local`).lastInsertRowid; +} + +function createBill(db, userId, name = 'City Water') { + return db.prepare(` + INSERT INTO bills (user_id, name, due_day, expected_amount) + VALUES (?, ?, 16, 85) + `).run(userId, name).lastInsertRowid; +} + +function createTransaction(db, userId, overrides = {}) { + const source = ensureManualDataSource(db, userId); + return db.prepare(` + INSERT INTO transactions + (user_id, data_source_id, source_type, posted_date, amount, currency, + description, payee, match_status, ignored) + VALUES (?, ?, 'manual', ?, ?, 'USD', ?, ?, 'unmatched', 0) + `).run( + userId, + source.id, + overrides.posted_date || '2026-05-16', + overrides.amount ?? -8500, + overrides.description || 'Water bill payment', + overrides.payee || 'City Water', + ).lastInsertRowid; +} + +function activePaymentsForTransaction(db, transactionId) { + return db.prepare(` + SELECT * + FROM payments + WHERE transaction_id = ? AND deleted_at IS NULL + ORDER BY id + `).all(transactionId); +} + +function createManualPayment(db, billId, overrides = {}) { + return db.prepare(` + INSERT INTO payments (bill_id, amount, paid_date, method, payment_source, notes) + VALUES (?, ?, ?, ?, ?, ?) + `).run( + billId, + overrides.amount ?? 85, + overrides.paid_date || '2026-05-16', + overrides.method || 'manual', + overrides.payment_source || 'manual', + overrides.notes || 'Manual payment', + ).lastInsertRowid; +} + +function trackerRow(userId, billId, today = '2026-05-20') { + const tracker = getTracker(userId, { year: 2026, month: 5 }, new Date(`${today}T12:00:00Z`)); + assert.equal(tracker.error, undefined); + const row = tracker.rows.find(item => item.id === billId); + assert.ok(row, 'tracker row should exist'); + return row; +} + +function callBillsRoute(routePath, { userId, params = {}, query = {} }) { + const billsRouter = require('../routes/bills'); + const layer = billsRouter.stack.find(item => item.route?.path === routePath && item.route.methods.get); + assert.ok(layer, `route ${routePath} should exist`); + const handler = layer.route.stack[0].handle; + + return new Promise((resolve, reject) => { + const req = { + params, + query, + user: { id: userId, role: 'user' }, + }; + const res = { + statusCode: 200, + status(code) { + this.statusCode = code; + return this; + }, + json(data) { + resolve({ status: this.statusCode, data }); + }, + }; + try { + handler(req, res); + } catch (err) { + reject(err); + } + }); +} + +function callPaymentsRoute(routePath, method, { userId, params = {}, query = {}, body = {} }) { + const paymentsRouter = require('../routes/payments'); + const layer = paymentsRouter.stack.find(item => item.route?.path === routePath && item.route.methods[method]); + assert.ok(layer, `route ${method.toUpperCase()} ${routePath} should exist`); + const handler = layer.route.stack[0].handle; + + return new Promise((resolve, reject) => { + const req = { + body, + params, + query, + user: { id: userId, role: 'user' }, + }; + const res = { + statusCode: 200, + status(code) { + this.statusCode = code; + return this; + }, + json(data) { + resolve({ status: this.statusCode, data }); + }, + }; + try { + handler(req, res); + } catch (err) { + reject(err); + } + }); +} + +test.after(() => { + closeDb(); + for (const suffix of ['', '-wal', '-shm']) { + fs.rmSync(`${dbPath}${suffix}`, { force: true }); + } +}); + +test('matching a transaction creates one active transaction_match payment and unmatch removes it', () => { + const db = getDb(); + const userId = createUser(db, 'basic'); + const billId = createBill(db, userId); + const transactionId = createTransaction(db, userId); + + const matched = matchTransactionToBill(userId, transactionId, billId); + + assert.equal(matched.transaction.id, transactionId); + assert.equal(matched.transaction.matched_bill_id, billId); + assert.equal(matched.transaction.match_status, 'matched'); + assert.equal(matched.transaction.ignored, 0); + assert.equal(matched.payment.bill_id, billId); + assert.equal(matched.payment.amount, 85); + assert.equal(matched.payment.paid_date, '2026-05-16'); + assert.equal(matched.payment.method, 'transaction_match'); + assert.equal(matched.payment.payment_source, 'transaction_match'); + assert.equal(matched.payment.transaction_id, transactionId); + + const matchedAgain = matchTransactionToBill(userId, transactionId, billId); + assert.equal(matchedAgain.payment.id, matched.payment.id); + assert.equal(activePaymentsForTransaction(db, transactionId).length, 1); + + const unmatched = unmatchTransaction(userId, transactionId); + assert.equal(unmatched.transaction.matched_bill_id, null); + assert.equal(unmatched.transaction.match_status, 'unmatched'); + assert.equal(unmatched.transaction.ignored, 0); + assert.equal(activePaymentsForTransaction(db, transactionId).length, 0); + + const deletedPayment = db.prepare('SELECT deleted_at FROM payments WHERE id = ?').get(matched.payment.id); + assert.ok(deletedPayment.deleted_at); +}); + +test('ignoring a matched transaction removes the match payment and blocks rematching until unignored', () => { + const db = getDb(); + const userId = createUser(db, 'ignore'); + const billId = createBill(db, userId, 'Internet'); + const transactionId = createTransaction(db, userId, { + description: 'Internet payment', + payee: 'Fiber Co', + amount: -6500, + }); + + matchTransactionToBill(userId, transactionId, billId); + const ignored = ignoreTransaction(userId, transactionId); + + assert.equal(ignored.transaction.match_status, 'ignored'); + assert.equal(ignored.transaction.ignored, 1); + assert.equal(ignored.transaction.matched_bill_id, null); + assert.equal(activePaymentsForTransaction(db, transactionId).length, 0); + + assert.throws( + () => matchTransactionToBill(userId, transactionId, billId), + /Ignored transactions must be unignored before matching/, + ); + + const unignored = unignoreTransaction(userId, transactionId); + assert.equal(unignored.transaction.match_status, 'unmatched'); + assert.equal(unignored.transaction.ignored, 0); + + const rematched = matchTransactionToBill(userId, transactionId, billId); + assert.equal(rematched.transaction.match_status, 'matched'); + assert.equal(activePaymentsForTransaction(db, transactionId).length, 1); +}); + +test('transaction match payments cannot be edited, deleted, or restored through payment routes', async () => { + const db = getDb(); + const userId = createUser(db, 'payment-route-lock'); + const billId = createBill(db, userId, 'Water'); + const transactionId = createTransaction(db, userId); + const matched = matchTransactionToBill(userId, transactionId, billId); + + const updateRes = await callPaymentsRoute('/:id', 'put', { + userId, + params: { id: String(matched.payment.id) }, + body: { + amount: 1, + paid_date: '2026-05-17', + method: 'manual', + payment_source: 'manual', + }, + }); + assert.equal(updateRes.status, 409); + + let payment = db.prepare('SELECT amount, paid_date, method, payment_source, transaction_id, deleted_at FROM payments WHERE id = ?').get(matched.payment.id); + assert.equal(payment.amount, 85); + assert.equal(payment.paid_date, '2026-05-16'); + assert.equal(payment.method, 'transaction_match'); + assert.equal(payment.payment_source, 'transaction_match'); + assert.equal(payment.transaction_id, transactionId); + assert.equal(payment.deleted_at, null); + + const deleteRes = await callPaymentsRoute('/:id', 'delete', { + userId, + params: { id: String(matched.payment.id) }, + }); + assert.equal(deleteRes.status, 409); + assert.equal(activePaymentsForTransaction(db, transactionId).length, 1); + assert.equal(db.prepare('SELECT match_status FROM transactions WHERE id = ?').get(transactionId).match_status, 'matched'); + + unmatchTransaction(userId, transactionId); + payment = db.prepare('SELECT deleted_at FROM payments WHERE id = ?').get(matched.payment.id); + assert.ok(payment.deleted_at); + + const restoreRes = await callPaymentsRoute('/:id/restore', 'post', { + userId, + params: { id: String(matched.payment.id) }, + }); + assert.equal(restoreRes.status, 409); + assert.equal(activePaymentsForTransaction(db, transactionId).length, 0); + assert.equal(db.prepare('SELECT match_status, matched_bill_id FROM transactions WHERE id = ?').get(transactionId).match_status, 'unmatched'); +}); + +test('matching marks the tracker row paid and unmatching recalculates it as unpaid', () => { + const db = getDb(); + const userId = createUser(db, 'tracker'); + const billId = createBill(db, userId, 'Electric'); + const transactionId = createTransaction(db, userId, { + description: 'Electric bill payment', + payee: 'Electric Utility', + }); + + assert.notEqual(trackerRow(userId, billId).status, 'paid'); + + matchTransactionToBill(userId, transactionId, billId); + const paidRow = trackerRow(userId, billId); + assert.equal(paidRow.status, 'paid'); + assert.equal(paidRow.has_payment, true); + assert.equal(paidRow.payments[0].transaction_id, transactionId); + + unmatchTransaction(userId, transactionId); + const unpaidRow = trackerRow(userId, billId); + assert.notEqual(unpaidRow.status, 'paid'); + assert.equal(unpaidRow.has_payment, false); + assert.equal(unpaidRow.total_paid, 0); +}); + +test('ignoring a transaction does not change bill status or manual payments', () => { + const db = getDb(); + const userId = createUser(db, 'ignore-status'); + const billId = createBill(db, userId, 'Phone'); + const transactionId = createTransaction(db, userId, { + description: 'Phone store charge', + payee: 'Phone Store', + amount: -8500, + }); + const manualPaymentId = createManualPayment(db, billId); + + const before = trackerRow(userId, billId); + assert.equal(before.status, 'paid'); + assert.equal(before.payments.some(payment => payment.id === manualPaymentId), true); + + ignoreTransaction(userId, transactionId); + + const after = trackerRow(userId, billId); + assert.equal(after.status, 'paid'); + assert.equal(after.total_paid, before.total_paid); + assert.deepEqual(activePaymentsForTransaction(db, transactionId), []); + assert.equal(after.payments.some(payment => payment.id === manualPaymentId), true); +}); + +test('match suggestions are read-only and rejections do not touch payments or transactions', () => { + const db = getDb(); + const userId = createUser(db, 'suggestions'); + const billId = createBill(db, userId); + const transactionId = createTransaction(db, userId); + const beforeTransaction = db.prepare('SELECT matched_bill_id, match_status, ignored FROM transactions WHERE id = ?').get(transactionId); + const beforePaymentCount = db.prepare('SELECT COUNT(*) AS n FROM payments WHERE bill_id = ? AND deleted_at IS NULL').get(billId).n; + + const suggestions = listMatchSuggestions(userId, { transactionId }); + const match = suggestions.find(item => item.transactionId === transactionId && item.billId === billId); + assert.ok(match, 'expected a suggestion for the matching bill'); + assert.equal(match.score > 0, true); + assert.ok(match.reasons.length > 0); + + const afterSuggestTransaction = db.prepare('SELECT matched_bill_id, match_status, ignored FROM transactions WHERE id = ?').get(transactionId); + const afterSuggestPaymentCount = db.prepare('SELECT COUNT(*) AS n FROM payments WHERE bill_id = ? AND deleted_at IS NULL').get(billId).n; + assert.deepEqual(afterSuggestTransaction, beforeTransaction); + assert.equal(afterSuggestPaymentCount, beforePaymentCount); + + const rejected = rejectMatchSuggestion(userId, suggestionId(transactionId, billId)); + assert.equal(rejected.rejected, true); + + const afterRejectTransaction = db.prepare('SELECT matched_bill_id, match_status, ignored FROM transactions WHERE id = ?').get(transactionId); + const afterRejectPaymentCount = db.prepare('SELECT COUNT(*) AS n FROM payments WHERE bill_id = ? AND deleted_at IS NULL').get(billId).n; + assert.deepEqual(afterRejectTransaction, beforeTransaction); + assert.equal(afterRejectPaymentCount, beforePaymentCount); + assert.equal(listMatchSuggestions(userId, { transactionId }).some(item => item.id === rejected.id), false); +}); + +test('manual payment history remains visible and suppresses duplicate suggestions for the same cycle', async () => { + const db = getDb(); + const userId = createUser(db, 'manual-history'); + const billId = createBill(db, userId, 'Internet'); + const manualPaymentId = createManualPayment(db, billId, { + amount: 65, + notes: 'Paid from checking', + }); + const transactionId = createTransaction(db, userId, { + amount: -6500, + description: 'Internet bill', + payee: 'Internet', + }); + + assert.equal( + listMatchSuggestions(userId, { transactionId }).some(item => item.billId === billId), + false, + ); + + const matched = matchTransactionToBill(userId, transactionId, billId); + + const paymentsRes = await callBillsRoute('/:id/payments', { + userId, + params: { id: String(billId) }, + query: { limit: '100' }, + }); + assert.equal(paymentsRes.status, 200); + assert.equal(paymentsRes.data.payments.some(payment => payment.id === manualPaymentId && payment.payment_source === 'manual'), true); + assert.equal(paymentsRes.data.payments.some(payment => payment.id === matched.payment.id && payment.payment_source === 'transaction_match'), true); + + const transactionsRes = await callBillsRoute('/:id/transactions', { + userId, + params: { id: String(billId) }, + }); + assert.equal(transactionsRes.status, 200); + assert.equal(transactionsRes.data.transactions.length, 1); + assert.equal(transactionsRes.data.transactions[0].id, transactionId); + assert.equal(transactionsRes.data.transactions[0].linked_payment.id, matched.payment.id); +});