diff --git a/client/api.js b/client/api.js index 3ff5e0b..239b99c 100644 --- a/client/api.js +++ b/client/api.js @@ -408,6 +408,30 @@ export const api = { return data; }, commitCsvTransactionImport: (data) => post('/import/csv/commit', data), + previewOfxTransactionImport: async (file) => { + const csrfToken = await getCsrfToken(); + const res = await fetch('/api/import/ofx/preview', { + method: 'POST', + credentials: 'include', + headers: { + 'Content-Type': 'application/x-ofx', + 'x-csrf-token': csrfToken, + ...(file.name ? { 'X-Filename': file.name } : {}), + }, + body: file, + }); + const data = await res.json(); + if (!res.ok) { + const err = new Error(data.message || data.error || `HTTP ${res.status}`); + err.status = res.status; + err.data = data; + err.details = data.details || []; + err.code = data.code; + throw err; + } + return data; + }, + commitOfxTransactionImport: (data) => post('/import/ofx/commit', data), importHistory: () => get('/import/history'), // Transactions diff --git a/client/components/data/ImportOfxSection.jsx b/client/components/data/ImportOfxSection.jsx new file mode 100644 index 0000000..99697d7 --- /dev/null +++ b/client/components/data/ImportOfxSection.jsx @@ -0,0 +1,123 @@ +import { useRef, useState } from 'react'; +import { toast } from 'sonner'; +import { FileText, Upload, Loader2, CheckCircle2, X } from 'lucide-react'; +import { api } from '@/api'; +import { cn } from '@/lib/utils'; +import { Button } from '@/components/ui/button'; +import { formatCentsUSD } from '@/lib/money'; +import { SectionCard, importErrorState } from './dataShared'; + +/** + * Import bank transactions from an OFX / QFX file. Unlike CSV, the file is + * structured, so there is no column-mapping step: upload → preview → import. + * Duplicates are skipped by the server (same dedupe scope as CSV/SimpleFIN). + */ +export default function ImportOfxSection({ onHistoryRefresh, cardProps = {} }) { + const fileRef = useRef(null); + const [preview, setPreview] = useState(null); // { import_session_id, count, sample } + const [busy, setBusy] = useState(null); // 'preview' | 'commit' | null + + async function handleFile(e) { + const file = e.target.files?.[0]; + if (!file) return; + setBusy('preview'); + setPreview(null); + try { + setPreview(await api.previewOfxTransactionImport(file)); + } catch (err) { + toast.error(importErrorState(err, 'Could not read that OFX/QFX file.').message); + } finally { + setBusy(null); + if (fileRef.current) fileRef.current.value = ''; + } + } + + async function handleImport() { + if (!preview?.import_session_id) return; + setBusy('commit'); + try { + const r = await api.commitOfxTransactionImport({ import_session_id: preview.import_session_id }); + const parts = [`${r.imported} imported`]; + if (r.skipped) parts.push(`${r.skipped} already present`); + if (r.failed) parts.push(`${r.failed} failed`); + toast.success(`OFX import complete — ${parts.join(', ')}.`); + setPreview(null); + onHistoryRefresh?.(); + } catch (err) { + toast.error(err.message || 'OFX import failed.'); + } finally { + setBusy(null); + } + } + + return ( + +
+
+
+ +

+ Many banks export .ofx or{' '} + .qfx files. Upload one to import its + transactions — duplicates are skipped automatically. +

+
+
+ + {!preview ? ( +
+ + +
+ ) : ( +
+
+

+ + {preview.count} transaction{preview.count === 1 ? '' : 's'} found +

+ +
+
    + {(preview.sample || []).map((tx, i) => ( +
  • + {tx.payee || tx.description || 'Transaction'} + {tx.posted_date} + + {formatCentsUSD(tx.amount, { signed: true })} + +
    • + ))} + {preview.count > (preview.sample || []).length && ( +
  • + + {preview.count - (preview.sample || []).length} more +
    • + )} +
+ +
+ )} +
+ + ); +} diff --git a/client/pages/DataPage.jsx b/client/pages/DataPage.jsx index 7004d6d..b244759 100644 --- a/client/pages/DataPage.jsx +++ b/client/pages/DataPage.jsx @@ -13,6 +13,7 @@ import DataNav from '@/components/data/DataNav'; import BankSyncSection from '@/components/data/BankSyncSection'; import BillRulesManager from '@/components/BillRulesManager'; import ImportTransactionCsvSection from '@/components/data/ImportTransactionCsvSection'; +import ImportOfxSection from '@/components/data/ImportOfxSection'; import TransactionMatchingSection from '@/components/data/TransactionMatchingSection'; import SeedDemoDataSection from '@/components/data/SeedDemoDataSection'; import DownloadMyDataSection from '@/components/data/DownloadMyDataSection'; @@ -195,7 +196,11 @@ export default function DataPage() { + { + try { + const rawFilename = req.headers['x-filename']; + const originalFilename = rawFilename + ? rawFilename.replace(/[^a-zA-Z0-9._\-\s]/g, '').trim().slice(0, 255) + : null; + const result = previewOfxTransactions(req.user.id, req.body, { original_filename: originalFilename }); + res.json(result); + } catch (err) { + return sendImportError(res, err, 'OFX/QFX preview failed', 'OFX_PREVIEW_ERROR'); + } + }, +); + +// ─── POST /api/import/ofx/commit ───────────────────────────────────────────── +// Commits a previewed OFX/QFX session (no mapping — the file is structured). +router.post('/ofx/commit', requireDataImportEnabled, express.json({ limit: '1mb' }), (req, res) => { + try { + const { import_session_id, options } = req.body || {}; + if (!import_session_id || typeof import_session_id !== 'string') { + return res.status(400).json(standardizeError('import_session_id is required', 'VALIDATION_ERROR', 'import_session_id')); + } + const result = commitOfxTransactions(req.user.id, import_session_id, options || {}); + res.json(result); + } catch (err) { + return sendImportError(res, err, 'OFX/QFX import failed', 'OFX_COMMIT_ERROR'); + } +}); + // ─── GET /api/import/history ────────────────────────────────────────────────── // Returns the authenticated user's import history (last 100 imports). diff --git a/services/csvTransactionImportService.js b/services/csvTransactionImportService.js index cda27ee..0bb2e09 100644 --- a/services/csvTransactionImportService.js +++ b/services/csvTransactionImportService.js @@ -553,4 +553,14 @@ module.exports = { FIELD_LABELS, commitCsvTransactions, previewCsvTransactions, + // Reusable transaction-import primitives (shared by the OFX/QFX importer so it + // dedupes and sessions identically — same import_sessions table, same + // (user_id, data_source_id, provider_transaction_id) dedupe scope). + saveImportSession, + loadImportSession, + deleteImportSession, + pruneExpiredSessions, + getOrCreateAccount, + stableHash, + parseCents, }; diff --git a/services/ofxImportService.js b/services/ofxImportService.js new file mode 100644 index 0000000..de4b7ca --- /dev/null +++ b/services/ofxImportService.js @@ -0,0 +1,214 @@ +'use strict'; + +// OFX / QFX transaction import. OFX 1.x is SGML (leaf tags often unclosed); OFX +// 2.x is XML; QFX is OFX plus Intuit-specific tags we can ignore. We extract the +// blocks and read each leaf tag's value up to the next '<' — which works +// for both the closed (XML) and unclosed (SGML) forms. +// +// Normalized transactions are written through the SAME session + dedupe + insert +// path as the CSV importer (shared primitives from csvTransactionImportService), +// so dedupe scope, the import_sessions table, and import_history are identical. + +const { getDb } = require('../db/database'); +const { decorateTransaction, ensureManualDataSource } = require('./transactionService'); +const { + saveImportSession, + loadImportSession, + deleteImportSession, + pruneExpiredSessions, + getOrCreateAccount, + stableHash, + parseCents, +} = require('./csvTransactionImportService'); + +const MAX_TX = 25000; + +function importError(status, message, code, details = []) { + const err = new Error(message); + err.status = status; + err.code = code; + err.details = details; + return err; +} + +// Read the first value of in `block`, up to the next '<' or line end. +function tagValue(block, tag) { + const m = new RegExp(`<${tag}>([^<\\r\\n]*)`, 'i').exec(block); + return m ? m[1].trim() : ''; +} + +// OFX date: YYYYMMDD[HHMMSS[.XXX]][ tz ] → 'YYYY-MM-DD' (posted date; tz dropped). +function ofxDate(value) { + const m = /^(\d{4})(\d{2})(\d{2})/.exec(String(value || '').trim()); + if (!m) return null; + const [, y, mo, d] = m; + const month = Number(mo); + const day = Number(d); + if (month < 1 || month > 12 || day < 1 || day > 31) return null; + return `${y}-${mo}-${d}`; +} + +function decodeEntities(s) { + return String(s || '') + .replace(/&/gi, '&').replace(/</gi, '<').replace(/>/gi, '>') + .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n))) + .trim(); +} + +/** + * Parse an OFX/QFX buffer into normalized transactions (same shape the CSV path + * produces). Throws importError on a file with no parsable transactions. + */ +function parseOfx(buffer) { + const text = Buffer.isBuffer(buffer) ? buffer.toString('utf8') : String(buffer || ''); + if (!//i.test(text) && !//i.test(text)) { + throw importError(400, 'This does not look like an OFX/QFX file.', 'OFX_INVALID'); + } + + // Account id (best effort) → a stable account name. + const acctId = tagValue(text, 'ACCTID'); + const curdef = (tagValue(text, 'CURDEF') || 'USD').toUpperCase().slice(0, 3) || 'USD'; + const accountName = acctId ? `OFX ${acctId.slice(-4).padStart(4, '•')}` : 'OFX import'; + + const blocks = text.match(/[\s\S]*?<\/STMTTRN>/gi) + || text.match(/[\s\S]*?(?=|<\/BANKTRANLIST>|<\/OFX>)/gi) + || []; + + const transactions = []; + for (const raw of blocks) { + if (transactions.length >= MAX_TX) break; + const postedDate = ofxDate(tagValue(raw, 'DTPOSTED')); + const amount = parseCents(tagValue(raw, 'TRNAMT')); + if (!postedDate || amount === null || amount === 0) continue; // skip incomplete rows + + const fitid = tagValue(raw, 'FITID'); + const name = decodeEntities(tagValue(raw, 'NAME')); + const memo = decodeEntities(tagValue(raw, 'MEMO')); + const trntype = tagValue(raw, 'TRNTYPE') || null; + const providerTransactionId = fitid + ? `ofx:id:${fitid}` + : `ofx:hash:${stableHash([postedDate, amount, name, memo])}`; + + transactions.push({ + provider_transaction_id: providerTransactionId, + transaction_type: trntype, + posted_date: postedDate, + transacted_at: null, + amount, + currency: curdef, + description: name || memo || null, + payee: name || null, + memo: memo || null, + category: null, + account_name: accountName, + raw_data: null, + }); + } + + if (transactions.length === 0) { + throw importError(400, 'No transactions found in the OFX/QFX file.', 'OFX_EMPTY'); + } + return transactions; +} + +function getOrCreateOfxDataSource(db, userId) { + ensureManualDataSource(db, userId); + const existing = db.prepare(` + SELECT * FROM data_sources + WHERE user_id = ? AND type = 'file_import' AND provider = 'ofx' AND name = 'OFX Import' + ORDER BY id ASC LIMIT 1 + `).get(userId); + if (existing) return existing; + const result = db.prepare(` + INSERT INTO data_sources (user_id, type, provider, name, status) + VALUES (?, 'file_import', 'ofx', 'OFX Import', 'active') + `).run(userId); + return db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(result.lastInsertRowid, userId); +} + +function previewOfxTransactions(userId, buffer, options = {}) { + const db = getDb(); + pruneExpiredSessions(db); + const transactions = parseOfx(buffer); + const sessionId = saveImportSession(db, userId, { + kind: 'ofx_transactions', + original_filename: options.original_filename || null, + transactions, + }); + return { + import_session_id: sessionId, + count: transactions.length, + // A small sample for the confirm screen (money kept in cents; client formats). + sample: transactions.slice(0, 12), + }; +} + +function commitOfxTransactions(userId, importSessionId, options = {}) { + const db = getDb(); + const session = loadImportSession(db, userId, importSessionId); + if (session.kind !== 'ofx_transactions') { + throw importError(400, 'Import session is not an OFX/QFX preview.', 'OFX_SESSION_INVALID'); + } + + const dataSource = getOrCreateOfxDataSource(db, userId); + const counts = { imported: 0, skipped: 0, failed: 0 }; + const details = []; + const insert = db.prepare(` + INSERT INTO transactions + (user_id, data_source_id, account_id, provider_transaction_id, source_type, + transaction_type, posted_date, transacted_at, amount, currency, description, + payee, memo, category, raw_data, match_status, ignored) + VALUES (?, ?, ?, ?, 'file_import', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'unmatched', 0) + `); + const existing = db.prepare( + 'SELECT id FROM transactions WHERE user_id = ? AND data_source_id = ? AND provider_transaction_id = ?', + ); + + const run = db.transaction(() => { + (session.transactions || []).forEach((tx, index) => { + try { + if (existing.get(userId, dataSource.id, tx.provider_transaction_id)) { + counts.skipped++; + details.push({ row: index + 1, result: 'skipped_duplicate', provider_transaction_id: tx.provider_transaction_id }); + return; + } + const account = getOrCreateAccount(db, userId, dataSource.id, tx.account_name); + const result = insert.run( + userId, dataSource.id, account?.id ?? null, tx.provider_transaction_id, + tx.transaction_type, tx.posted_date, tx.transacted_at, tx.amount, tx.currency, + tx.description, tx.payee, tx.memo, tx.category, tx.raw_data, + ); + counts.imported++; + details.push({ + row: index + 1, + result: 'imported', + transaction: decorateTransaction({ + ...tx, id: result.lastInsertRowid, user_id: userId, data_source_id: dataSource.id, + source_type: 'file_import', account_id: account?.id ?? null, match_status: 'unmatched', ignored: 0, + }), + }); + } catch (err) { + counts.failed++; + details.push({ row: index + 1, result: 'failed', message: err.message }); + } + }); + + db.prepare(` + INSERT INTO import_history ( + user_id, imported_at, source_filename, file_type, sheet_name, + rows_parsed, rows_created, rows_updated, rows_skipped, rows_ambiguous, + rows_errored, options_json, summary_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + userId, new Date().toISOString(), session.original_filename, 'ofx_transactions', null, + (session.transactions || []).length, counts.imported, 0, counts.skipped, 0, counts.failed, + JSON.stringify({ options }), JSON.stringify(details.slice(0, 500)), + ); + }); + + run(); + deleteImportSession(db, importSessionId); + return { success: true, imported: counts.imported, skipped: counts.skipped, failed: counts.failed, details }; +} + +module.exports = { parseOfx, previewOfxTransactions, commitOfxTransactions }; diff --git a/tests/ofxImportService.test.js b/tests/ofxImportService.test.js new file mode 100644 index 0000000..04ff8d6 --- /dev/null +++ b/tests/ofxImportService.test.js @@ -0,0 +1,122 @@ +'use strict'; + +// Batch 3: OFX/QFX import — parser handles SGML (OFX 1.x, unclosed leaf tags) and +// XML (OFX 2.x / QFX); commit inserts with the same dedupe scope as the CSV path. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const os = require('node:os'); +const path = require('node:path'); +const fs = require('node:fs'); + +const dbPath = path.join(os.tmpdir(), `bill-tracker-ofx-${process.pid}.sqlite`); +process.env.DB_PATH = dbPath; + +const { getDb, closeDb } = require('../db/database'); +const { parseOfx, previewOfxTransactions, commitOfxTransactions } = require('../services/ofxImportService'); + +// OFX 1.x SGML — container tags closed, leaf tags unclosed. +const OFX_SGML = `OFXHEADER:100 +DATA:OFXSGML +VERSION:102 + + + +USD +000123456789 + + +DEBIT +20260615120000 +-42.50 +TXN-1001 +Netflix +Monthly subscription + + +CREDIT +20260616 +1500.00 +TXN-1002 +Payroll + + + +`; + +// OFX 2.x / QFX — XML (closed leaf tags) + Intuit tag we must ignore. +const QFX_XML = ` + + 00123 + + DEBIT + 20260701 + -9.99 + X-1 + Spotify & Co + +`; + +let userId; +test.before(() => { + const db = getDb(); + userId = db.prepare("INSERT INTO users (username, password_hash, role, active) VALUES ('ofx-user','x','user',1)").run().lastInsertRowid; +}); +test.after(() => { + closeDb(); + for (const s of ['', '-wal', '-shm']) { try { fs.unlinkSync(dbPath + s); } catch {} } +}); + +test('parseOfx reads SGML transactions (date, signed cents, fitid, name/memo)', () => { + const txns = parseOfx(OFX_SGML); + assert.equal(txns.length, 2); + const netflix = txns[0]; + assert.equal(netflix.posted_date, '2026-06-15'); + assert.equal(netflix.amount, -4250, 'signed cents'); + assert.equal(netflix.provider_transaction_id, 'ofx:id:TXN-1001'); + assert.equal(netflix.payee, 'Netflix'); + assert.equal(netflix.memo, 'Monthly subscription'); + assert.equal(txns[1].amount, 150000, 'credit is positive cents'); +}); + +test('parseOfx reads XML/QFX + decodes entities + ignores Intuit tags', () => { + const txns = parseOfx(QFX_XML); + assert.equal(txns.length, 1); + assert.equal(txns[0].amount, -999); + assert.equal(txns[0].posted_date, '2026-07-01'); + assert.equal(txns[0].payee, 'Spotify & Co'); +}); + +test('parseOfx rejects non-OFX and empty files', () => { + assert.throws(() => parseOfx('just some text'), /does not look like|OFX/i); + assert.throws(() => parseOfx(''), /No transactions/i); +}); + +test('preview → commit inserts, and re-commit dedupes', () => { + const db = getDb(); + const before = db.prepare('SELECT COUNT(*) n FROM transactions WHERE user_id=?').get(userId).n; + + const p1 = previewOfxTransactions(userId, Buffer.from(OFX_SGML), { original_filename: 'stmt.ofx' }); + assert.equal(p1.count, 2); + const r1 = commitOfxTransactions(userId, p1.import_session_id); + assert.equal(r1.imported, 2); + assert.equal(r1.skipped, 0); + + const mid = db.prepare('SELECT COUNT(*) n FROM transactions WHERE user_id=?').get(userId).n; + assert.equal(mid - before, 2); + + // amounts persisted as integer cents + const amt = db.prepare("SELECT amount FROM transactions WHERE provider_transaction_id='ofx:id:TXN-1001' AND user_id=?").get(userId).amount; + assert.equal(amt, -4250); + + // re-import the same file → all skipped, no new rows + const p2 = previewOfxTransactions(userId, Buffer.from(OFX_SGML), { original_filename: 'stmt.ofx' }); + const r2 = commitOfxTransactions(userId, p2.import_session_id); + assert.equal(r2.imported, 0); + assert.equal(r2.skipped, 2); + const after = db.prepare('SELECT COUNT(*) n FROM transactions WHERE user_id=?').get(userId).n; + assert.equal(after, mid, 'no duplicates on re-import'); + + // logged to import_history + const hist = db.prepare("SELECT file_type, rows_created, rows_skipped FROM import_history WHERE user_id=? ORDER BY id DESC LIMIT 1").get(userId); + assert.equal(hist.file_type, 'ofx_transactions'); +});