feat(import): OFX/QFX transaction import (Batch 3)
New services/ofxImportService.js parses OFX 1.x (SGML, unclosed leaf tags),
OFX 2.x (XML) and QFX (+ Intuit tags ignored) into the same normalized shape the
CSV path produces, then writes through the SAME shared primitives (session table,
(user_id, data_source_id, provider_transaction_id) dedupe, import_history) — now
exported from csvTransactionImportService (additive; CSV tests still pass).
- Routes POST /api/import/ofx/{preview,commit} mirror the CSV two-step (raw
upload → structured commit; no column mapping since OFX is structured).
- UI: ImportOfxSection (upload → preview list → import) in the Import pane;
amounts shown via formatCentsUSD; toasts on preview/commit/malformed.
- Gap handling: signed TRNAMT → signed cents; DTPOSTED → YYYY-MM-DD; FITID →
stable provider id (hash fallback); non-OFX / empty files rejected clearly.
Tests: tests/ofxImportService.test.js (SGML + XML/QFX parse, entity decode,
signed cents, preview→commit, re-import dedupe, import_history). Server 129 pass.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
null
parent
c7b110cd68
commit
bd1eee00b0
|
|
@ -408,6 +408,30 @@ export const api = {
|
|||
return data;
|
||||
},
|
||||
commitCsvTransactionImport: (data) => post('/import/csv/commit', data),
|
||||
previewOfxTransactionImport: async (file) => {
|
||||
const csrfToken = await getCsrfToken();
|
||||
const res = await fetch('/api/import/ofx/preview', {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
headers: {
|
||||
'Content-Type': 'application/x-ofx',
|
||||
'x-csrf-token': csrfToken,
|
||||
...(file.name ? { 'X-Filename': file.name } : {}),
|
||||
},
|
||||
body: file,
|
||||
});
|
||||
const data = await res.json();
|
||||
if (!res.ok) {
|
||||
const err = new Error(data.message || data.error || `HTTP ${res.status}`);
|
||||
err.status = res.status;
|
||||
err.data = data;
|
||||
err.details = data.details || [];
|
||||
err.code = data.code;
|
||||
throw err;
|
||||
}
|
||||
return data;
|
||||
},
|
||||
commitOfxTransactionImport: (data) => post('/import/ofx/commit', data),
|
||||
importHistory: () => get('/import/history'),
|
||||
|
||||
// Transactions
|
||||
|
|
|
|||
|
|
@ -0,0 +1,123 @@
|
|||
import { useRef, useState } from 'react';
|
||||
import { toast } from 'sonner';
|
||||
import { FileText, Upload, Loader2, CheckCircle2, X } from 'lucide-react';
|
||||
import { api } from '@/api';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { formatCentsUSD } from '@/lib/money';
|
||||
import { SectionCard, importErrorState } from './dataShared';
|
||||
|
||||
/**
|
||||
* Import bank transactions from an OFX / QFX file. Unlike CSV, the file is
|
||||
* structured, so there is no column-mapping step: upload → preview → import.
|
||||
* Duplicates are skipped by the server (same dedupe scope as CSV/SimpleFIN).
|
||||
*/
|
||||
export default function ImportOfxSection({ onHistoryRefresh, cardProps = {} }) {
|
||||
const fileRef = useRef(null);
|
||||
const [preview, setPreview] = useState(null); // { import_session_id, count, sample }
|
||||
const [busy, setBusy] = useState(null); // 'preview' | 'commit' | null
|
||||
|
||||
async function handleFile(e) {
|
||||
const file = e.target.files?.[0];
|
||||
if (!file) return;
|
||||
setBusy('preview');
|
||||
setPreview(null);
|
||||
try {
|
||||
setPreview(await api.previewOfxTransactionImport(file));
|
||||
} catch (err) {
|
||||
toast.error(importErrorState(err, 'Could not read that OFX/QFX file.').message);
|
||||
} finally {
|
||||
setBusy(null);
|
||||
if (fileRef.current) fileRef.current.value = '';
|
||||
}
|
||||
}
|
||||
|
||||
async function handleImport() {
|
||||
if (!preview?.import_session_id) return;
|
||||
setBusy('commit');
|
||||
try {
|
||||
const r = await api.commitOfxTransactionImport({ import_session_id: preview.import_session_id });
|
||||
const parts = [`${r.imported} imported`];
|
||||
if (r.skipped) parts.push(`${r.skipped} already present`);
|
||||
if (r.failed) parts.push(`${r.failed} failed`);
|
||||
toast.success(`OFX import complete — ${parts.join(', ')}.`);
|
||||
setPreview(null);
|
||||
onHistoryRefresh?.();
|
||||
} catch (err) {
|
||||
toast.error(err.message || 'OFX import failed.');
|
||||
} finally {
|
||||
setBusy(null);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<SectionCard {...cardProps}>
|
||||
<div className="space-y-4 px-6 py-5">
|
||||
<div className="rounded-lg border border-border/60 bg-muted/25 p-4">
|
||||
<div className="flex items-start gap-3">
|
||||
<FileText className="mt-0.5 h-5 w-5 shrink-0 text-primary" />
|
||||
<p className="min-w-0 text-sm text-muted-foreground">
|
||||
Many banks export <span className="font-medium text-foreground">.ofx</span> or{' '}
|
||||
<span className="font-medium text-foreground">.qfx</span> files. Upload one to import its
|
||||
transactions — duplicates are skipped automatically.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!preview ? (
|
||||
<div>
|
||||
<input
|
||||
ref={fileRef}
|
||||
id="ofx-upload"
|
||||
type="file"
|
||||
accept=".ofx,.qfx,application/x-ofx"
|
||||
className="hidden"
|
||||
onChange={handleFile}
|
||||
disabled={busy === 'preview'}
|
||||
/>
|
||||
<label htmlFor="ofx-upload">
|
||||
<Button asChild variant="outline" className="cursor-pointer gap-2">
|
||||
<span>
|
||||
{busy === 'preview' ? <Loader2 className="h-4 w-4 animate-spin" /> : <Upload className="h-4 w-4" />}
|
||||
{busy === 'preview' ? 'Reading…' : 'Choose .ofx / .qfx file'}
|
||||
</span>
|
||||
</Button>
|
||||
</label>
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-3">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<p className="text-sm font-medium text-foreground">
|
||||
<CheckCircle2 className="mr-1.5 inline h-4 w-4 text-emerald-500" />
|
||||
{preview.count} transaction{preview.count === 1 ? '' : 's'} found
|
||||
</p>
|
||||
<Button variant="ghost" size="sm" className="gap-1 text-xs" onClick={() => setPreview(null)} disabled={busy === 'commit'}>
|
||||
<X className="h-3.5 w-3.5" /> Cancel
|
||||
</Button>
|
||||
</div>
|
||||
<ul className="max-h-56 divide-y divide-border/50 overflow-y-auto rounded-lg border border-border/50">
|
||||
{(preview.sample || []).map((tx, i) => (
|
||||
<li key={i} className="flex items-center justify-between gap-3 px-3 py-2 text-sm">
|
||||
<span className="min-w-0 flex-1 truncate">{tx.payee || tx.description || 'Transaction'}</span>
|
||||
<span className="shrink-0 text-xs text-muted-foreground tabular-nums">{tx.posted_date}</span>
|
||||
<span className={cn('shrink-0 tabular-nums', tx.amount < 0 ? 'text-foreground' : 'text-emerald-600 dark:text-emerald-400')}>
|
||||
{formatCentsUSD(tx.amount, { signed: true })}
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
{preview.count > (preview.sample || []).length && (
|
||||
<li className="px-3 py-2 text-center text-xs text-muted-foreground">
|
||||
+ {preview.count - (preview.sample || []).length} more
|
||||
</li>
|
||||
)}
|
||||
</ul>
|
||||
<Button className="gap-2" onClick={handleImport} disabled={busy === 'commit'}>
|
||||
{busy === 'commit' ? <Loader2 className="h-4 w-4 animate-spin" /> : <Upload className="h-4 w-4" />}
|
||||
Import {preview.count} transaction{preview.count === 1 ? '' : 's'}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</SectionCard>
|
||||
);
|
||||
}
|
||||
|
|
@ -13,6 +13,7 @@ import DataNav from '@/components/data/DataNav';
|
|||
import BankSyncSection from '@/components/data/BankSyncSection';
|
||||
import BillRulesManager from '@/components/BillRulesManager';
|
||||
import ImportTransactionCsvSection from '@/components/data/ImportTransactionCsvSection';
|
||||
import ImportOfxSection from '@/components/data/ImportOfxSection';
|
||||
import TransactionMatchingSection from '@/components/data/TransactionMatchingSection';
|
||||
import SeedDemoDataSection from '@/components/data/SeedDemoDataSection';
|
||||
import DownloadMyDataSection from '@/components/data/DownloadMyDataSection';
|
||||
|
|
@ -195,7 +196,11 @@ export default function DataPage() {
|
|||
</Suspense>
|
||||
<ImportTransactionCsvSection
|
||||
onHistoryRefresh={handleTransactionImportComplete}
|
||||
cardProps={{ title: 'Import transactions', subtitle: 'Upload a bank or card CSV export.', icon: FileText, collapsible: true, defaultOpen: false, storageKey: 'billtracker:data.card.transactionCsv', summary: 'Upload bank or credit-card CSV transaction files.' }}
|
||||
cardProps={{ title: 'Import transactions (CSV)', subtitle: 'Upload a bank or card CSV export.', icon: FileText, collapsible: true, defaultOpen: false, storageKey: 'billtracker:data.card.transactionCsv', summary: 'Upload bank or credit-card CSV transaction files.' }}
|
||||
/>
|
||||
<ImportOfxSection
|
||||
onHistoryRefresh={handleTransactionImportComplete}
|
||||
cardProps={{ title: 'Import transactions (OFX/QFX)', subtitle: 'Upload a bank OFX or QFX export — no column mapping needed.', icon: FileText, collapsible: true, defaultOpen: false, storageKey: 'billtracker:data.card.ofx', summary: 'Upload an OFX/QFX transaction file.' }}
|
||||
/>
|
||||
<SeedDemoDataSection
|
||||
onSeeded={loadHistory}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,10 @@ const {
|
|||
previewCsvTransactions,
|
||||
commitCsvTransactions,
|
||||
} = require('../services/csvTransactionImportService');
|
||||
const {
|
||||
previewOfxTransactions,
|
||||
commitOfxTransactions,
|
||||
} = require('../services/ofxImportService');
|
||||
|
||||
function dataImportEnabled() {
|
||||
return String(process.env.DATA_IMPORT_ENABLED ?? 'true').toLowerCase() !== 'false';
|
||||
|
|
@ -237,6 +241,52 @@ router.post('/csv/commit', requireDataImportEnabled, express.json({ limit: '1mb'
|
|||
}
|
||||
});
|
||||
|
||||
// ─── POST /api/import/ofx/preview ────────────────────────────────────────────
|
||||
// Accepts an OFX/QFX file as raw bytes and returns a parsed transaction sample +
|
||||
// count. Writes no transactions.
|
||||
router.post(
|
||||
'/ofx/preview',
|
||||
requireDataImportEnabled,
|
||||
express.raw({
|
||||
type: [
|
||||
'application/x-ofx',
|
||||
'application/vnd.intu.qfx',
|
||||
'application/octet-stream',
|
||||
'text/plain',
|
||||
'text/xml',
|
||||
'application/xml',
|
||||
],
|
||||
limit: '10mb',
|
||||
}),
|
||||
(req, res) => {
|
||||
try {
|
||||
const rawFilename = req.headers['x-filename'];
|
||||
const originalFilename = rawFilename
|
||||
? rawFilename.replace(/[^a-zA-Z0-9._\-\s]/g, '').trim().slice(0, 255)
|
||||
: null;
|
||||
const result = previewOfxTransactions(req.user.id, req.body, { original_filename: originalFilename });
|
||||
res.json(result);
|
||||
} catch (err) {
|
||||
return sendImportError(res, err, 'OFX/QFX preview failed', 'OFX_PREVIEW_ERROR');
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// ─── POST /api/import/ofx/commit ─────────────────────────────────────────────
|
||||
// Commits a previewed OFX/QFX session (no mapping — the file is structured).
|
||||
router.post('/ofx/commit', requireDataImportEnabled, express.json({ limit: '1mb' }), (req, res) => {
|
||||
try {
|
||||
const { import_session_id, options } = req.body || {};
|
||||
if (!import_session_id || typeof import_session_id !== 'string') {
|
||||
return res.status(400).json(standardizeError('import_session_id is required', 'VALIDATION_ERROR', 'import_session_id'));
|
||||
}
|
||||
const result = commitOfxTransactions(req.user.id, import_session_id, options || {});
|
||||
res.json(result);
|
||||
} catch (err) {
|
||||
return sendImportError(res, err, 'OFX/QFX import failed', 'OFX_COMMIT_ERROR');
|
||||
}
|
||||
});
|
||||
|
||||
// ─── GET /api/import/history ──────────────────────────────────────────────────
|
||||
// Returns the authenticated user's import history (last 100 imports).
|
||||
|
||||
|
|
|
|||
|
|
@ -553,4 +553,14 @@ module.exports = {
|
|||
FIELD_LABELS,
|
||||
commitCsvTransactions,
|
||||
previewCsvTransactions,
|
||||
// Reusable transaction-import primitives (shared by the OFX/QFX importer so it
|
||||
// dedupes and sessions identically — same import_sessions table, same
|
||||
// (user_id, data_source_id, provider_transaction_id) dedupe scope).
|
||||
saveImportSession,
|
||||
loadImportSession,
|
||||
deleteImportSession,
|
||||
pruneExpiredSessions,
|
||||
getOrCreateAccount,
|
||||
stableHash,
|
||||
parseCents,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,214 @@
|
|||
'use strict';
|
||||
|
||||
// OFX / QFX transaction import. OFX 1.x is SGML (leaf tags often unclosed); OFX
|
||||
// 2.x is XML; QFX is OFX plus Intuit-specific tags we can ignore. We extract the
|
||||
// <STMTTRN> blocks and read each leaf tag's value up to the next '<' — which works
|
||||
// for both the closed (XML) and unclosed (SGML) forms.
|
||||
//
|
||||
// Normalized transactions are written through the SAME session + dedupe + insert
|
||||
// path as the CSV importer (shared primitives from csvTransactionImportService),
|
||||
// so dedupe scope, the import_sessions table, and import_history are identical.
|
||||
|
||||
const { getDb } = require('../db/database');
|
||||
const { decorateTransaction, ensureManualDataSource } = require('./transactionService');
|
||||
const {
|
||||
saveImportSession,
|
||||
loadImportSession,
|
||||
deleteImportSession,
|
||||
pruneExpiredSessions,
|
||||
getOrCreateAccount,
|
||||
stableHash,
|
||||
parseCents,
|
||||
} = require('./csvTransactionImportService');
|
||||
|
||||
const MAX_TX = 25000;
|
||||
|
||||
function importError(status, message, code, details = []) {
|
||||
const err = new Error(message);
|
||||
err.status = status;
|
||||
err.code = code;
|
||||
err.details = details;
|
||||
return err;
|
||||
}
|
||||
|
||||
// Read the first value of <TAG> in `block`, up to the next '<' or line end.
|
||||
function tagValue(block, tag) {
|
||||
const m = new RegExp(`<${tag}>([^<\\r\\n]*)`, 'i').exec(block);
|
||||
return m ? m[1].trim() : '';
|
||||
}
|
||||
|
||||
// OFX date: YYYYMMDD[HHMMSS[.XXX]][ tz ] → 'YYYY-MM-DD' (posted date; tz dropped).
|
||||
function ofxDate(value) {
|
||||
const m = /^(\d{4})(\d{2})(\d{2})/.exec(String(value || '').trim());
|
||||
if (!m) return null;
|
||||
const [, y, mo, d] = m;
|
||||
const month = Number(mo);
|
||||
const day = Number(d);
|
||||
if (month < 1 || month > 12 || day < 1 || day > 31) return null;
|
||||
return `${y}-${mo}-${d}`;
|
||||
}
|
||||
|
||||
function decodeEntities(s) {
|
||||
return String(s || '')
|
||||
.replace(/&/gi, '&').replace(/</gi, '<').replace(/>/gi, '>')
|
||||
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an OFX/QFX buffer into normalized transactions (same shape the CSV path
|
||||
* produces). Throws importError on a file with no parsable transactions.
|
||||
*/
|
||||
function parseOfx(buffer) {
|
||||
const text = Buffer.isBuffer(buffer) ? buffer.toString('utf8') : String(buffer || '');
|
||||
if (!/<OFX>/i.test(text) && !/<STMTTRN>/i.test(text)) {
|
||||
throw importError(400, 'This does not look like an OFX/QFX file.', 'OFX_INVALID');
|
||||
}
|
||||
|
||||
// Account id (best effort) → a stable account name.
|
||||
const acctId = tagValue(text, 'ACCTID');
|
||||
const curdef = (tagValue(text, 'CURDEF') || 'USD').toUpperCase().slice(0, 3) || 'USD';
|
||||
const accountName = acctId ? `OFX ${acctId.slice(-4).padStart(4, '•')}` : 'OFX import';
|
||||
|
||||
const blocks = text.match(/<STMTTRN>[\s\S]*?<\/STMTTRN>/gi)
|
||||
|| text.match(/<STMTTRN>[\s\S]*?(?=<STMTTRN>|<\/BANKTRANLIST>|<\/OFX>)/gi)
|
||||
|| [];
|
||||
|
||||
const transactions = [];
|
||||
for (const raw of blocks) {
|
||||
if (transactions.length >= MAX_TX) break;
|
||||
const postedDate = ofxDate(tagValue(raw, 'DTPOSTED'));
|
||||
const amount = parseCents(tagValue(raw, 'TRNAMT'));
|
||||
if (!postedDate || amount === null || amount === 0) continue; // skip incomplete rows
|
||||
|
||||
const fitid = tagValue(raw, 'FITID');
|
||||
const name = decodeEntities(tagValue(raw, 'NAME'));
|
||||
const memo = decodeEntities(tagValue(raw, 'MEMO'));
|
||||
const trntype = tagValue(raw, 'TRNTYPE') || null;
|
||||
const providerTransactionId = fitid
|
||||
? `ofx:id:${fitid}`
|
||||
: `ofx:hash:${stableHash([postedDate, amount, name, memo])}`;
|
||||
|
||||
transactions.push({
|
||||
provider_transaction_id: providerTransactionId,
|
||||
transaction_type: trntype,
|
||||
posted_date: postedDate,
|
||||
transacted_at: null,
|
||||
amount,
|
||||
currency: curdef,
|
||||
description: name || memo || null,
|
||||
payee: name || null,
|
||||
memo: memo || null,
|
||||
category: null,
|
||||
account_name: accountName,
|
||||
raw_data: null,
|
||||
});
|
||||
}
|
||||
|
||||
if (transactions.length === 0) {
|
||||
throw importError(400, 'No transactions found in the OFX/QFX file.', 'OFX_EMPTY');
|
||||
}
|
||||
return transactions;
|
||||
}
|
||||
|
||||
function getOrCreateOfxDataSource(db, userId) {
|
||||
ensureManualDataSource(db, userId);
|
||||
const existing = db.prepare(`
|
||||
SELECT * FROM data_sources
|
||||
WHERE user_id = ? AND type = 'file_import' AND provider = 'ofx' AND name = 'OFX Import'
|
||||
ORDER BY id ASC LIMIT 1
|
||||
`).get(userId);
|
||||
if (existing) return existing;
|
||||
const result = db.prepare(`
|
||||
INSERT INTO data_sources (user_id, type, provider, name, status)
|
||||
VALUES (?, 'file_import', 'ofx', 'OFX Import', 'active')
|
||||
`).run(userId);
|
||||
return db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(result.lastInsertRowid, userId);
|
||||
}
|
||||
|
||||
function previewOfxTransactions(userId, buffer, options = {}) {
|
||||
const db = getDb();
|
||||
pruneExpiredSessions(db);
|
||||
const transactions = parseOfx(buffer);
|
||||
const sessionId = saveImportSession(db, userId, {
|
||||
kind: 'ofx_transactions',
|
||||
original_filename: options.original_filename || null,
|
||||
transactions,
|
||||
});
|
||||
return {
|
||||
import_session_id: sessionId,
|
||||
count: transactions.length,
|
||||
// A small sample for the confirm screen (money kept in cents; client formats).
|
||||
sample: transactions.slice(0, 12),
|
||||
};
|
||||
}
|
||||
|
||||
function commitOfxTransactions(userId, importSessionId, options = {}) {
|
||||
const db = getDb();
|
||||
const session = loadImportSession(db, userId, importSessionId);
|
||||
if (session.kind !== 'ofx_transactions') {
|
||||
throw importError(400, 'Import session is not an OFX/QFX preview.', 'OFX_SESSION_INVALID');
|
||||
}
|
||||
|
||||
const dataSource = getOrCreateOfxDataSource(db, userId);
|
||||
const counts = { imported: 0, skipped: 0, failed: 0 };
|
||||
const details = [];
|
||||
const insert = db.prepare(`
|
||||
INSERT INTO transactions
|
||||
(user_id, data_source_id, account_id, provider_transaction_id, source_type,
|
||||
transaction_type, posted_date, transacted_at, amount, currency, description,
|
||||
payee, memo, category, raw_data, match_status, ignored)
|
||||
VALUES (?, ?, ?, ?, 'file_import', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'unmatched', 0)
|
||||
`);
|
||||
const existing = db.prepare(
|
||||
'SELECT id FROM transactions WHERE user_id = ? AND data_source_id = ? AND provider_transaction_id = ?',
|
||||
);
|
||||
|
||||
const run = db.transaction(() => {
|
||||
(session.transactions || []).forEach((tx, index) => {
|
||||
try {
|
||||
if (existing.get(userId, dataSource.id, tx.provider_transaction_id)) {
|
||||
counts.skipped++;
|
||||
details.push({ row: index + 1, result: 'skipped_duplicate', provider_transaction_id: tx.provider_transaction_id });
|
||||
return;
|
||||
}
|
||||
const account = getOrCreateAccount(db, userId, dataSource.id, tx.account_name);
|
||||
const result = insert.run(
|
||||
userId, dataSource.id, account?.id ?? null, tx.provider_transaction_id,
|
||||
tx.transaction_type, tx.posted_date, tx.transacted_at, tx.amount, tx.currency,
|
||||
tx.description, tx.payee, tx.memo, tx.category, tx.raw_data,
|
||||
);
|
||||
counts.imported++;
|
||||
details.push({
|
||||
row: index + 1,
|
||||
result: 'imported',
|
||||
transaction: decorateTransaction({
|
||||
...tx, id: result.lastInsertRowid, user_id: userId, data_source_id: dataSource.id,
|
||||
source_type: 'file_import', account_id: account?.id ?? null, match_status: 'unmatched', ignored: 0,
|
||||
}),
|
||||
});
|
||||
} catch (err) {
|
||||
counts.failed++;
|
||||
details.push({ row: index + 1, result: 'failed', message: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
db.prepare(`
|
||||
INSERT INTO import_history (
|
||||
user_id, imported_at, source_filename, file_type, sheet_name,
|
||||
rows_parsed, rows_created, rows_updated, rows_skipped, rows_ambiguous,
|
||||
rows_errored, options_json, summary_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
userId, new Date().toISOString(), session.original_filename, 'ofx_transactions', null,
|
||||
(session.transactions || []).length, counts.imported, 0, counts.skipped, 0, counts.failed,
|
||||
JSON.stringify({ options }), JSON.stringify(details.slice(0, 500)),
|
||||
);
|
||||
});
|
||||
|
||||
run();
|
||||
deleteImportSession(db, importSessionId);
|
||||
return { success: true, imported: counts.imported, skipped: counts.skipped, failed: counts.failed, details };
|
||||
}
|
||||
|
||||
module.exports = { parseOfx, previewOfxTransactions, commitOfxTransactions };
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
'use strict';
|
||||
|
||||
// Batch 3: OFX/QFX import — parser handles SGML (OFX 1.x, unclosed leaf tags) and
|
||||
// XML (OFX 2.x / QFX); commit inserts with the same dedupe scope as the CSV path.
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const fs = require('node:fs');
|
||||
|
||||
const dbPath = path.join(os.tmpdir(), `bill-tracker-ofx-${process.pid}.sqlite`);
|
||||
process.env.DB_PATH = dbPath;
|
||||
|
||||
const { getDb, closeDb } = require('../db/database');
|
||||
const { parseOfx, previewOfxTransactions, commitOfxTransactions } = require('../services/ofxImportService');
|
||||
|
||||
// OFX 1.x SGML — container tags closed, leaf tags unclosed.
|
||||
const OFX_SGML = `OFXHEADER:100
|
||||
DATA:OFXSGML
|
||||
VERSION:102
|
||||
|
||||
<OFX>
|
||||
<BANKMSGSRSV1><STMTTRNRS><STMTRS>
|
||||
<CURDEF>USD
|
||||
<BANKACCTFROM><ACCTID>000123456789</BANKACCTFROM>
|
||||
<BANKTRANLIST>
|
||||
<STMTTRN>
|
||||
<TRNTYPE>DEBIT
|
||||
<DTPOSTED>20260615120000
|
||||
<TRNAMT>-42.50
|
||||
<FITID>TXN-1001
|
||||
<NAME>Netflix
|
||||
<MEMO>Monthly subscription
|
||||
</STMTTRN>
|
||||
<STMTTRN>
|
||||
<TRNTYPE>CREDIT
|
||||
<DTPOSTED>20260616
|
||||
<TRNAMT>1500.00
|
||||
<FITID>TXN-1002
|
||||
<NAME>Payroll
|
||||
</STMTTRN>
|
||||
</BANKTRANLIST>
|
||||
</STMTRS></STMTTRNRS></BANKMSGSRSV1>
|
||||
</OFX>`;
|
||||
|
||||
// OFX 2.x / QFX — XML (closed leaf tags) + Intuit tag we must ignore.
|
||||
const QFX_XML = `<?xml version="1.0"?>
|
||||
<OFX>
|
||||
<SONRS><INTU.BID>00123</INTU.BID></SONRS>
|
||||
<STMTTRN>
|
||||
<TRNTYPE>DEBIT</TRNTYPE>
|
||||
<DTPOSTED>20260701</DTPOSTED>
|
||||
<TRNAMT>-9.99</TRNAMT>
|
||||
<FITID>X-1</FITID>
|
||||
<NAME>Spotify & Co</NAME>
|
||||
</STMTTRN>
|
||||
</OFX>`;
|
||||
|
||||
let userId;
|
||||
test.before(() => {
|
||||
const db = getDb();
|
||||
userId = db.prepare("INSERT INTO users (username, password_hash, role, active) VALUES ('ofx-user','x','user',1)").run().lastInsertRowid;
|
||||
});
|
||||
test.after(() => {
|
||||
closeDb();
|
||||
for (const s of ['', '-wal', '-shm']) { try { fs.unlinkSync(dbPath + s); } catch {} }
|
||||
});
|
||||
|
||||
test('parseOfx reads SGML transactions (date, signed cents, fitid, name/memo)', () => {
|
||||
const txns = parseOfx(OFX_SGML);
|
||||
assert.equal(txns.length, 2);
|
||||
const netflix = txns[0];
|
||||
assert.equal(netflix.posted_date, '2026-06-15');
|
||||
assert.equal(netflix.amount, -4250, 'signed cents');
|
||||
assert.equal(netflix.provider_transaction_id, 'ofx:id:TXN-1001');
|
||||
assert.equal(netflix.payee, 'Netflix');
|
||||
assert.equal(netflix.memo, 'Monthly subscription');
|
||||
assert.equal(txns[1].amount, 150000, 'credit is positive cents');
|
||||
});
|
||||
|
||||
test('parseOfx reads XML/QFX + decodes entities + ignores Intuit tags', () => {
|
||||
const txns = parseOfx(QFX_XML);
|
||||
assert.equal(txns.length, 1);
|
||||
assert.equal(txns[0].amount, -999);
|
||||
assert.equal(txns[0].posted_date, '2026-07-01');
|
||||
assert.equal(txns[0].payee, 'Spotify & Co');
|
||||
});
|
||||
|
||||
test('parseOfx rejects non-OFX and empty files', () => {
|
||||
assert.throws(() => parseOfx('just some text'), /does not look like|OFX/i);
|
||||
assert.throws(() => parseOfx('<OFX></OFX>'), /No transactions/i);
|
||||
});
|
||||
|
||||
test('preview → commit inserts, and re-commit dedupes', () => {
|
||||
const db = getDb();
|
||||
const before = db.prepare('SELECT COUNT(*) n FROM transactions WHERE user_id=?').get(userId).n;
|
||||
|
||||
const p1 = previewOfxTransactions(userId, Buffer.from(OFX_SGML), { original_filename: 'stmt.ofx' });
|
||||
assert.equal(p1.count, 2);
|
||||
const r1 = commitOfxTransactions(userId, p1.import_session_id);
|
||||
assert.equal(r1.imported, 2);
|
||||
assert.equal(r1.skipped, 0);
|
||||
|
||||
const mid = db.prepare('SELECT COUNT(*) n FROM transactions WHERE user_id=?').get(userId).n;
|
||||
assert.equal(mid - before, 2);
|
||||
|
||||
// amounts persisted as integer cents
|
||||
const amt = db.prepare("SELECT amount FROM transactions WHERE provider_transaction_id='ofx:id:TXN-1001' AND user_id=?").get(userId).amount;
|
||||
assert.equal(amt, -4250);
|
||||
|
||||
// re-import the same file → all skipped, no new rows
|
||||
const p2 = previewOfxTransactions(userId, Buffer.from(OFX_SGML), { original_filename: 'stmt.ofx' });
|
||||
const r2 = commitOfxTransactions(userId, p2.import_session_id);
|
||||
assert.equal(r2.imported, 0);
|
||||
assert.equal(r2.skipped, 2);
|
||||
const after = db.prepare('SELECT COUNT(*) n FROM transactions WHERE user_id=?').get(userId).n;
|
||||
assert.equal(after, mid, 'no duplicates on re-import');
|
||||
|
||||
// logged to import_history
|
||||
const hist = db.prepare("SELECT file_type, rows_created, rows_skipped FROM import_history WHERE user_id=? ORDER BY id DESC LIMIT 1").get(userId);
|
||||
assert.equal(hist.file_type, 'ofx_transactions');
|
||||
});
|
||||
Loading…
Reference in New Issue