'use strict'; const { assertEncryptionReady, encryptSecret, decryptSecret } = require('./encryptionService'); const { claimSetupToken, fetchAccountsAndTransactions, normalizeAccount, normalizeTransaction, sanitizeErrorMessage, } = require('./simplefinService'); const { getBankSyncConfig, SYNC_DAYS_EFFECTIVE, SYNC_DAYS_DEFAULT } = require('./bankSyncConfigService'); const { decorateDataSource } = require('./transactionService'); const { applyMerchantRules } = require('./billMerchantRuleService'); const { applySpendingCategoryRules } = require('./spendingService'); const { autoMatchForUser } = require('./matchSuggestionService'); function sinceEpochDays(days) { return Math.floor((Date.now() - days * 86400 * 1000) / 1000); } function safeErrorMessage(err) { return sanitizeErrorMessage(err?.message || String(err || 'Sync failed')); } // Upsert a single financial account, return the local row. function upsertAccount(db, accountRow) { const existing = db.prepare(` SELECT id, monitored FROM financial_accounts WHERE data_source_id = ? AND provider_account_id = ? AND user_id = ? `).get(accountRow.data_source_id, accountRow.provider_account_id, accountRow.user_id); if (existing) { db.prepare(` UPDATE financial_accounts SET name = ?, org_name = ?, currency = ?, balance = ?, available_balance = ?, raw_data = ?, updated_at = datetime('now') WHERE id = ? `).run( accountRow.name, accountRow.org_name, accountRow.currency, accountRow.balance, accountRow.available_balance, accountRow.raw_data, existing.id, ); return existing; } const result = db.prepare(` INSERT INTO financial_accounts (user_id, data_source_id, provider_account_id, name, org_name, account_type, currency, balance, available_balance, raw_data) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `).run( accountRow.user_id, accountRow.data_source_id, accountRow.provider_account_id, accountRow.name, accountRow.org_name, accountRow.account_type, accountRow.currency, accountRow.balance, accountRow.available_balance, accountRow.raw_data, ); return { id: result.lastInsertRowid, monitored: 1 }; } // Insert a transaction, or update one we previously stored as PENDING. A pending charge // can change amount before it settles, and eventually flips pending → posted (gaining a // real posted_date). A transaction we already recorded as posted is final and left alone; // a row the user has matched or ignored is never touched. // Returns: 'inserted' | 'posted' (pending→settled) | 'updated' (pending refreshed) | 'skipped'. function upsertTransaction(db, txRow) { const existing = db.prepare(` SELECT id, pending, match_status FROM transactions WHERE data_source_id = ? AND provider_transaction_id = ? `).get(txRow.data_source_id, txRow.provider_transaction_id); if (!existing) { try { db.prepare(` INSERT INTO transactions (user_id, data_source_id, account_id, provider_transaction_id, source_type, posted_date, transacted_at, amount, currency, description, payee, memo, match_status, ignored, pending, raw_data) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `).run( txRow.user_id, txRow.data_source_id, txRow.account_id, txRow.provider_transaction_id, txRow.source_type, txRow.posted_date, txRow.transacted_at, txRow.amount, txRow.currency, txRow.description, txRow.payee, txRow.memo, txRow.match_status, txRow.ignored, txRow.pending, txRow.raw_data, ); return 'inserted'; } catch (err) { if (err.code === 'SQLITE_CONSTRAINT_UNIQUE' || (err.message || '').includes('UNIQUE')) { return 'skipped'; } throw err; } } // Only refresh rows still pending in our DB and not yet acted on by the user. if (existing.pending === 1 && existing.match_status === 'unmatched') { db.prepare(` UPDATE transactions SET posted_date = ?, transacted_at = ?, amount = ?, currency = ?, description = ?, payee = ?, memo = ?, pending = ?, raw_data = ?, updated_at = datetime('now') WHERE id = ? `).run( txRow.posted_date, txRow.transacted_at, txRow.amount, txRow.currency, txRow.description, txRow.payee, txRow.memo, txRow.pending, txRow.raw_data, existing.id, ); return txRow.pending === 0 ? 'posted' : 'updated'; } return 'skipped'; } async function runSync(db, userId, dataSource, { days, debug = false } = {}) { const accessUrl = decryptSecret(dataSource.encrypted_secret); const isFirstSync = !dataSource.last_sync_at; // Explicit `days` param (e.g. backfill) takes precedence. // Initial seed always uses the full SYNC_DAYS_EFFECTIVE window regardless of admin config. // Routine syncs use the admin-configured sync_days (default 30); falls back to SYNC_DAYS_DEFAULT. const config = getBankSyncConfig(); const syncDays = days ?? (isFirstSync ? SYNC_DAYS_EFFECTIVE : (config.sync_days || SYNC_DAYS_DEFAULT)); const since = sinceEpochDays(syncDays); if (debug) console.log(`[bankSync:debug] Source #${dataSource.id} user ${userId}: fetching ${syncDays} days from SimpleFIN (since epoch ${since})`); const raw = await fetchAccountsAndTransactions(accessUrl, since); const accounts = Array.isArray(raw.accounts) ? raw.accounts : []; if (debug) console.log(`[bankSync:debug] Source #${dataSource.id}: SimpleFIN returned ${accounts.length} account(s)`); if (raw._errlistSummary) { console.warn(`[bankSync] errlist for source ${dataSource.id}: ${raw._errlistSummary}`); } let accountsUpserted = 0; let transactionsNew = 0; let transactionsSkip = 0; let transactionsPosted = 0; // pending → settled this cycle let pendingCleared = 0; // stale pending rows pruned (re-posted under new id / dropped) // Remove pending rows we hold for an account that are no longer in the feed — a pending // charge that re-posted under a new id, or was dropped by the bank before settling. const pruneOrphanPending = db.prepare(` DELETE FROM transactions WHERE account_id = ? AND user_id = ? AND pending = 1 AND match_status = 'unmatched' AND provider_transaction_id NOT IN (SELECT value FROM json_each(?)) `); for (const rawAccount of accounts) { const accountRow = normalizeAccount(rawAccount, dataSource.id, userId); const localAccount = upsertAccount(db, accountRow); accountsUpserted += 1; const txList = rawAccount.transactions || []; if (debug) console.log(`[bankSync:debug] Account "${rawAccount.name}" (monitored=${localAccount.monitored}): ${txList.length} transaction(s)`); if (localAccount.monitored === 0) continue; const seenTxIds = []; for (const rawTx of txList) { const txRow = normalizeTransaction( rawTx, localAccount.id, dataSource.id, userId, rawAccount.id, rawAccount.currency, ); seenTxIds.push(txRow.provider_transaction_id); const outcome = upsertTransaction(db, txRow); if (outcome === 'inserted') { transactionsNew += 1; if (debug) console.log(`[bankSync:debug] tx ${txRow.provider_transaction_id}: inserted${txRow.pending ? ' (pending)' : ''} (${rawTx.description || rawTx.payee || '—'}, ${txRow.amount}¢)`); } else if (outcome === 'posted') { transactionsPosted += 1; if (debug) console.log(`[bankSync:debug] tx ${txRow.provider_transaction_id}: pending → posted`); } else if (outcome === 'updated') { if (debug) console.log(`[bankSync:debug] tx ${txRow.provider_transaction_id}: pending amount refreshed`); } else { transactionsSkip += 1; if (debug) console.log(`[bankSync:debug] tx ${txRow.provider_transaction_id}: duplicate — skipped`); } } const orphans = pruneOrphanPending.run(localAccount.id, userId, JSON.stringify(seenTxIds)); if (orphans.changes > 0) { pendingCleared += orphans.changes; if (debug) console.log(`[bankSync:debug] Account "${rawAccount.name}": pruned ${orphans.changes} stale pending row(s)`); } } // Store any errlist warnings alongside a successful sync so users can see them const partialError = raw._errlistSummary ? sanitizeErrorMessage(`Partial sync — some connections failed: ${raw._errlistSummary}`) : null; db.prepare(` UPDATE data_sources SET last_sync_at = datetime('now'), last_error = ?, status = 'active', updated_at = datetime('now') WHERE id = ? AND user_id = ? `).run(partialError, dataSource.id, userId); if (debug) console.log(`[bankSync:debug] Source #${dataSource.id}: applying merchant rules + auto-match`); // Apply stored merchant→bill rules, then spending category rules, then score-based auto-match const { matched: autoMatched, matched_bills: matchedBills, late_attributions: lateAttributions } = applyMerchantRules(db, userId); try { applySpendingCategoryRules(db, userId); } catch { /* non-blocking */ } try { autoMatchForUser(userId); } catch { /* non-blocking */ } if (debug) console.log(`[bankSync:debug] Source #${dataSource.id}: auto-matched ${autoMatched} transaction(s)`); return { accountsUpserted, transactionsNew, transactionsSkip, transactionsPosted, pendingCleared, autoMatched, matched_bills: matchedBills || [], late_attributions: lateAttributions || [], errlist: raw._errlistSummary || null }; } // ─── Public API ─────────────────────────────────────────────────────────────── async function connectSimplefin(db, userId, setupToken) { assertEncryptionReady(); const accessUrl = await claimSetupToken(setupToken); const encrypted = encryptSecret(accessUrl); const result = db.prepare(` INSERT INTO data_sources (user_id, type, provider, name, status, encrypted_secret) VALUES (?, 'provider_sync', 'simplefin', 'SimpleFIN', 'active', ?) `).run(userId, encrypted); const dataSourceId = result.lastInsertRowid; const dataSource = db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(dataSourceId, userId); let syncResult = { accountsUpserted: 0, transactionsNew: 0, transactionsSkip: 0 }; try { syncResult = await runSync(db, userId, dataSource); } catch (err) { const msg = safeErrorMessage(err); db.prepare(` UPDATE data_sources SET last_error = ?, status = 'error', updated_at = datetime('now') WHERE id = ? `).run(msg, dataSourceId); } const fresh = db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(dataSourceId, userId); return { dataSource: decorateDataSource(fresh), ...syncResult }; } async function syncDataSource(db, userId, dataSourceId, { debug } = {}) { assertEncryptionReady(); const dataSource = db.prepare(` SELECT * FROM data_sources WHERE id = ? AND user_id = ? AND type = 'provider_sync' AND provider = 'simplefin' `).get(dataSourceId, userId); if (!dataSource) throw Object.assign(new Error('SimpleFIN connection not found'), { status: 404 }); if (!dataSource.encrypted_secret) throw new Error('No stored credentials for this connection'); const useDebug = debug ?? getBankSyncConfig().debug_logging; let syncResult; try { syncResult = await runSync(db, userId, dataSource, { debug: useDebug }); } catch (err) { const msg = safeErrorMessage(err); db.prepare(` UPDATE data_sources SET last_error = ?, status = 'error', updated_at = datetime('now') WHERE id = ? `).run(msg, dataSourceId); // Re-throw so the route can surface a meaningful error throw err; } const fresh = db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(dataSourceId, userId); return { dataSource: decorateDataSource(fresh), ...syncResult }; } async function backfillDataSource(db, userId, dataSourceId) { assertEncryptionReady(); const dataSource = db.prepare(` SELECT * FROM data_sources WHERE id = ? AND user_id = ? AND type = 'provider_sync' AND provider = 'simplefin' `).get(dataSourceId, userId); if (!dataSource) throw Object.assign(new Error('SimpleFIN connection not found'), { status: 404 }); if (!dataSource.encrypted_secret) throw new Error('No stored credentials for this connection'); let syncResult; try { syncResult = await runSync(db, userId, dataSource, { days: SYNC_DAYS_EFFECTIVE }); } catch (err) { const msg = safeErrorMessage(err); db.prepare(` UPDATE data_sources SET last_error = ?, status = 'error', updated_at = datetime('now') WHERE id = ? `).run(msg, dataSourceId); throw err; } const fresh = db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(dataSourceId, userId); return { dataSource: decorateDataSource(fresh), ...syncResult }; } function disconnectDataSource(db, userId, dataSourceId) { const row = db.prepare(` SELECT id FROM data_sources WHERE id = ? AND user_id = ? AND provider = 'simplefin' `).get(dataSourceId, userId); if (!row) throw Object.assign(new Error('SimpleFIN connection not found'), { status: 404 }); // Financial accounts cascade-delete. Transactions get data_source_id = NULL (SET NULL FK). db.prepare('DELETE FROM data_sources WHERE id = ? AND user_id = ?').run(dataSourceId, userId); } module.exports = { connectSimplefin, syncDataSource, backfillDataSource, disconnectDataSource };