BillTracker/services/bankSyncService.js

'use strict';

const { assertEncryptionReady, encryptSecret, decryptSecret } = require('./encryptionService');
const {
  claimSetupToken,
  fetchAccountsAndTransactions,
  normalizeAccount,
  normalizeTransaction,
  sanitizeErrorMessage,
} = require('./simplefinService');
const { getBankSyncConfig, SYNC_DAYS_EFFECTIVE, SYNC_DAYS_DEFAULT } = require('./bankSyncConfigService');
const { decorateDataSource } = require('./transactionService');
const { applyMerchantRules } = require('./billMerchantRuleService');
const { applySpendingCategoryRules } = require('./spendingService');
const { applyMerchantStoreMatches } = require('./merchantStoreMatchService');
const { autoMatchForUser } = require('./matchSuggestionService');
const { getUserSettings } = require('./userSettings');

function sinceEpochDays(days) {
  return Math.floor((Date.now() - days * 86400 * 1000) / 1000);
}

function safeErrorMessage(err) {
  return sanitizeErrorMessage(err?.message || String(err || 'Sync failed'));
}

// Upsert a single financial account, return the local row.
function upsertAccount(db, accountRow) {
  const existing = db.prepare(`
    SELECT id, monitored FROM financial_accounts
    WHERE data_source_id = ? AND provider_account_id = ? AND user_id = ?
  `).get(accountRow.data_source_id, accountRow.provider_account_id, accountRow.user_id);

  if (existing) {
    db.prepare(`
      UPDATE financial_accounts
      SET name = ?, org_name = ?, currency = ?, balance = ?, available_balance = ?,
          raw_data = ?, updated_at = datetime('now')
      WHERE id = ?
    `).run(
      accountRow.name, accountRow.org_name, accountRow.currency,
      accountRow.balance, accountRow.available_balance, accountRow.raw_data,
      existing.id,
    );
    return existing;
  }

  const result = db.prepare(`
    INSERT INTO financial_accounts
      (user_id, data_source_id, provider_account_id, name, org_name, account_type,
       currency, balance, available_balance, raw_data)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  `).run(
    accountRow.user_id, accountRow.data_source_id, accountRow.provider_account_id,
    accountRow.name, accountRow.org_name, accountRow.account_type,
    accountRow.currency, accountRow.balance, accountRow.available_balance, accountRow.raw_data,
  );
  return { id: result.lastInsertRowid, monitored: 1 };
}

// Insert a transaction, or update one we previously stored as PENDING. A pending charge
// can change amount before it settles, and eventually flips pending → posted (gaining a
// real posted_date). A transaction we already recorded as posted is final and left alone;
// a row the user has matched or ignored is never touched.
// Returns: 'inserted' | 'posted' (pending→settled) | 'updated' (pending refreshed) | 'skipped'.
function upsertTransaction(db, txRow) {
  const existing = db.prepare(`
    SELECT id, pending, match_status FROM transactions
    WHERE data_source_id = ? AND provider_transaction_id = ?
  `).get(txRow.data_source_id, txRow.provider_transaction_id);

  if (!existing) {
    try {
      db.prepare(`
        INSERT INTO transactions
          (user_id, data_source_id, account_id, provider_transaction_id,
           source_type, posted_date, transacted_at, amount, currency,
           description, payee, memo, match_status, ignored, pending, raw_data)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
      `).run(
        txRow.user_id, txRow.data_source_id, txRow.account_id, txRow.provider_transaction_id,
        txRow.source_type, txRow.posted_date, txRow.transacted_at, txRow.amount, txRow.currency,
        txRow.description, txRow.payee, txRow.memo, txRow.match_status, txRow.ignored, txRow.pending, txRow.raw_data,
      );
      return 'inserted';
    } catch (err) {
      if (err.code === 'SQLITE_CONSTRAINT_UNIQUE' || (err.message || '').includes('UNIQUE')) {
        return 'skipped';
      }
      throw err;
    }
  }

  // Only refresh rows still pending in our DB and not yet acted on by the user.
  if (existing.pending === 1 && existing.match_status === 'unmatched') {
    db.prepare(`
      UPDATE transactions
      SET posted_date = ?, transacted_at = ?, amount = ?, currency = ?,
          description = ?, payee = ?, memo = ?, pending = ?, raw_data = ?,
          updated_at = datetime('now')
      WHERE id = ?
    `).run(
      txRow.posted_date, txRow.transacted_at, txRow.amount, txRow.currency,
      txRow.description, txRow.payee, txRow.memo, txRow.pending, txRow.raw_data,
      existing.id,
    );
    return txRow.pending === 0 ? 'posted' : 'updated';
  }

  return 'skipped';
}

async function runSync(db, userId, dataSource, { days, debug = false } = {}) {
  const accessUrl   = decryptSecret(dataSource.encrypted_secret);
  const isFirstSync = !dataSource.last_sync_at;
  // Explicit `days` param (e.g. backfill) takes precedence.
  // Initial seed always uses the full SYNC_DAYS_EFFECTIVE window regardless of admin config.
  // Routine syncs use the admin-configured sync_days (default 30); falls back to SYNC_DAYS_DEFAULT.
  const config    = getBankSyncConfig();
  const syncDays  = days ?? (isFirstSync ? SYNC_DAYS_EFFECTIVE : (config.sync_days || SYNC_DAYS_DEFAULT));
  const since = sinceEpochDays(syncDays);

  if (debug) console.log(`[bankSync:debug] Source #${dataSource.id} user ${userId}: fetching ${syncDays} days from SimpleFIN (since epoch ${since})`);

  const raw = await fetchAccountsAndTransactions(accessUrl, since);
  const accounts = Array.isArray(raw.accounts) ? raw.accounts : [];

  if (debug) console.log(`[bankSync:debug] Source #${dataSource.id}: SimpleFIN returned ${accounts.length} account(s)`);

  if (raw._errlistSummary) {
    console.warn(`[bankSync] errlist for source ${dataSource.id}: ${raw._errlistSummary}`);
  }

  let accountsUpserted    = 0;
  let transactionsNew     = 0;
  let transactionsSkip    = 0;
  let transactionsPosted  = 0; // pending → settled this cycle
  let pendingCleared      = 0; // stale pending rows pruned (re-posted under new id / dropped)

  // Remove pending rows we hold for an account that are no longer in the feed — a pending
  // charge that re-posted under a new id, or was dropped by the bank before settling.
  const pruneOrphanPending = db.prepare(`
    DELETE FROM transactions
    WHERE account_id = ? AND user_id = ? AND pending = 1 AND match_status = 'unmatched'
      AND provider_transaction_id NOT IN (SELECT value FROM json_each(?))
  `);

  for (const rawAccount of accounts) {
    const accountRow   = normalizeAccount(rawAccount, dataSource.id, userId);
    const localAccount = upsertAccount(db, accountRow);
    accountsUpserted  += 1;

    const txList = rawAccount.transactions || [];
    if (debug) console.log(`[bankSync:debug]   Account "${rawAccount.name}" (monitored=${localAccount.monitored}): ${txList.length} transaction(s)`);

    if (localAccount.monitored === 0) continue;

    const seenTxIds = [];
    for (const rawTx of txList) {
      const txRow = normalizeTransaction(
        rawTx, localAccount.id, dataSource.id, userId, rawAccount.id, rawAccount.currency,
      );
      seenTxIds.push(txRow.provider_transaction_id);
      const outcome = upsertTransaction(db, txRow);
      if (outcome === 'inserted') {
        transactionsNew += 1;
        if (debug) console.log(`[bankSync:debug]     tx ${txRow.provider_transaction_id}: inserted${txRow.pending ? ' (pending)' : ''} (${rawTx.description || rawTx.payee || '—'}, ${txRow.amount}¢)`);
      } else if (outcome === 'posted') {
        transactionsPosted += 1;
        if (debug) console.log(`[bankSync:debug]     tx ${txRow.provider_transaction_id}: pending → posted`);
      } else if (outcome === 'updated') {
        if (debug) console.log(`[bankSync:debug]     tx ${txRow.provider_transaction_id}: pending amount refreshed`);
      } else {
        transactionsSkip += 1;
        if (debug) console.log(`[bankSync:debug]     tx ${txRow.provider_transaction_id}: duplicate — skipped`);
      }
    }

    const orphans = pruneOrphanPending.run(localAccount.id, userId, JSON.stringify(seenTxIds));
    if (orphans.changes > 0) {
      pendingCleared += orphans.changes;
      if (debug) console.log(`[bankSync:debug]   Account "${rawAccount.name}": pruned ${orphans.changes} stale pending row(s)`);
    }
  }

  // Store any errlist warnings alongside a successful sync so users can see them
  const partialError = raw._errlistSummary
    ? sanitizeErrorMessage(`Partial sync — some connections failed: ${raw._errlistSummary}`)
    : null;

  db.prepare(`
    UPDATE data_sources
    SET last_sync_at = datetime('now'), last_error = ?, status = 'active', updated_at = datetime('now')
    WHERE id = ? AND user_id = ?
  `).run(partialError, dataSource.id, userId);

  if (debug) console.log(`[bankSync:debug] Source #${dataSource.id}: applying merchant rules + auto-match`);

  // Apply stored merchant→bill rules, then spending category rules, then score-based auto-match
  const { matched: autoMatched, matched_bills: matchedBills, late_attributions: lateAttributions } = applyMerchantRules(db, userId);
  try { applySpendingCategoryRules(db, userId); } catch { /* non-blocking */ }
  try {
    if (getUserSettings(userId).bank_auto_categorize_merchants !== 'false') {
      applyMerchantStoreMatches(db, userId);
    }
  } catch { /* non-blocking */ }
  try { autoMatchForUser(userId); } catch { /* non-blocking */ }

  if (debug) console.log(`[bankSync:debug] Source #${dataSource.id}: auto-matched ${autoMatched} transaction(s)`);

  return { accountsUpserted, transactionsNew, transactionsSkip, transactionsPosted, pendingCleared, autoMatched, matched_bills: matchedBills || [], late_attributions: lateAttributions || [], errlist: raw._errlistSummary || null };
}

// ─── Public API ───────────────────────────────────────────────────────────────

async function connectSimplefin(db, userId, setupToken) {
  assertEncryptionReady();

  const accessUrl = await claimSetupToken(setupToken);
  const encrypted = encryptSecret(accessUrl);

  const result = db.prepare(`
    INSERT INTO data_sources (user_id, type, provider, name, status, encrypted_secret)
    VALUES (?, 'provider_sync', 'simplefin', 'SimpleFIN', 'active', ?)
  `).run(userId, encrypted);

  const dataSourceId = result.lastInsertRowid;
  const dataSource   = db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(dataSourceId, userId);

  let syncResult = { accountsUpserted: 0, transactionsNew: 0, transactionsSkip: 0 };
  try {
    syncResult = await runSync(db, userId, dataSource);
  } catch (err) {
    const msg = safeErrorMessage(err);
    db.prepare(`
      UPDATE data_sources SET last_error = ?, status = 'error', updated_at = datetime('now')
      WHERE id = ?
    `).run(msg, dataSourceId);
  }

  const fresh = db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(dataSourceId, userId);
  return { dataSource: decorateDataSource(fresh), ...syncResult };
}

async function syncDataSource(db, userId, dataSourceId, { debug } = {}) {
  assertEncryptionReady();

  const dataSource = db.prepare(`
    SELECT * FROM data_sources
    WHERE id = ? AND user_id = ? AND type = 'provider_sync' AND provider = 'simplefin'
  `).get(dataSourceId, userId);

  if (!dataSource) throw Object.assign(new Error('SimpleFIN connection not found'), { status: 404 });
  if (!dataSource.encrypted_secret) throw new Error('No stored credentials for this connection');

  const useDebug = debug ?? getBankSyncConfig().debug_logging;

  let syncResult;
  try {
    syncResult = await runSync(db, userId, dataSource, { debug: useDebug });
  } catch (err) {
    const msg = safeErrorMessage(err);
    db.prepare(`
      UPDATE data_sources SET last_error = ?, status = 'error', updated_at = datetime('now')
      WHERE id = ?
    `).run(msg, dataSourceId);
    // Re-throw so the route can surface a meaningful error
    throw err;
  }

  const fresh = db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(dataSourceId, userId);
  return { dataSource: decorateDataSource(fresh), ...syncResult };
}

async function backfillDataSource(db, userId, dataSourceId) {
  assertEncryptionReady();

  const dataSource = db.prepare(`
    SELECT * FROM data_sources
    WHERE id = ? AND user_id = ? AND type = 'provider_sync' AND provider = 'simplefin'
  `).get(dataSourceId, userId);

  if (!dataSource) throw Object.assign(new Error('SimpleFIN connection not found'), { status: 404 });
  if (!dataSource.encrypted_secret) throw new Error('No stored credentials for this connection');

  let syncResult;
  try {
    syncResult = await runSync(db, userId, dataSource, { days: SYNC_DAYS_EFFECTIVE });
  } catch (err) {
    const msg = safeErrorMessage(err);
    db.prepare(`
      UPDATE data_sources SET last_error = ?, status = 'error', updated_at = datetime('now')
      WHERE id = ?
    `).run(msg, dataSourceId);
    throw err;
  }

  const fresh = db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(dataSourceId, userId);
  return { dataSource: decorateDataSource(fresh), ...syncResult };
}

function disconnectDataSource(db, userId, dataSourceId) {
  const row = db.prepare(`
    SELECT id FROM data_sources WHERE id = ? AND user_id = ? AND provider = 'simplefin'
  `).get(dataSourceId, userId);

  if (!row) throw Object.assign(new Error('SimpleFIN connection not found'), { status: 404 });

  // Financial accounts cascade-delete. Transactions get data_source_id = NULL (SET NULL FK).
  db.prepare('DELETE FROM data_sources WHERE id = ? AND user_id = ?').run(dataSourceId, userId);
}

module.exports = { connectSimplefin, syncDataSource, backfillDataSource, disconnectDataSource };