BillTracker/services/userDbImportService.js

'use strict';

const crypto = require('crypto');
const fs = require('fs');
const os = require('os');
const path = require('path');
const Database = require('better-sqlite3');
const { getDb } = require('../db/database');

const MAX_SQLITE_BYTES = 50 * 1024 * 1024;
const SESSION_TTL_HOURS = 24;
const REQUIRED_TABLES = ['export_metadata', 'categories', 'bills', 'payments', 'monthly_bill_state'];
const VALID_BILLING_CYCLES = new Set(['monthly', 'quarterly', 'annually', 'irregular']);
const VALID_AUTODRAFT = new Set(['none', 'pending', 'assumed_paid', 'confirmed']);

function importError(status, message, code, details = []) {
  const err = new Error(message);
  err.status = status;
  err.code = code;
  err.details = details;
  return err;
}

function sanitizeFilename(value) {
  return value
    ? String(value).replace(/[^a-zA-Z0-9._\-\s]/g, '').trim().slice(0, 255) || null
    : null;
}

function assertSqliteBuffer(buffer) {
  if (!Buffer.isBuffer(buffer) || buffer.length === 0) {
    throw importError(400, 'SQLite export file is required.', 'USER_DB_IMPORT_FILE_REQUIRED');
  }
  if (buffer.length > MAX_SQLITE_BYTES) {
    throw importError(400, 'SQLite export file is too large. Maximum size is 50 MB.', 'USER_DB_IMPORT_FILE_TOO_LARGE');
  }
  if (buffer.length < 16 || buffer.subarray(0, 16).toString('binary') !== 'SQLite format 3\u0000') {
    throw importError(400, 'File is not a valid SQLite database export.', 'USER_DB_IMPORT_INVALID_SQLITE');
  }
}

function withTempSqlite(buffer, fn) {
  const file = path.join(os.tmpdir(), `bill-tracker-user-import-${Date.now()}-${crypto.randomBytes(4).toString('hex')}.sqlite`);
  fs.writeFileSync(file, buffer, { mode: 0o600 });
  let src = null;
  try {
    src = new Database(file, { readonly: true, fileMustExist: true });
    src.prepare('SELECT name FROM sqlite_master LIMIT 1').get();
    return fn(src);
  } finally {
    try { src?.close(); } catch {}
    try { fs.unlinkSync(file); } catch {}
  }
}

function tableNames(db) {
  return new Set(db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map(r => r.name));
}

function tableColumns(db, table) {
  return new Set(db.prepare(`PRAGMA table_info(${table})`).all().map(c => c.name));
}

function selectKnown(db, table, columns) {
  const cols = tableColumns(db, table);
  const selected = columns.filter(c => cols.has(c));
  if (!selected.length) return [];
  return db.prepare(`SELECT ${selected.join(', ')} FROM ${table}`).all();
}

function toInt(value, fallback = null) {
  if (value === undefined || value === null || value === '') return fallback;
  const n = Number(value);
  return Number.isInteger(n) ? n : fallback;
}

function toNumber(value, fallback = null) {
  if (value === undefined || value === null || value === '') return fallback;
  const n = Number(value);
  return Number.isFinite(n) ? n : fallback;
}

function cleanText(value, max = 1000) {
  if (value === undefined || value === null) return null;
  const s = String(value).trim();
  return s ? s.slice(0, max) : null;
}

function cleanDate(value) {
  const s = cleanText(value, 32);
  return /^\d{4}-\d{2}-\d{2}$/.test(s || '') ? s : null;
}

function normalizeName(value) {
  return String(value || '').trim().toLowerCase();
}

function parseMetadata(src) {
  const row = src.prepare("SELECT value FROM export_metadata WHERE key = 'metadata_json'").get();
  if (!row?.value) {
    throw importError(400, 'SQLite file is not a BillTracker user data export.', 'USER_DB_IMPORT_NOT_USER_EXPORT');
  }
  let metadata;
  try {
    metadata = JSON.parse(row.value);
  } catch {
    throw importError(400, 'SQLite export metadata could not be read.', 'USER_DB_IMPORT_BAD_METADATA');
  }
  if (metadata?.export_type !== 'user_data') {
    throw importError(400, 'SQLite file is not a user data export created by this app.', 'USER_DB_IMPORT_NOT_USER_EXPORT');
  }
  return metadata;
}

function sanitizeCategory(row) {
  const name = cleanText(row.name, 120);
  if (!name) return null;
  return {
    old_id: toInt(row.id),
    name,
    created_at: cleanText(row.created_at, 32),
    updated_at: cleanText(row.updated_at, 32),
  };
}

function sanitizeBill(row) {
  const name = cleanText(row.name, 160);
  const dueDay = toInt(row.due_day);
  if (!name || dueDay < 1 || dueDay > 31) return null;
  const interestRate = toNumber(row.interest_rate, null);
  return {
    old_id: toInt(row.id),
    name,
    category_id: toInt(row.category_id),
    due_day: dueDay,
    override_due_date: cleanText(row.override_due_date, 32),
    bucket: dueDay <= 14 ? '1st' : '15th',
    expected_amount: Math.max(0, toNumber(row.expected_amount, 0) ?? 0),
    interest_rate: interestRate == null || interestRate < 0 || interestRate > 100 ? null : interestRate,
    billing_cycle: VALID_BILLING_CYCLES.has(row.billing_cycle) ? row.billing_cycle : 'monthly',
    autopay_enabled: toInt(row.autopay_enabled, 0) ? 1 : 0,
    autodraft_status: VALID_AUTODRAFT.has(row.autodraft_status) ? row.autodraft_status : 'none',
    website: cleanText(row.website, 500),
    username: cleanText(row.username, 255),
    account_info: cleanText(row.account_info, 500),
    has_2fa: toInt(row.has_2fa, 0) ? 1 : 0,
    active: row.active === undefined ? 1 : (toInt(row.active, 1) ? 1 : 0),
    notes: cleanText(row.notes, 2000),
    created_at: cleanText(row.created_at, 32),
    updated_at: cleanText(row.updated_at, 32),
  };
}

function sanitizePayment(row, validBillIds) {
  const billId = toInt(row.bill_id);
  const amount = toNumber(row.amount);
  const paidDate = cleanDate(row.paid_date);
  if (!billId || !validBillIds.has(billId) || amount == null || amount < 0 || !paidDate) return null;
  return {
    old_id: toInt(row.id),
    bill_id: billId,
    amount,
    paid_date: paidDate,
    method: cleanText(row.method, 120),
    notes: cleanText(row.notes, 2000),
    created_at: cleanText(row.created_at, 32),
    updated_at: cleanText(row.updated_at, 32),
  };
}

function sanitizeMonthlyState(row, validBillIds) {
  const billId = toInt(row.bill_id);
  const year = toInt(row.year);
  const month = toInt(row.month);
  if (!billId || !validBillIds.has(billId) || year < 2000 || year > 2100 || month < 1 || month > 12) return null;
  const actual = toNumber(row.actual_amount, null);
  return {
    old_id: toInt(row.id),
    bill_id: billId,
    year,
    month,
    actual_amount: actual == null || actual < 0 ? null : actual,
    notes: cleanText(row.notes, 2000),
    is_skipped: toInt(row.is_skipped, 0) ? 1 : 0,
    created_at: cleanText(row.created_at, 32),
    updated_at: cleanText(row.updated_at, 32),
  };
}

function readExportData(src) {
  const names = tableNames(src);
  const missing = REQUIRED_TABLES.filter(t => !names.has(t));
  if (missing.length) {
    throw importError(400, 'SQLite file is not a supported BillTracker user export.', 'USER_DB_IMPORT_UNSUPPORTED_SCHEMA', missing.map(table => ({
      table,
      message: 'Required user export table is missing',
    })));
  }

  const metadata = parseMetadata(src);
  const categories = selectKnown(src, 'categories', ['id', 'name', 'created_at', 'updated_at'])
    .map(sanitizeCategory).filter(Boolean);
  const bills = selectKnown(src, 'bills', [
    'id', 'name', 'category_id', 'due_day', 'override_due_date', 'bucket', 'expected_amount', 'interest_rate',
    'billing_cycle', 'autopay_enabled', 'autodraft_status', 'website', 'username', 'account_info', 'has_2fa',
    'active', 'notes', 'created_at', 'updated_at',
  ]).map(sanitizeBill).filter(Boolean);
  const validBillIds = new Set(bills.map(b => b.old_id).filter(Boolean));
  const payments = selectKnown(src, 'payments', ['id', 'bill_id', 'amount', 'paid_date', 'method', 'notes', 'created_at', 'updated_at'])
    .map(row => sanitizePayment(row, validBillIds)).filter(Boolean);
  const monthlyState = selectKnown(src, 'monthly_bill_state', ['id', 'bill_id', 'year', 'month', 'actual_amount', 'notes', 'is_skipped', 'created_at', 'updated_at'])
    .map(row => sanitizeMonthlyState(row, validBillIds)).filter(Boolean);
  const notes = names.has('notes')
    ? selectKnown(src, 'notes', ['type', 'bill_id', 'payment_id', 'monthly_state_id', 'year', 'month', 'notes'])
      .map(n => ({ ...n, notes: cleanText(n.notes, 2000) })).filter(n => n.notes)
    : [];

  return { metadata, categories, bills, payments, monthly_bill_state: monthlyState, notes };
}

function existingLookups(db, userId) {
  const categories = db.prepare('SELECT id, name FROM categories WHERE user_id = ?').all(userId);
  const bills = db.prepare('SELECT id, name, due_day FROM bills WHERE user_id = ?').all(userId);
  return {
    categoryByName: new Map(categories.map(c => [normalizeName(c.name), c])),
    billByKey: new Map(bills.map(b => [`${normalizeName(b.name)}|${b.due_day}`, b])),
  };
}

function buildPreview(userId, data, originalFilename) {
  const db = getDb();
  const lookups = existingLookups(db, userId);
  const categoryPlan = data.categories.map(c => {
    const existing = lookups.categoryByName.get(normalizeName(c.name));
    return {
      old_id: c.old_id,
      name: c.name,
      action: existing ? 'skip' : 'create',
      existing_id: existing?.id ?? null,
      reason: existing ? 'Category already exists for this user' : 'Category is missing for this user',
    };
  });

  const categoryMap = new Map(categoryPlan.map(c => [c.old_id, c]));
  const billPlan = data.bills.map(b => {
    const existing = lookups.billByKey.get(`${normalizeName(b.name)}|${b.due_day}`);
    const category = categoryMap.get(b.category_id);
    return {
      old_id: b.old_id,
      name: b.name,
      due_day: b.due_day,
      category_name: category?.name ?? null,
      action: existing ? 'skip' : 'create',
      existing_id: existing?.id ?? null,
      reason: existing ? 'Bill with same name and due day already exists for this user' : 'Bill is missing for this user',
    };
  });

  const billPlanByOldId = new Map(billPlan.map(b => [b.old_id, b]));
  const paymentPlan = data.payments.map(p => ({
    old_id: p.old_id,
    bill_old_id: p.bill_id,
    action: billPlanByOldId.has(p.bill_id) ? 'create_or_skip_duplicate' : 'conflict',
    reason: billPlanByOldId.has(p.bill_id) ? 'Will import if no duplicate payment exists' : 'Referenced bill is not present in export',
  }));
  const monthlyPlan = data.monthly_bill_state.map(m => ({
    old_id: m.old_id,
    bill_old_id: m.bill_id,
    year: m.year,
    month: m.month,
    action: billPlanByOldId.has(m.bill_id) ? 'create_or_skip_duplicate' : 'conflict',
    reason: billPlanByOldId.has(m.bill_id) ? 'Will import if monthly state is missing' : 'Referenced bill is not present in export',
  }));

  const summary = {
    categories: {
      total: data.categories.length,
      create: categoryPlan.filter(x => x.action === 'create').length,
      skip: categoryPlan.filter(x => x.action === 'skip').length,
      conflict: 0,
    },
    bills: {
      total: data.bills.length,
      create: billPlan.filter(x => x.action === 'create').length,
      skip: billPlan.filter(x => x.action === 'skip').length,
      conflict: 0,
    },
    payments: {
      total: data.payments.length,
      create: data.payments.length,
      skip: 0,
      conflict: paymentPlan.filter(x => x.action === 'conflict').length,
    },
    monthly_bill_state: {
      total: data.monthly_bill_state.length,
      create: data.monthly_bill_state.length,
      skip: 0,
      conflict: monthlyPlan.filter(x => x.action === 'conflict').length,
    },
    notes: {
      total: data.notes.length,
      create: 0,
      skip: data.notes.length,
      conflict: 0,
    },
  };

  const warnings = [
    'Conflicts and duplicates are skipped by default.',
    'This import does not restore users, passwords, sessions, admin settings, SMTP credentials, backups, or server paths.',
  ];

  return {
    import_type: 'user_db',
    source_filename: originalFilename,
    metadata: data.metadata,
    counts: {
      bills: data.bills.length,
      categories: data.categories.length,
      payments: data.payments.length,
      monthly_bill_state: data.monthly_bill_state.length,
      notes: data.notes.length,
    },
    summary,
    proposed_actions: {
      categories: categoryPlan.slice(0, 100),
      bills: billPlan.slice(0, 100),
      payments: paymentPlan.slice(0, 100),
      monthly_bill_state: monthlyPlan.slice(0, 100),
    },
    warnings,
  };
}

function saveSession(db, userId, sessionData) {
  const id = crypto.randomUUID();
  const now = new Date();
  const expires = new Date(now.getTime() + SESSION_TTL_HOURS * 60 * 60 * 1000);
  db.prepare(`
    INSERT INTO import_sessions (id, user_id, created_at, expires_at, preview_json)
    VALUES (?, ?, ?, ?, ?)
  `).run(id, userId, now.toISOString(), expires.toISOString(), JSON.stringify(sessionData));
  return id;
}

function loadSession(db, userId, importSessionId) {
  const row = db.prepare(`
    SELECT preview_json FROM import_sessions
    WHERE id = ? AND user_id = ? AND expires_at > ?
  `).get(importSessionId, userId, new Date().toISOString());
  if (!row) {
    throw importError(400, 'Import session is invalid or expired. Preview the SQLite export again.', 'USER_DB_IMPORT_SESSION_INVALID');
  }
  const data = JSON.parse(row.preview_json);
  if (data.import_type !== 'user_db') {
    throw importError(400, 'Import session is not a SQLite user export preview.', 'USER_DB_IMPORT_SESSION_TYPE');
  }
  return data;
}

async function previewUserDbImport(userId, buffer, options = {}) {
  assertSqliteBuffer(buffer);
  const originalFilename = sanitizeFilename(options.original_filename);
  return withTempSqlite(buffer, (src) => {
    const data = readExportData(src);
    const preview = buildPreview(userId, data, originalFilename);
    const db = getDb();
    const importSessionId = saveSession(db, userId, {
      import_type: 'user_db',
      source_filename: originalFilename,
      metadata: data.metadata,
      data,
      preview,
    });
    return {
      import_session_id: importSessionId,
      ...preview,
    };
  });
}

function ensureCategory(db, userId, category, summary, details) {
  const existing = db.prepare('SELECT id FROM categories WHERE user_id = ? AND LOWER(name) = LOWER(?)').get(userId, category.name);
  if (existing) {
    summary.rows_skipped++;
    summary.rows_conflicted++;
    details.categories.skipped++;
    return existing.id;
  }
  const result = db.prepare('INSERT INTO categories (user_id, name) VALUES (?, ?)').run(userId, category.name);
  summary.rows_created++;
  details.categories.created++;
  return result.lastInsertRowid;
}

function ensureBill(db, userId, bill, categoryId, summary, details) {
  const existing = db.prepare('SELECT id FROM bills WHERE user_id = ? AND LOWER(name) = LOWER(?) AND due_day = ?')
    .get(userId, bill.name, bill.due_day);
  if (existing) {
    summary.rows_skipped++;
    summary.rows_conflicted++;
    details.bills.skipped++;
    return existing.id;
  }
  const result = db.prepare(`
    INSERT INTO bills
      (user_id, name, category_id, due_day, override_due_date, bucket, expected_amount, interest_rate,
       billing_cycle, autopay_enabled, autodraft_status, website, username, account_info, has_2fa,
       active, notes)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  `).run(
    userId,
    bill.name,
    categoryId || null,
    bill.due_day,
    bill.override_due_date || null,
    bill.bucket,
    bill.expected_amount,
    bill.interest_rate,
    bill.billing_cycle,
    bill.autopay_enabled,
    bill.autodraft_status,
    bill.website,
    bill.username,
    bill.account_info,
    bill.has_2fa,
    bill.active,
    bill.notes,
  );
  summary.rows_created++;
  details.bills.created++;
  return result.lastInsertRowid;
}

function importPayment(db, targetBillId, payment, summary, details) {
  const duplicate = db.prepare(`
    SELECT p.id FROM payments p
    WHERE p.bill_id = ? AND p.paid_date = ? AND p.amount = ? AND COALESCE(p.method, '') = COALESCE(?, '')
      AND p.deleted_at IS NULL
  `).get(targetBillId, payment.paid_date, payment.amount, payment.method);
  if (duplicate) {
    summary.rows_skipped++;
    summary.rows_conflicted++;
    details.payments.skipped++;
    return;
  }
  db.prepare(`
    INSERT INTO payments (bill_id, amount, paid_date, method, notes)
    VALUES (?, ?, ?, ?, ?)
  `).run(targetBillId, payment.amount, payment.paid_date, payment.method, payment.notes);
  summary.rows_created++;
  details.payments.created++;
}

function importMonthlyState(db, targetBillId, row, summary, details) {
  const duplicate = db.prepare(`
    SELECT id FROM monthly_bill_state WHERE bill_id = ? AND year = ? AND month = ?
  `).get(targetBillId, row.year, row.month);
  if (duplicate) {
    summary.rows_skipped++;
    summary.rows_conflicted++;
    details.monthly_bill_state.skipped++;
    return;
  }
  db.prepare(`
    INSERT INTO monthly_bill_state (bill_id, year, month, actual_amount, notes, is_skipped)
    VALUES (?, ?, ?, ?, ?, ?)
  `).run(targetBillId, row.year, row.month, row.actual_amount, row.notes, row.is_skipped);
  summary.rows_created++;
  details.monthly_bill_state.created++;
}

async function applyUserDbImport(userId, importSessionId, options = {}) {
  if (options.overwrite) {
    throw importError(400, 'Overwrite is not supported for user SQLite imports. Existing data is skipped.', 'USER_DB_IMPORT_OVERWRITE_UNSUPPORTED');
  }

  const db = getDb();
  const session = loadSession(db, userId, importSessionId);
  const { data } = session;
  const summary = {
    import_type: 'user_db',
    rows_created: 0,
    rows_updated: 0,
    rows_skipped: 0,
    rows_conflicted: 0,
    rows_errored: 0,
  };
  const details = {
    categories: { created: 0, skipped: 0, errored: 0 },
    bills: { created: 0, skipped: 0, errored: 0 },
    payments: { created: 0, skipped: 0, errored: 0 },
    monthly_bill_state: { created: 0, skipped: 0, errored: 0 },
    notes: { created: 0, skipped: data.notes?.length || 0, errored: 0 },
  };
  summary.rows_skipped += details.notes.skipped;

  const tx = db.transaction(() => {
    const categoryIdMap = new Map();
    for (const category of data.categories) {
      categoryIdMap.set(category.old_id, ensureCategory(db, userId, category, summary, details));
    }

    const billIdMap = new Map();
    for (const bill of data.bills) {
      const targetCategoryId = bill.category_id ? categoryIdMap.get(bill.category_id) : null;
      billIdMap.set(bill.old_id, ensureBill(db, userId, bill, targetCategoryId, summary, details));
    }

    for (const payment of data.payments) {
      const targetBillId = billIdMap.get(payment.bill_id);
      if (!targetBillId) {
        summary.rows_skipped++;
        summary.rows_conflicted++;
        details.payments.skipped++;
        continue;
      }
      importPayment(db, targetBillId, payment, summary, details);
    }

    for (const row of data.monthly_bill_state) {
      const targetBillId = billIdMap.get(row.bill_id);
      if (!targetBillId) {
        summary.rows_skipped++;
        summary.rows_conflicted++;
        details.monthly_bill_state.skipped++;
        continue;
      }
      importMonthlyState(db, targetBillId, row, summary, details);
    }

    db.prepare(`
      INSERT INTO import_history
        (user_id, imported_at, source_filename, file_type, sheet_name, rows_parsed,
         rows_created, rows_updated, rows_skipped, rows_ambiguous, rows_errored,
         options_json, summary_json)
      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    `).run(
      userId,
      new Date().toISOString(),
      session.source_filename || null,
      'sqlite',
      'User SQLite export',
      data.categories.length + data.bills.length + data.payments.length + data.monthly_bill_state.length + (data.notes?.length || 0),
      summary.rows_created,
      summary.rows_updated,
      summary.rows_skipped,
      summary.rows_conflicted,
      summary.rows_errored,
      JSON.stringify({ overwrite: false }),
      JSON.stringify({ ...summary, details }),
    );

    db.prepare('DELETE FROM import_sessions WHERE id = ? AND user_id = ?').run(importSessionId, userId);
  });

  tx();
  return { ...summary, details };
}

module.exports = {
  previewUserDbImport,
  applyUserDbImport,
};