'use strict'; const crypto = require('crypto'); const fs = require('fs'); const os = require('os'); const path = require('path'); const Database = require('better-sqlite3'); const { getDb } = require('../db/database'); const MAX_SQLITE_BYTES = 50 * 1024 * 1024; const SESSION_TTL_HOURS = 24; const REQUIRED_TABLES = ['export_metadata', 'categories', 'bills', 'payments', 'monthly_bill_state']; const VALID_BILLING_CYCLES = new Set(['monthly', 'quarterly', 'annually', 'irregular']); const VALID_AUTODRAFT = new Set(['none', 'pending', 'assumed_paid', 'confirmed']); function importError(status, message, code, details = []) { const err = new Error(message); err.status = status; err.code = code; err.details = details; return err; } function sanitizeFilename(value) { return value ? String(value).replace(/[^a-zA-Z0-9._\-\s]/g, '').trim().slice(0, 255) || null : null; } function assertSqliteBuffer(buffer) { if (!Buffer.isBuffer(buffer) || buffer.length === 0) { throw importError(400, 'SQLite export file is required.', 'USER_DB_IMPORT_FILE_REQUIRED'); } if (buffer.length > MAX_SQLITE_BYTES) { throw importError(400, 'SQLite export file is too large. Maximum size is 50 MB.', 'USER_DB_IMPORT_FILE_TOO_LARGE'); } if (buffer.length < 16 || buffer.subarray(0, 16).toString('binary') !== 'SQLite format 3\u0000') { throw importError(400, 'File is not a valid SQLite database export.', 'USER_DB_IMPORT_INVALID_SQLITE'); } } function withTempSqlite(buffer, fn) { const file = path.join(os.tmpdir(), `bill-tracker-user-import-${Date.now()}-${crypto.randomBytes(4).toString('hex')}.sqlite`); fs.writeFileSync(file, buffer, { mode: 0o600 }); let src = null; try { src = new Database(file, { readonly: true, fileMustExist: true }); src.prepare('SELECT name FROM sqlite_master LIMIT 1').get(); return fn(src); } finally { try { src?.close(); } catch {} try { fs.unlinkSync(file); } catch {} } } function tableNames(db) { return new Set(db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map(r => r.name)); } function tableColumns(db, table) { return new Set(db.prepare(`PRAGMA table_info(${table})`).all().map(c => c.name)); } function selectKnown(db, table, columns) { const cols = tableColumns(db, table); const selected = columns.filter(c => cols.has(c)); if (!selected.length) return []; return db.prepare(`SELECT ${selected.join(', ')} FROM ${table}`).all(); } function toInt(value, fallback = null) { if (value === undefined || value === null || value === '') return fallback; const n = Number(value); return Number.isInteger(n) ? n : fallback; } function toNumber(value, fallback = null) { if (value === undefined || value === null || value === '') return fallback; const n = Number(value); return Number.isFinite(n) ? n : fallback; } function cleanText(value, max = 1000) { if (value === undefined || value === null) return null; const s = String(value).trim(); return s ? s.slice(0, max) : null; } function cleanDate(value) { const s = cleanText(value, 32); return /^\d{4}-\d{2}-\d{2}$/.test(s || '') ? s : null; } function normalizeName(value) { return String(value || '').trim().toLowerCase(); } function parseMetadata(src) { const row = src.prepare("SELECT value FROM export_metadata WHERE key = 'metadata_json'").get(); if (!row?.value) { throw importError(400, 'SQLite file is not a BillTracker user data export.', 'USER_DB_IMPORT_NOT_USER_EXPORT'); } let metadata; try { metadata = JSON.parse(row.value); } catch { throw importError(400, 'SQLite export metadata could not be read.', 'USER_DB_IMPORT_BAD_METADATA'); } if (metadata?.export_type !== 'user_data') { throw importError(400, 'SQLite file is not a user data export created by this app.', 'USER_DB_IMPORT_NOT_USER_EXPORT'); } return metadata; } function sanitizeCategory(row) { const name = cleanText(row.name, 120); if (!name) return null; return { old_id: toInt(row.id), name, created_at: cleanText(row.created_at, 32), updated_at: cleanText(row.updated_at, 32), }; } function sanitizeBill(row) { const name = cleanText(row.name, 160); const dueDay = toInt(row.due_day); if (!name || dueDay < 1 || dueDay > 31) return null; const interestRate = toNumber(row.interest_rate, null); return { old_id: toInt(row.id), name, category_id: toInt(row.category_id), due_day: dueDay, override_due_date: cleanText(row.override_due_date, 32), bucket: dueDay <= 14 ? '1st' : '15th', expected_amount: Math.max(0, toNumber(row.expected_amount, 0) ?? 0), interest_rate: interestRate == null || interestRate < 0 || interestRate > 100 ? null : interestRate, billing_cycle: VALID_BILLING_CYCLES.has(row.billing_cycle) ? row.billing_cycle : 'monthly', autopay_enabled: toInt(row.autopay_enabled, 0) ? 1 : 0, autodraft_status: VALID_AUTODRAFT.has(row.autodraft_status) ? row.autodraft_status : 'none', website: cleanText(row.website, 500), username: cleanText(row.username, 255), account_info: cleanText(row.account_info, 500), has_2fa: toInt(row.has_2fa, 0) ? 1 : 0, active: row.active === undefined ? 1 : (toInt(row.active, 1) ? 1 : 0), notes: cleanText(row.notes, 2000), created_at: cleanText(row.created_at, 32), updated_at: cleanText(row.updated_at, 32), }; } function sanitizePayment(row, validBillIds) { const billId = toInt(row.bill_id); const amount = toNumber(row.amount); const paidDate = cleanDate(row.paid_date); if (!billId || !validBillIds.has(billId) || amount == null || amount < 0 || !paidDate) return null; return { old_id: toInt(row.id), bill_id: billId, amount, paid_date: paidDate, method: cleanText(row.method, 120), notes: cleanText(row.notes, 2000), created_at: cleanText(row.created_at, 32), updated_at: cleanText(row.updated_at, 32), }; } function sanitizeMonthlyState(row, validBillIds) { const billId = toInt(row.bill_id); const year = toInt(row.year); const month = toInt(row.month); if (!billId || !validBillIds.has(billId) || year < 2000 || year > 2100 || month < 1 || month > 12) return null; const actual = toNumber(row.actual_amount, null); return { old_id: toInt(row.id), bill_id: billId, year, month, actual_amount: actual == null || actual < 0 ? null : actual, notes: cleanText(row.notes, 2000), is_skipped: toInt(row.is_skipped, 0) ? 1 : 0, created_at: cleanText(row.created_at, 32), updated_at: cleanText(row.updated_at, 32), }; } function readExportData(src) { const names = tableNames(src); const missing = REQUIRED_TABLES.filter(t => !names.has(t)); if (missing.length) { throw importError(400, 'SQLite file is not a supported BillTracker user export.', 'USER_DB_IMPORT_UNSUPPORTED_SCHEMA', missing.map(table => ({ table, message: 'Required user export table is missing', }))); } const metadata = parseMetadata(src); const categories = selectKnown(src, 'categories', ['id', 'name', 'created_at', 'updated_at']) .map(sanitizeCategory).filter(Boolean); const bills = selectKnown(src, 'bills', [ 'id', 'name', 'category_id', 'due_day', 'override_due_date', 'bucket', 'expected_amount', 'interest_rate', 'billing_cycle', 'autopay_enabled', 'autodraft_status', 'website', 'username', 'account_info', 'has_2fa', 'active', 'notes', 'created_at', 'updated_at', ]).map(sanitizeBill).filter(Boolean); const validBillIds = new Set(bills.map(b => b.old_id).filter(Boolean)); const payments = selectKnown(src, 'payments', ['id', 'bill_id', 'amount', 'paid_date', 'method', 'notes', 'created_at', 'updated_at']) .map(row => sanitizePayment(row, validBillIds)).filter(Boolean); const monthlyState = selectKnown(src, 'monthly_bill_state', ['id', 'bill_id', 'year', 'month', 'actual_amount', 'notes', 'is_skipped', 'created_at', 'updated_at']) .map(row => sanitizeMonthlyState(row, validBillIds)).filter(Boolean); const notes = names.has('notes') ? selectKnown(src, 'notes', ['type', 'bill_id', 'payment_id', 'monthly_state_id', 'year', 'month', 'notes']) .map(n => ({ ...n, notes: cleanText(n.notes, 2000) })).filter(n => n.notes) : []; return { metadata, categories, bills, payments, monthly_bill_state: monthlyState, notes }; } function existingLookups(db, userId) { const categories = db.prepare('SELECT id, name FROM categories WHERE user_id = ?').all(userId); const bills = db.prepare('SELECT id, name, due_day FROM bills WHERE user_id = ?').all(userId); return { categoryByName: new Map(categories.map(c => [normalizeName(c.name), c])), billByKey: new Map(bills.map(b => [`${normalizeName(b.name)}|${b.due_day}`, b])), }; } function buildPreview(userId, data, originalFilename) { const db = getDb(); const lookups = existingLookups(db, userId); const categoryPlan = data.categories.map(c => { const existing = lookups.categoryByName.get(normalizeName(c.name)); return { old_id: c.old_id, name: c.name, action: existing ? 'skip' : 'create', existing_id: existing?.id ?? null, reason: existing ? 'Category already exists for this user' : 'Category is missing for this user', }; }); const categoryMap = new Map(categoryPlan.map(c => [c.old_id, c])); const billPlan = data.bills.map(b => { const existing = lookups.billByKey.get(`${normalizeName(b.name)}|${b.due_day}`); const category = categoryMap.get(b.category_id); return { old_id: b.old_id, name: b.name, due_day: b.due_day, category_name: category?.name ?? null, action: existing ? 'skip' : 'create', existing_id: existing?.id ?? null, reason: existing ? 'Bill with same name and due day already exists for this user' : 'Bill is missing for this user', }; }); const billPlanByOldId = new Map(billPlan.map(b => [b.old_id, b])); const paymentPlan = data.payments.map(p => ({ old_id: p.old_id, bill_old_id: p.bill_id, action: billPlanByOldId.has(p.bill_id) ? 'create_or_skip_duplicate' : 'conflict', reason: billPlanByOldId.has(p.bill_id) ? 'Will import if no duplicate payment exists' : 'Referenced bill is not present in export', })); const monthlyPlan = data.monthly_bill_state.map(m => ({ old_id: m.old_id, bill_old_id: m.bill_id, year: m.year, month: m.month, action: billPlanByOldId.has(m.bill_id) ? 'create_or_skip_duplicate' : 'conflict', reason: billPlanByOldId.has(m.bill_id) ? 'Will import if monthly state is missing' : 'Referenced bill is not present in export', })); const summary = { categories: { total: data.categories.length, create: categoryPlan.filter(x => x.action === 'create').length, skip: categoryPlan.filter(x => x.action === 'skip').length, conflict: 0, }, bills: { total: data.bills.length, create: billPlan.filter(x => x.action === 'create').length, skip: billPlan.filter(x => x.action === 'skip').length, conflict: 0, }, payments: { total: data.payments.length, create: data.payments.length, skip: 0, conflict: paymentPlan.filter(x => x.action === 'conflict').length, }, monthly_bill_state: { total: data.monthly_bill_state.length, create: data.monthly_bill_state.length, skip: 0, conflict: monthlyPlan.filter(x => x.action === 'conflict').length, }, notes: { total: data.notes.length, create: 0, skip: data.notes.length, conflict: 0, }, }; const warnings = [ 'Conflicts and duplicates are skipped by default.', 'This import does not restore users, passwords, sessions, admin settings, SMTP credentials, backups, or server paths.', ]; return { import_type: 'user_db', source_filename: originalFilename, metadata: data.metadata, counts: { bills: data.bills.length, categories: data.categories.length, payments: data.payments.length, monthly_bill_state: data.monthly_bill_state.length, notes: data.notes.length, }, summary, proposed_actions: { categories: categoryPlan.slice(0, 100), bills: billPlan.slice(0, 100), payments: paymentPlan.slice(0, 100), monthly_bill_state: monthlyPlan.slice(0, 100), }, warnings, }; } function saveSession(db, userId, sessionData) { const id = crypto.randomUUID(); const now = new Date(); const expires = new Date(now.getTime() + SESSION_TTL_HOURS * 60 * 60 * 1000); db.prepare(` INSERT INTO import_sessions (id, user_id, created_at, expires_at, preview_json) VALUES (?, ?, ?, ?, ?) `).run(id, userId, now.toISOString(), expires.toISOString(), JSON.stringify(sessionData)); return id; } function loadSession(db, userId, importSessionId) { const row = db.prepare(` SELECT preview_json FROM import_sessions WHERE id = ? AND user_id = ? AND expires_at > ? `).get(importSessionId, userId, new Date().toISOString()); if (!row) { throw importError(400, 'Import session is invalid or expired. Preview the SQLite export again.', 'USER_DB_IMPORT_SESSION_INVALID'); } const data = JSON.parse(row.preview_json); if (data.import_type !== 'user_db') { throw importError(400, 'Import session is not a SQLite user export preview.', 'USER_DB_IMPORT_SESSION_TYPE'); } return data; } async function previewUserDbImport(userId, buffer, options = {}) { assertSqliteBuffer(buffer); const originalFilename = sanitizeFilename(options.original_filename); return withTempSqlite(buffer, (src) => { const data = readExportData(src); const preview = buildPreview(userId, data, originalFilename); const db = getDb(); const importSessionId = saveSession(db, userId, { import_type: 'user_db', source_filename: originalFilename, metadata: data.metadata, data, preview, }); return { import_session_id: importSessionId, ...preview, }; }); } function ensureCategory(db, userId, category, summary, details) { const existing = db.prepare('SELECT id FROM categories WHERE user_id = ? AND LOWER(name) = LOWER(?)').get(userId, category.name); if (existing) { summary.rows_skipped++; summary.rows_conflicted++; details.categories.skipped++; return existing.id; } const result = db.prepare('INSERT INTO categories (user_id, name) VALUES (?, ?)').run(userId, category.name); summary.rows_created++; details.categories.created++; return result.lastInsertRowid; } function ensureBill(db, userId, bill, categoryId, summary, details) { const existing = db.prepare('SELECT id FROM bills WHERE user_id = ? AND LOWER(name) = LOWER(?) AND due_day = ?') .get(userId, bill.name, bill.due_day); if (existing) { summary.rows_skipped++; summary.rows_conflicted++; details.bills.skipped++; return existing.id; } const result = db.prepare(` INSERT INTO bills (user_id, name, category_id, due_day, override_due_date, bucket, expected_amount, interest_rate, billing_cycle, autopay_enabled, autodraft_status, website, username, account_info, has_2fa, active, notes) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `).run( userId, bill.name, categoryId || null, bill.due_day, bill.override_due_date || null, bill.bucket, bill.expected_amount, bill.interest_rate, bill.billing_cycle, bill.autopay_enabled, bill.autodraft_status, bill.website, bill.username, bill.account_info, bill.has_2fa, bill.active, bill.notes, ); summary.rows_created++; details.bills.created++; return result.lastInsertRowid; } function importPayment(db, targetBillId, payment, summary, details) { const duplicate = db.prepare(` SELECT p.id FROM payments p WHERE p.bill_id = ? AND p.paid_date = ? AND p.amount = ? AND COALESCE(p.method, '') = COALESCE(?, '') AND p.deleted_at IS NULL `).get(targetBillId, payment.paid_date, payment.amount, payment.method); if (duplicate) { summary.rows_skipped++; summary.rows_conflicted++; details.payments.skipped++; return; } db.prepare(` INSERT INTO payments (bill_id, amount, paid_date, method, notes) VALUES (?, ?, ?, ?, ?) `).run(targetBillId, payment.amount, payment.paid_date, payment.method, payment.notes); summary.rows_created++; details.payments.created++; } function importMonthlyState(db, targetBillId, row, summary, details) { const duplicate = db.prepare(` SELECT id FROM monthly_bill_state WHERE bill_id = ? AND year = ? AND month = ? `).get(targetBillId, row.year, row.month); if (duplicate) { summary.rows_skipped++; summary.rows_conflicted++; details.monthly_bill_state.skipped++; return; } db.prepare(` INSERT INTO monthly_bill_state (bill_id, year, month, actual_amount, notes, is_skipped) VALUES (?, ?, ?, ?, ?, ?) `).run(targetBillId, row.year, row.month, row.actual_amount, row.notes, row.is_skipped); summary.rows_created++; details.monthly_bill_state.created++; } async function applyUserDbImport(userId, importSessionId, options = {}) { if (options.overwrite) { throw importError(400, 'Overwrite is not supported for user SQLite imports. Existing data is skipped.', 'USER_DB_IMPORT_OVERWRITE_UNSUPPORTED'); } const db = getDb(); const session = loadSession(db, userId, importSessionId); const { data } = session; const summary = { import_type: 'user_db', rows_created: 0, rows_updated: 0, rows_skipped: 0, rows_conflicted: 0, rows_errored: 0, }; const details = { categories: { created: 0, skipped: 0, errored: 0 }, bills: { created: 0, skipped: 0, errored: 0 }, payments: { created: 0, skipped: 0, errored: 0 }, monthly_bill_state: { created: 0, skipped: 0, errored: 0 }, notes: { created: 0, skipped: data.notes?.length || 0, errored: 0 }, }; summary.rows_skipped += details.notes.skipped; const tx = db.transaction(() => { const categoryIdMap = new Map(); for (const category of data.categories) { categoryIdMap.set(category.old_id, ensureCategory(db, userId, category, summary, details)); } const billIdMap = new Map(); for (const bill of data.bills) { const targetCategoryId = bill.category_id ? categoryIdMap.get(bill.category_id) : null; billIdMap.set(bill.old_id, ensureBill(db, userId, bill, targetCategoryId, summary, details)); } for (const payment of data.payments) { const targetBillId = billIdMap.get(payment.bill_id); if (!targetBillId) { summary.rows_skipped++; summary.rows_conflicted++; details.payments.skipped++; continue; } importPayment(db, targetBillId, payment, summary, details); } for (const row of data.monthly_bill_state) { const targetBillId = billIdMap.get(row.bill_id); if (!targetBillId) { summary.rows_skipped++; summary.rows_conflicted++; details.monthly_bill_state.skipped++; continue; } importMonthlyState(db, targetBillId, row, summary, details); } db.prepare(` INSERT INTO import_history (user_id, imported_at, source_filename, file_type, sheet_name, rows_parsed, rows_created, rows_updated, rows_skipped, rows_ambiguous, rows_errored, options_json, summary_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `).run( userId, new Date().toISOString(), session.source_filename || null, 'sqlite', 'User SQLite export', data.categories.length + data.bills.length + data.payments.length + data.monthly_bill_state.length + (data.notes?.length || 0), summary.rows_created, summary.rows_updated, summary.rows_skipped, summary.rows_conflicted, summary.rows_errored, JSON.stringify({ overwrite: false }), JSON.stringify({ ...summary, details }), ); db.prepare('DELETE FROM import_sessions WHERE id = ? AND user_id = ?').run(importSessionId, userId); }); tx(); return { ...summary, details }; } module.exports = { previewUserDbImport, applyUserDbImport, };