1300 lines
49 KiB
JavaScript
1300 lines
49 KiB
JavaScript
const Database = require('better-sqlite3');
|
|
const path = require('path');
|
|
const fs = require('fs');
|
|
|
|
// Lazy import for auditService — cannot require at top level due to circular dependency
|
|
// (auditService -> database.js -> auditService). Use getLogAudit() instead of logAudit directly.
|
|
let _logAudit = null;
|
|
function getLogAudit() {
|
|
if (!_logAudit) {
|
|
try { _logAudit = require('../services/auditService').logAudit; } catch { _logAudit = () => {}; }
|
|
}
|
|
return _logAudit;
|
|
}
|
|
|
|
const DB_PATH = process.env.DB_PATH || path.join(__dirname, 'bills.db');
|
|
const SCHEMA_PATH = path.join(__dirname, 'schema.sql');
|
|
const DEFAULT_CATEGORIES = [
|
|
'Housing',
|
|
'Utilities',
|
|
'Credit Cards',
|
|
'Food',
|
|
'Loans',
|
|
'Insurance',
|
|
'Beauty',
|
|
'Entertainment',
|
|
'Subscriptions',
|
|
'Pets',
|
|
'Phone & Internet',
|
|
'Transportation',
|
|
'Medical',
|
|
'Other',
|
|
];
|
|
|
|
// ── SQL Whitelist Mappings ────────────────────────────────────────────────────
|
|
// Security FIX (2026-05-08): Whitelist all allowed column names to prevent SQL injection
|
|
// in migrations that use dynamic ALTER TABLE statements.
|
|
|
|
const COLUMN_WHITELIST = new Set([
|
|
// users table columns
|
|
'active', 'is_default_admin', 'notification_email', 'notifications_enabled',
|
|
'notify_3d', 'notify_1d', 'notify_due', 'notify_overdue', 'notify_amount_change',
|
|
'display_name', 'last_password_change_at', 'auth_provider', 'external_subject',
|
|
'email', 'last_login_at',
|
|
// payments table columns
|
|
'deleted_at', 'payment_source', 'transaction_id',
|
|
// monthly_starting_amounts table columns
|
|
'other_amount',
|
|
// bills table columns
|
|
'history_visibility', 'interest_rate', 'user_id',
|
|
'current_balance', 'minimum_payment', 'snowball_order', 'snowball_include',
|
|
'sort_order', 'snowball_exempt', 'is_subscription', 'subscription_type', 'reminder_days_before',
|
|
'subscription_source', 'subscription_detected_at', 'deleted_at', 'drift_snoozed_until',
|
|
// categories table columns
|
|
'sort_order',
|
|
// sessions table columns
|
|
'created_at',
|
|
// financial_accounts table columns
|
|
'monitored',
|
|
]);
|
|
|
|
// Security validation function for column names
|
|
function isValidColumnName(col) {
|
|
if (!col || typeof col !== 'string') return false;
|
|
// Must be in whitelist AND match valid SQL identifier pattern
|
|
return COLUMN_WHITELIST.has(col) && /^[a-z0-9_]+$/i.test(col);
|
|
}
|
|
|
|
// Security validation function for SQL definition fragments
|
|
function isValidSqlDefinition(def) {
|
|
if (!def || typeof def !== 'string') return false;
|
|
// Allow standard column definitions but reject any user input
|
|
// This is safe because all definitions are hardcoded here
|
|
return /^[\w\s\(\)\',!@#$%^&*+=\[\]<>\-.]+$/i.test(def);
|
|
}
|
|
|
|
// Subscription catalog seed rows live in their own module (pure data, no logic).
|
|
const { SUBSCRIPTION_CATALOG_ROWS, SUBSCRIPTION_CATALOG_V2_ROWS } = require('./subscriptionCatalogSeed');
|
|
// The versioned migration list lives in its own module (a factory injected with db + helpers).
|
|
const buildVersionedMigrations = require('./migrations/versionedMigrations');
|
|
// The legacy-reconcile migration list (same factory pattern, legacy DBs only).
|
|
const buildLegacyReconcileMigrations = require('./migrations/legacyReconcileMigrations');
|
|
|
|
function runSubscriptionCatalogV2Migration(database) {
|
|
// Category fixes for existing rows
|
|
database.prepare(`
|
|
UPDATE subscription_catalog SET subscription_type = 'software'
|
|
WHERE name = 'Discord Nitro' AND subscription_type = 'news'
|
|
`).run();
|
|
database.prepare(`
|
|
UPDATE subscription_catalog SET subscription_type = 'streaming'
|
|
WHERE name = 'Twitch Turbo' AND subscription_type = 'news'
|
|
`).run();
|
|
database.prepare(`
|
|
UPDATE subscription_catalog SET subscription_type = 'software'
|
|
WHERE name = 'X Premium' AND subscription_type = 'news'
|
|
`).run();
|
|
|
|
// New entries — skip any name already in the catalog
|
|
const existing = new Set(
|
|
database.prepare('SELECT name FROM subscription_catalog').all().map(r => r.name)
|
|
);
|
|
const toInsert = SUBSCRIPTION_CATALOG_V2_ROWS.filter(r => !existing.has(r[1]));
|
|
if (toInsert.length > 0) {
|
|
const insert = database.prepare(
|
|
'INSERT INTO subscription_catalog (rank, name, category, subscription_type, website, domain) VALUES (?,?,?,?,?,?)'
|
|
);
|
|
const insertMany = database.transaction((rows) => {
|
|
for (const row of rows) insert.run(...row);
|
|
});
|
|
insertMany(toInsert);
|
|
console.log(`[migration] subscription_catalog v2: added ${toInsert.length} new entries`);
|
|
}
|
|
}
|
|
|
|
function runAdvisoryFiltersMigration(database) {
|
|
database.exec(`
|
|
CREATE TABLE IF NOT EXISTS advisory_non_bill_filters (
|
|
id TEXT PRIMARY KEY,
|
|
pattern TEXT NOT NULL,
|
|
confidence TEXT NOT NULL CHECK(confidence IN ('high', 'medium')),
|
|
category TEXT NOT NULL,
|
|
rationale TEXT
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_advisory_filters_confidence
|
|
ON advisory_non_bill_filters(confidence);
|
|
|
|
CREATE TABLE IF NOT EXISTS advisory_bill_like_overrides (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
term TEXT NOT NULL UNIQUE
|
|
);
|
|
`);
|
|
|
|
const filterCount = database.prepare('SELECT COUNT(*) as n FROM advisory_non_bill_filters').get();
|
|
if (filterCount.n === 0) {
|
|
const jsonPath = path.join(__dirname, '..', 'docs', 'advisory_non_bill_transaction_filters_us_ms_5000.json');
|
|
const raw = fs.readFileSync(jsonPath, 'utf8');
|
|
const data = JSON.parse(raw);
|
|
|
|
const insertFilter = database.prepare(
|
|
'INSERT INTO advisory_non_bill_filters (id, pattern, confidence, category, rationale) VALUES (?,?,?,?,?)'
|
|
);
|
|
const insertFilters = database.transaction((rows) => {
|
|
for (const row of rows) {
|
|
insertFilter.run(row.id, row.pattern, row.confidence, row.category, row.rationale || null);
|
|
}
|
|
});
|
|
insertFilters(data.patterns || []);
|
|
console.log(`[migration] advisory_non_bill_filters: seeded ${(data.patterns || []).length} rows`);
|
|
|
|
const overrideTerms = data.bill_like_override_terms || [];
|
|
if (overrideTerms.length > 0) {
|
|
const insertOverride = database.prepare(
|
|
'INSERT OR IGNORE INTO advisory_bill_like_overrides (term) VALUES (?)'
|
|
);
|
|
const insertOverrides = database.transaction((terms) => {
|
|
for (const term of terms) insertOverride.run(term);
|
|
});
|
|
insertOverrides(overrideTerms);
|
|
console.log(`[migration] advisory_bill_like_overrides: seeded ${overrideTerms.length} rows`);
|
|
}
|
|
}
|
|
}
|
|
|
|
function runMerchantStoreMatchMigration(database) {
|
|
database.exec(`
|
|
CREATE TABLE IF NOT EXISTS merchant_store_matches (
|
|
id TEXT PRIMARY KEY,
|
|
entry_kind TEXT NOT NULL,
|
|
canonical_merchant_id TEXT NOT NULL,
|
|
canonical_name TEXT NOT NULL,
|
|
display_name TEXT NOT NULL,
|
|
category TEXT NOT NULL,
|
|
merchant_type TEXT,
|
|
scope TEXT NOT NULL,
|
|
priority INTEGER NOT NULL DEFAULT 0,
|
|
match_patterns TEXT NOT NULL,
|
|
negative_patterns TEXT,
|
|
locality_city TEXT,
|
|
locality_state TEXT
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_merchant_store_matches_canonical
|
|
ON merchant_store_matches(canonical_merchant_id);
|
|
`);
|
|
|
|
const count = database.prepare('SELECT COUNT(*) as n FROM merchant_store_matches').get();
|
|
if (count.n === 0) {
|
|
const jsonPath = path.join(__dirname, '..', 'docs', 'merchant_store_match_us_nems_online_5k_v0_2.json');
|
|
const raw = fs.readFileSync(jsonPath, 'utf8');
|
|
const data = JSON.parse(raw);
|
|
|
|
const insertEntry = database.prepare(`
|
|
INSERT INTO merchant_store_matches
|
|
(id, entry_kind, canonical_merchant_id, canonical_name, display_name, category,
|
|
merchant_type, scope, priority, match_patterns, negative_patterns,
|
|
locality_city, locality_state)
|
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)
|
|
`);
|
|
const insertEntries = database.transaction((rows) => {
|
|
for (const row of rows) {
|
|
insertEntry.run(
|
|
row.id,
|
|
row.entry_kind,
|
|
row.canonical_merchant_id,
|
|
row.canonical_name,
|
|
row.display_name,
|
|
row.category,
|
|
row.merchant_type || null,
|
|
row.scope,
|
|
row.priority || 0,
|
|
JSON.stringify(row.match_patterns || []),
|
|
JSON.stringify(row.negative_patterns || []),
|
|
row.locality?.city || null,
|
|
row.locality?.state || null,
|
|
);
|
|
}
|
|
});
|
|
insertEntries(data.merchant_store_entries || []);
|
|
console.log(`[migration] merchant_store_matches: seeded ${(data.merchant_store_entries || []).length} rows`);
|
|
}
|
|
}
|
|
|
|
function runSubscriptionCatalogMigration(database) {
|
|
database.exec(`
|
|
CREATE TABLE IF NOT EXISTS subscription_catalog (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
rank INTEGER NOT NULL,
|
|
name TEXT NOT NULL,
|
|
category TEXT NOT NULL,
|
|
subscription_type TEXT NOT NULL,
|
|
website TEXT,
|
|
domain TEXT,
|
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_subscription_catalog_rank ON subscription_catalog(rank);
|
|
CREATE INDEX IF NOT EXISTS idx_subscription_catalog_type ON subscription_catalog(subscription_type);
|
|
`);
|
|
|
|
const existing = database.prepare('SELECT COUNT(*) as n FROM subscription_catalog').get();
|
|
if (existing.n === 0) {
|
|
const insert = database.prepare(
|
|
'INSERT INTO subscription_catalog (rank, name, category, subscription_type, website, domain) VALUES (?,?,?,?,?,?)'
|
|
);
|
|
const insertMany = database.transaction((rows) => {
|
|
for (const row of rows) insert.run(...row);
|
|
});
|
|
insertMany(SUBSCRIPTION_CATALOG_ROWS);
|
|
console.log(`[migration] subscription_catalog: seeded ${SUBSCRIPTION_CATALOG_ROWS.length} rows`);
|
|
}
|
|
}
|
|
|
|
function seedManualDataSources(database = db) {
|
|
if (!database) return;
|
|
const hasDataSources = database.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='data_sources'").get();
|
|
const hasUsers = database.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='users'").get();
|
|
if (!hasDataSources || !hasUsers) return;
|
|
|
|
database.exec(`
|
|
INSERT INTO data_sources (user_id, type, provider, name, status)
|
|
SELECT u.id, 'manual', 'manual', 'Manual Entry', 'active'
|
|
FROM users u
|
|
WHERE NOT EXISTS (
|
|
SELECT 1
|
|
FROM data_sources ds
|
|
WHERE ds.user_id = u.id
|
|
AND ds.type = 'manual'
|
|
AND ds.provider = 'manual'
|
|
)
|
|
`);
|
|
}
|
|
|
|
function ensureTransactionFoundationSchema(database = db) {
|
|
database.exec(`
|
|
CREATE TABLE IF NOT EXISTS data_sources (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
|
type TEXT NOT NULL,
|
|
provider TEXT,
|
|
name TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'active',
|
|
config_json TEXT,
|
|
encrypted_secret TEXT,
|
|
last_sync_at TEXT,
|
|
last_error TEXT,
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS financial_accounts (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
|
data_source_id INTEGER,
|
|
provider_account_id TEXT,
|
|
name TEXT NOT NULL,
|
|
org_name TEXT,
|
|
account_type TEXT,
|
|
currency TEXT,
|
|
balance INTEGER,
|
|
available_balance INTEGER,
|
|
monitored INTEGER NOT NULL DEFAULT 1,
|
|
raw_data TEXT,
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
|
|
FOREIGN KEY (data_source_id) REFERENCES data_sources(id) ON DELETE SET NULL,
|
|
UNIQUE(data_source_id, provider_account_id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS transactions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
|
data_source_id INTEGER,
|
|
account_id INTEGER,
|
|
provider_transaction_id TEXT,
|
|
source_type TEXT NOT NULL,
|
|
transaction_type TEXT,
|
|
posted_date TEXT,
|
|
transacted_at TEXT,
|
|
amount INTEGER NOT NULL,
|
|
currency TEXT,
|
|
description TEXT,
|
|
payee TEXT,
|
|
memo TEXT,
|
|
category TEXT,
|
|
raw_data TEXT,
|
|
matched_bill_id INTEGER,
|
|
match_status TEXT NOT NULL DEFAULT 'unmatched',
|
|
ignored INTEGER NOT NULL DEFAULT 0,
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
|
|
FOREIGN KEY (data_source_id) REFERENCES data_sources(id) ON DELETE SET NULL,
|
|
FOREIGN KEY (account_id) REFERENCES financial_accounts(id) ON DELETE SET NULL,
|
|
FOREIGN KEY (matched_bill_id) REFERENCES bills(id) ON DELETE SET NULL
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_data_sources_user_type ON data_sources(user_id, type, status);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_data_sources_user_manual
|
|
ON data_sources(user_id, type, provider)
|
|
WHERE type = 'manual' AND provider = 'manual';
|
|
CREATE INDEX IF NOT EXISTS idx_financial_accounts_user_source ON financial_accounts(user_id, data_source_id);
|
|
CREATE INDEX IF NOT EXISTS idx_transactions_user_date ON transactions(user_id, posted_date, transacted_at);
|
|
CREATE INDEX IF NOT EXISTS idx_transactions_user_match ON transactions(user_id, match_status, ignored);
|
|
CREATE INDEX IF NOT EXISTS idx_transactions_account ON transactions(account_id);
|
|
CREATE INDEX IF NOT EXISTS idx_transactions_matched_bill ON transactions(matched_bill_id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_transactions_provider_dedupe
|
|
ON transactions (data_source_id, provider_transaction_id)
|
|
WHERE provider_transaction_id IS NOT NULL;
|
|
`);
|
|
|
|
seedManualDataSources(database);
|
|
}
|
|
|
|
fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
|
|
|
|
let db = null;
|
|
let initializing = false;
|
|
|
|
// Populated by runMigrations() so reconcileLegacyMigrations() can assert
|
|
// its own version list matches. Catches drift between the two arrays at startup
|
|
// on any legacy-DB upgrade path, rather than silently misconfiguring the schema.
|
|
let _runMigrationVersions = null;
|
|
|
|
function assertWritableDbPath() {
|
|
const dir = path.dirname(DB_PATH);
|
|
const probe = path.join(dir, `.write-test-${process.pid}-${Date.now()}`);
|
|
|
|
try {
|
|
fs.mkdirSync(dir, { recursive: true });
|
|
fs.writeFileSync(probe, 'ok');
|
|
fs.unlinkSync(probe);
|
|
|
|
if (fs.existsSync(DB_PATH)) {
|
|
fs.accessSync(DB_PATH, fs.constants.R_OK | fs.constants.W_OK);
|
|
}
|
|
} catch (err) {
|
|
const message = [
|
|
`Database path is not writable: ${DB_PATH}`,
|
|
`Ensure the DB directory is writable by the app user. In Docker, rebuild the image and recreate the container so the entrypoint can chown /data.`,
|
|
`Original error: ${err.message}`,
|
|
].join('\n');
|
|
const wrapped = new Error(message);
|
|
wrapped.code = err.code;
|
|
throw wrapped;
|
|
} finally {
|
|
try {
|
|
if (fs.existsSync(probe)) fs.unlinkSync(probe);
|
|
} catch {}
|
|
}
|
|
}
|
|
|
|
function getDb() {
|
|
// already ready
|
|
if (db) return db;
|
|
|
|
// Node is single-threaded and initialization below is fully synchronous, so
|
|
// the only way to observe `initializing === true` here is a re-entrant call
|
|
// from inside init itself (e.g. a migration requiring a module that calls
|
|
// getDb() at load time). Blocking/spinning can never resolve that — it would
|
|
// deadlock the process — so fail fast with a clear message instead.
|
|
if (initializing) {
|
|
throw new Error('getDb() called re-entrantly during database initialization');
|
|
}
|
|
|
|
initializing = true;
|
|
|
|
try {
|
|
console.log('Opening DB at:', path.basename(DB_PATH));
|
|
assertWritableDbPath();
|
|
|
|
db = new Database(DB_PATH, {
|
|
timeout: 5000
|
|
});
|
|
|
|
db.pragma('busy_timeout = 5000');
|
|
|
|
try {
|
|
db.pragma('journal_mode = WAL');
|
|
} catch (e) {
|
|
console.warn('WAL failed:', e.message);
|
|
}
|
|
|
|
db.pragma('foreign_keys = ON');
|
|
|
|
initSchema();
|
|
seedDefaults();
|
|
|
|
console.log('DB initialized successfully');
|
|
|
|
return db;
|
|
} catch (err) {
|
|
console.error('DB init failed:', err);
|
|
throw err;
|
|
} finally {
|
|
initializing = false;
|
|
}
|
|
}
|
|
|
|
function initSchema() {
|
|
const schema = fs.readFileSync(SCHEMA_PATH, 'utf8');
|
|
db.exec(schema);
|
|
|
|
// Create schema_migrations table for tracking applied migrations
|
|
db.exec(`
|
|
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
version TEXT NOT NULL UNIQUE,
|
|
description TEXT NOT NULL,
|
|
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
)
|
|
`);
|
|
|
|
// Check if this is a legacy database (tables exist but no migration tracking)
|
|
handleLegacyDatabase();
|
|
|
|
// After legacy reconciliation and user seeding, reset the default admin password
|
|
// when INIT_ADMIN_PASS is set. This ensures legacy DBs can be accessed after migration.
|
|
// The must_change_password flag forces the admin to pick a new password on first login.
|
|
if (process.env.INIT_ADMIN_PASS) {
|
|
const initUser = process.env.INIT_ADMIN_USER || 'admin';
|
|
const initPass = process.env.INIT_ADMIN_PASS;
|
|
const bcrypt = require('bcryptjs');
|
|
const newPasswordHash = bcrypt.hashSync(initPass, 12);
|
|
|
|
// Reset password for the default admin user if INIT_ADMIN_PASS is set
|
|
const result = db.prepare(`
|
|
UPDATE users SET password_hash = ?, first_login = 0, must_change_password = 0
|
|
WHERE username = ? AND is_default_admin = 1
|
|
`).run(newPasswordHash, initUser);
|
|
|
|
if (result.changes > 0) {
|
|
console.log('[init] Reset password and flags for default admin user');
|
|
}
|
|
}
|
|
|
|
runMigrations();
|
|
}
|
|
|
|
function hasMigrationBeenApplied(version) {
|
|
const stmt = db.prepare('SELECT 1 FROM schema_migrations WHERE version = ?');
|
|
return !!stmt.get(version);
|
|
}
|
|
|
|
function handleLegacyDatabase() {
|
|
// Check if schema_migrations table exists but is empty
|
|
// This indicates a legacy database that predates migration tracking
|
|
const migrationCount = db.prepare('SELECT COUNT(*) as count FROM schema_migrations').get().count;
|
|
|
|
if (migrationCount === 0) {
|
|
// This might be a legacy database. Check if core tables exist.
|
|
const tableCheck = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name IN ('users', 'bills', 'payments', 'categories', 'settings')").all();
|
|
|
|
// If we have core tables but no migrations tracked, this is likely a legacy DB
|
|
if (tableCheck.length >= 3) { // At least some core tables exist
|
|
console.log('[migration] Detected legacy database, reconciling schema migrations...');
|
|
|
|
// For each migration, check if its changes are already present and mark as applied if so
|
|
reconcileLegacyMigrations();
|
|
}
|
|
}
|
|
}
|
|
|
|
function reconcileLegacyMigrations() {
|
|
// Define all migrations with explicit version tracking
|
|
const migrations = buildLegacyReconcileMigrations({
|
|
db,
|
|
isValidColumnName,
|
|
isValidSqlDefinition,
|
|
ensureTransactionFoundationSchema,
|
|
runSubscriptionCatalogMigration,
|
|
runSubscriptionCatalogV2Migration,
|
|
runAdvisoryFiltersMigration,
|
|
runMerchantStoreMatchMigration,
|
|
});
|
|
|
|
// Check for legacy notification columns
|
|
const userCols = db.prepare('PRAGMA table_info(users)').all().map(c => c.name);
|
|
const newUserCols = [
|
|
'active', 'is_default_admin', 'notification_email', 'notifications_enabled',
|
|
'notify_3d', 'notify_1d', 'notify_due', 'notify_overdue'
|
|
];
|
|
const hasNotificationColumns = newUserCols.every(col => userCols.includes(col));
|
|
|
|
// If notification columns exist, mark that migration as applied
|
|
if (hasNotificationColumns) {
|
|
try {
|
|
recordMigration('legacy-notification-columns', 'users: notification columns');
|
|
console.log('[migration] Recorded legacy notification columns migration');
|
|
} catch (e) {
|
|
// Ignore if already recorded
|
|
}
|
|
}
|
|
|
|
// Assert version sync with runMigrations() to catch drift between the two arrays.
|
|
// runMigrations() always runs first and populates _runMigrationVersions.
|
|
if (_runMigrationVersions !== null) {
|
|
const reconcileVersions = migrations.map(m => m.version);
|
|
const runSet = new Set(_runMigrationVersions);
|
|
const reconcileSet = new Set(reconcileVersions);
|
|
const onlyInRun = _runMigrationVersions.filter(v => !reconcileSet.has(v));
|
|
const onlyInReconcile = reconcileVersions.filter(v => !runSet.has(v));
|
|
if (onlyInRun.length || onlyInReconcile.length) {
|
|
const msg =
|
|
'[migration-sync] reconcileLegacyMigrations and runMigrations version lists are out of sync.' +
|
|
(onlyInRun.length ? ` Only in runMigrations: ${onlyInRun.join(', ')}.` : '') +
|
|
(onlyInReconcile.length ? ` Only in reconcile: ${onlyInReconcile.join(', ')}.` : '') +
|
|
' Add the missing version to both arrays.';
|
|
console.error(msg);
|
|
throw new Error(msg);
|
|
}
|
|
}
|
|
|
|
// Process all versioned migrations
|
|
for (const migration of migrations) {
|
|
if (migration.check()) {
|
|
try {
|
|
recordMigration(migration.version, migration.description);
|
|
console.log(`[migration] Recorded legacy migration ${migration.version}: ${migration.description}`);
|
|
} catch (e) {
|
|
// Ignore if already recorded
|
|
}
|
|
} else {
|
|
// Migration changes are NOT present - run the migration to apply them
|
|
try {
|
|
console.log(`[migration] Running legacy migration ${migration.version}: ${migration.description}`);
|
|
// Wrap legacy migration in transaction
|
|
db.exec('BEGIN');
|
|
console.log(`[migration] Transaction BEGIN for legacy ${migration.version}`);
|
|
migration.run();
|
|
recordMigration(migration.version, migration.description);
|
|
db.exec('COMMIT');
|
|
console.log(`[migration] Transaction COMMIT for legacy ${migration.version}`);
|
|
} catch (err) {
|
|
db.exec('ROLLBACK');
|
|
console.error(`[migration-error] Failed to apply legacy migration ${migration.version}: ${err.message}. Rolled back.`);
|
|
throw err;
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log('[migration] Legacy database reconciliation complete');
|
|
}
|
|
|
|
function recordMigration(version, description) {
|
|
const stmt = db.prepare('INSERT INTO schema_migrations (version, description) VALUES (?, ?)');
|
|
stmt.run(version, description);
|
|
console.log(`[migration] Applied ${version}: ${description}`);
|
|
}
|
|
|
|
function validateMigrationDependencies(migration, appliedVersions) {
|
|
// Validate that all dependencies for a migration have been applied
|
|
const deps = migration.dependsOn || [];
|
|
const missing = deps.filter(dep => !appliedVersions.has(dep));
|
|
if (missing.length === 0) {
|
|
return { valid: true };
|
|
}
|
|
return { valid: false, missing };
|
|
}
|
|
|
|
function runMigrations() {
|
|
console.log('[migration] Starting database migrations');
|
|
const startTime = Date.now();
|
|
|
|
|
|
// Log start of migrations to audit log
|
|
try {
|
|
getLogAudit()({
|
|
action: 'migration.start',
|
|
entity_type: 'migration',
|
|
entity_id: null,
|
|
details: { message: 'Starting database migrations' }
|
|
});
|
|
} catch (auditErr) {
|
|
console.error(`[audit-error] Failed to log migration start to audit log: ${auditErr.message}`);
|
|
}
|
|
// Define all migrations with explicit version tracking and dependency chains
|
|
const migrations = buildVersionedMigrations({
|
|
db,
|
|
isValidColumnName,
|
|
isValidSqlDefinition,
|
|
ensureTransactionFoundationSchema,
|
|
runSubscriptionCatalogMigration,
|
|
runSubscriptionCatalogV2Migration,
|
|
runAdvisoryFiltersMigration,
|
|
runMerchantStoreMatchMigration,
|
|
});
|
|
|
|
// ── users: notification columns ───────────────────────────────────────────
|
|
// This migration needs to run first since it's not versioned in the schema
|
|
console.log('[migration] Applying unversioned user notification columns');
|
|
const unversionedStartTime = Date.now();
|
|
|
|
try {
|
|
db.exec('BEGIN');
|
|
console.log('[migration] Transaction BEGIN for unversioned user notification columns');
|
|
const userCols = db.prepare('PRAGMA table_info(users)').all().map(c => c.name);
|
|
const newUserCols = [
|
|
['active', 'INTEGER NOT NULL DEFAULT 1'],
|
|
['is_default_admin', 'INTEGER NOT NULL DEFAULT 0'],
|
|
['notification_email', 'TEXT'],
|
|
['notifications_enabled', 'INTEGER NOT NULL DEFAULT 0'],
|
|
['notify_3d', 'INTEGER NOT NULL DEFAULT 1'],
|
|
['notify_1d', 'INTEGER NOT NULL DEFAULT 1'],
|
|
['notify_due', 'INTEGER NOT NULL DEFAULT 1'],
|
|
['notify_overdue', 'INTEGER NOT NULL DEFAULT 1'],
|
|
];
|
|
for (const [col, def] of newUserCols) {
|
|
if (!userCols.includes(col)) {
|
|
// Security FIX (2026-05-08): Validate column name and definition to prevent SQL injection
|
|
if (!isValidColumnName(col) || !isValidSqlDefinition(def)) {
|
|
throw new Error(`Invalid migration: column '${col}' not in whitelist`);
|
|
}
|
|
db.exec(`ALTER TABLE users ADD COLUMN ${col} ${def}`);
|
|
}
|
|
}
|
|
const defaultAdminName = process.env.INIT_ADMIN_USER || 'admin';
|
|
db.prepare(`
|
|
UPDATE users
|
|
SET is_default_admin = 1
|
|
WHERE role = 'admin'
|
|
AND username = ?
|
|
AND NOT EXISTS (SELECT 1 FROM users WHERE is_default_admin = 1)
|
|
`).run(defaultAdminName);
|
|
db.exec(`
|
|
UPDATE users
|
|
SET is_default_admin = 1
|
|
WHERE id = (
|
|
SELECT id FROM users
|
|
WHERE role = 'admin'
|
|
ORDER BY id
|
|
LIMIT 1
|
|
)
|
|
AND NOT EXISTS (SELECT 1 FROM users WHERE is_default_admin = 1)
|
|
`);
|
|
db.exec('COMMIT');
|
|
console.log('[migration] Transaction COMMIT for unversioned user notification columns');
|
|
|
|
// Log successful completion with timing
|
|
const elapsed = Date.now() - unversionedStartTime;
|
|
console.log(`[migration] Unversioned user notification columns completed in ${elapsed}ms`);
|
|
} catch (err) {
|
|
db.exec('ROLLBACK');
|
|
const elapsed = Date.now() - unversionedStartTime;
|
|
console.error(`[migration-error] Failed to apply unversioned user notification columns after ${elapsed}ms: ${err.message}. Rolled back.`);
|
|
|
|
// Log migration failure to audit log (only safe after initSchema completes)
|
|
try {
|
|
getLogAudit()({
|
|
action: 'migration.failure',
|
|
entity_type: 'migration',
|
|
entity_id: null,
|
|
details: {
|
|
version: 'unversioned-user-notification-columns',
|
|
description: 'users: notification columns',
|
|
error: err.message,
|
|
elapsed_ms: elapsed
|
|
}
|
|
});
|
|
} catch (auditErr) {
|
|
console.error(`[audit-error] Failed to log migration failure to audit log: ${auditErr.message}`);
|
|
}
|
|
|
|
throw err;
|
|
}
|
|
|
|
// Store version list so reconcileLegacyMigrations() can assert sync.
|
|
_runMigrationVersions = migrations.map(m => m.version);
|
|
|
|
// Build set of already-applied versions for dependency checking
|
|
const appliedVersions = new Set(
|
|
db.prepare('SELECT version FROM schema_migrations').all().map(r => r.version)
|
|
);
|
|
|
|
// Process all versioned migrations
|
|
for (const migration of migrations) {
|
|
if (!hasMigrationBeenApplied(migration.version)) {
|
|
// Validate dependencies before applying
|
|
const depCheck = validateMigrationDependencies(migration, appliedVersions);
|
|
if (!depCheck.valid) {
|
|
console.error(`[migration-error] ${migration.version} depends on [${depCheck.missing.join(', ')}] which have not been applied. Skipping.`);
|
|
continue;
|
|
}
|
|
|
|
console.log(`[migration] Applying ${migration.version}: ${migration.description}`);
|
|
if (migration.dependsOn && migration.dependsOn.length > 0) {
|
|
console.log(`[migration] ${migration.version} depends on [${migration.dependsOn.join(', ')}] — satisfied`);
|
|
}
|
|
|
|
// Timing for migration execution
|
|
const migrationStartTime = Date.now();
|
|
|
|
try {
|
|
// Special handling for v0.40 migration which uses PRAGMA statements
|
|
if (migration.version === 'v0.40') {
|
|
// PRAGMA foreign_keys cannot run inside a transaction, so we
|
|
// disable FK checks before BEGIN and re-enable in a finally block
|
|
// to guarantee FK is always restored even on failure.
|
|
const billCols = db.prepare('PRAGMA table_info(bills)').all().map(c => c.name);
|
|
const categoryCols = db.prepare('PRAGMA table_info(categories)').all().map(c => c.name);
|
|
const needsForeignKeyOff = !billCols.includes('user_id') || !categoryCols.includes('user_id');
|
|
|
|
if (needsForeignKeyOff) {
|
|
db.exec('PRAGMA foreign_keys = OFF');
|
|
}
|
|
try {
|
|
db.exec('BEGIN');
|
|
console.log(`[migration] Transaction BEGIN for ${migration.version}`);
|
|
migration.run();
|
|
recordMigration(migration.version, migration.description);
|
|
db.exec('COMMIT');
|
|
console.log(`[migration] Transaction COMMIT for ${migration.version}`);
|
|
|
|
// Log successful completion with timing
|
|
const elapsed = Date.now() - migrationStartTime;
|
|
console.log(`[migration] ${migration.version} completed in ${elapsed}ms`);
|
|
appliedVersions.add(migration.version);
|
|
} catch (innerErr) {
|
|
db.exec('ROLLBACK');
|
|
const elapsed = Date.now() - migrationStartTime;
|
|
console.error(`[migration-error] ${migration.version} failed after ${elapsed}ms: ${innerErr.message}. Rolled back.`);
|
|
|
|
// Log migration failure to audit log (only safe after initSchema completes)
|
|
try {
|
|
getLogAudit()({
|
|
action: 'migration.failure',
|
|
entity_type: 'migration',
|
|
entity_id: null,
|
|
details: {
|
|
version: migration.version,
|
|
description: migration.description,
|
|
error: innerErr.message,
|
|
elapsed_ms: elapsed
|
|
}
|
|
});
|
|
} catch (auditErr) {
|
|
console.error(`[audit-error] Failed to log migration failure to audit log: ${auditErr.message}`);
|
|
}
|
|
|
|
throw innerErr;
|
|
} finally {
|
|
// Always restore FK checks — even on failure path
|
|
if (needsForeignKeyOff) {
|
|
db.exec('PRAGMA foreign_keys = ON');
|
|
}
|
|
}
|
|
} else {
|
|
// Standard transaction wrapping for other migrations
|
|
db.exec('BEGIN');
|
|
console.log(`[migration] Transaction BEGIN for ${migration.version}`);
|
|
migration.run();
|
|
recordMigration(migration.version, migration.description);
|
|
db.exec('COMMIT');
|
|
console.log(`[migration] Transaction COMMIT for ${migration.version}`);
|
|
|
|
// Log successful completion with timing
|
|
const elapsed = Date.now() - migrationStartTime;
|
|
console.log(`[migration] ${migration.version} completed in ${elapsed}ms`);
|
|
appliedVersions.add(migration.version);
|
|
}
|
|
} catch (err) {
|
|
db.exec('ROLLBACK');
|
|
const elapsed = Date.now() - migrationStartTime;
|
|
console.error(`[migration-error] Failed to apply ${migration.version} after ${elapsed}ms: ${err.message}. Rolled back.`);
|
|
|
|
// Log migration failure to audit log (only safe after initSchema completes)
|
|
try {
|
|
getLogAudit()({
|
|
action: 'migration.failure',
|
|
entity_type: 'migration',
|
|
entity_id: null,
|
|
details: {
|
|
version: migration.version,
|
|
description: migration.description,
|
|
error: err.message,
|
|
elapsed_ms: elapsed
|
|
}
|
|
});
|
|
} catch (auditErr) {
|
|
console.error(`[audit-error] Failed to log migration failure to audit log: ${auditErr.message}`);
|
|
}
|
|
|
|
throw err;
|
|
}
|
|
} else {
|
|
console.log(`[migration] Skipping already applied ${migration.version}: ${migration.description}`);
|
|
}
|
|
}
|
|
|
|
// Log total migration time
|
|
|
|
// Log completion of all migrations to audit log
|
|
try {
|
|
getLogAudit()({
|
|
action: 'migration.complete',
|
|
entity_type: 'migration',
|
|
entity_id: null,
|
|
details: {
|
|
total_time_ms: Date.now() - startTime,
|
|
message: 'All migrations completed successfully'
|
|
}
|
|
});
|
|
} catch (auditErr) {
|
|
console.error(`[audit-error] Failed to log migration completion to audit log: ${auditErr.message}`);
|
|
}
|
|
const totalTime = Date.now() - startTime;
|
|
console.log(`[migration] All migrations completed in ${totalTime}ms`);
|
|
|
|
// All migrations are now versioned
|
|
}
|
|
|
|
function seedDefaults() {
|
|
const defaults = [
|
|
['currency', 'USD'],
|
|
['date_format', 'MM/DD/YYYY'],
|
|
['grace_period_days', '5'],
|
|
['notify_days_before', '3'],
|
|
['backup_enabled', 'false'],
|
|
['backup_frequency_days', '1'],
|
|
['backup_keep_count', '14'],
|
|
['backup_path', process.env.BACKUP_PATH || path.join(__dirname, '..', 'backups')],
|
|
['backup_schedule_enabled', 'false'],
|
|
['backup_schedule_frequency', 'daily'],
|
|
['backup_schedule_time', '02:00'],
|
|
['backup_schedule_retention_count', '2'],
|
|
['backup_schedule_last_run_at', ''],
|
|
['backup_schedule_last_error', ''],
|
|
['auth_mode', 'multi'],
|
|
['default_user_id', ''],
|
|
['notify_smtp_enabled', 'false'],
|
|
['notify_sender_name', 'Bill Tracker'],
|
|
['notify_sender_address', ''],
|
|
['notify_smtp_host', ''],
|
|
['notify_smtp_port', '587'],
|
|
['notify_smtp_encryption', 'starttls'],
|
|
['notify_smtp_self_signed', 'false'],
|
|
['notify_smtp_username', ''],
|
|
['notify_smtp_password', ''],
|
|
['notify_allow_user_config', 'false'],
|
|
['notify_global_recipient', ''],
|
|
// Cleanup worker settings (v0.15)
|
|
['cleanup_import_sessions_enabled', 'true'],
|
|
['cleanup_temp_exports_enabled', 'true'],
|
|
['cleanup_temp_export_max_age_hours', '2'],
|
|
['cleanup_backup_partials_enabled', 'true'],
|
|
['cleanup_import_history_enabled', 'false'],
|
|
['cleanup_import_history_max_age_days', '365'],
|
|
['cleanup_last_run_at', ''],
|
|
['cleanup_last_result', ''],
|
|
// Auth method settings (v0.18)
|
|
['local_login_enabled', 'true'],
|
|
['oidc_login_enabled', 'false'],
|
|
['oidc_provider_name', 'authentik'],
|
|
['oidc_issuer_url', ''],
|
|
['oidc_client_id', ''],
|
|
['oidc_client_secret', ''],
|
|
['oidc_token_auth_method', 'client_secret_basic'],
|
|
['oidc_redirect_uri', ''],
|
|
['oidc_scopes', 'openid email profile groups'],
|
|
['oidc_auto_provision', 'true'],
|
|
['oidc_admin_group', ''],
|
|
['oidc_default_role', 'user'],
|
|
// Privacy settings (v0.94)
|
|
['geolocation_enabled', 'false'],
|
|
];
|
|
|
|
const insert = db.prepare(
|
|
'INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)'
|
|
);
|
|
|
|
for (const [key, value] of defaults) {
|
|
insert.run(key, value);
|
|
}
|
|
|
|
// Category defaults are user-scoped. They are applied by
|
|
// ensureUserDefaultCategories(userId) when user-owned category/bill data is read.
|
|
|
|
// ── Create initial admin user if none exists ─────────────────────────────
|
|
const userCount = db.prepare('SELECT COUNT(*) as cnt FROM users').get().cnt;
|
|
if (userCount === 0) {
|
|
const initUser = process.env.INIT_ADMIN_USER || 'admin';
|
|
const initPass = process.env.INIT_ADMIN_PASS || 'admin123';
|
|
// Use bcryptjs sync for database init (safe, runs once at startup)
|
|
const bcrypt = require('bcryptjs');
|
|
const password_hash = bcrypt.hashSync(initPass, 12);
|
|
db.prepare(`
|
|
INSERT INTO users (username, password_hash, role, is_default_admin, active, email, created_at, updated_at)
|
|
VALUES (?, ?, 'admin', 1, 1, ?, datetime('now'), datetime('now'))
|
|
`).run(initUser, password_hash, initUser + '@local');
|
|
console.log(`[seed] Created initial admin user: ${initUser}`);
|
|
}
|
|
|
|
seedManualDataSources(db);
|
|
}
|
|
|
|
function ensureUserDefaultCategories(userId) {
|
|
const db = getDb();
|
|
const insert = db.prepare('INSERT INTO categories (user_id, name) VALUES (?, ?)');
|
|
for (const name of DEFAULT_CATEGORIES) {
|
|
const existing = db.prepare('SELECT id FROM categories WHERE user_id = ? AND name = ? COLLATE NOCASE')
|
|
.get(userId, name);
|
|
if (!existing) insert.run(userId, name);
|
|
}
|
|
}
|
|
|
|
function getSetting(key) {
|
|
const db = getDb();
|
|
const row = db.prepare('SELECT value FROM settings WHERE key = ?').get(key);
|
|
return row ? row.value : null;
|
|
}
|
|
|
|
function setSetting(key, value) {
|
|
const db = getDb();
|
|
db.prepare(
|
|
"INSERT OR REPLACE INTO settings (key, value, updated_at) VALUES (?, ?, datetime('now'))"
|
|
).run(key, String(value));
|
|
}
|
|
|
|
function closeDb() {
|
|
if (!db) return;
|
|
db.close();
|
|
db = null;
|
|
}
|
|
|
|
function getDbPath() {
|
|
return DB_PATH;
|
|
}
|
|
|
|
// Rollback SQL definitions
|
|
const ROLLBACK_SQL_MAP = {
|
|
'v1.04': {
|
|
description: 'bill_templates.data JSON: money fields dollars -> integer cents',
|
|
sql: [
|
|
"UPDATE bill_templates SET data = json_set(data, '$.expected_amount', ROUND(json_extract(data, '$.expected_amount') / 100.0, 2)) WHERE json_extract(data, '$.expected_amount') IS NOT NULL",
|
|
"UPDATE bill_templates SET data = json_set(data, '$.current_balance', ROUND(json_extract(data, '$.current_balance') / 100.0, 2)) WHERE json_extract(data, '$.current_balance') IS NOT NULL",
|
|
"UPDATE bill_templates SET data = json_set(data, '$.minimum_payment', ROUND(json_extract(data, '$.minimum_payment') / 100.0, 2)) WHERE json_extract(data, '$.minimum_payment') IS NOT NULL",
|
|
]
|
|
},
|
|
'v1.03': {
|
|
description: 'money columns: dollars (REAL) -> integer cents',
|
|
sql: [
|
|
'UPDATE bills SET expected_amount = ROUND(expected_amount / 100.0, 2) WHERE expected_amount IS NOT NULL',
|
|
'UPDATE bills SET current_balance = ROUND(current_balance / 100.0, 2) WHERE current_balance IS NOT NULL',
|
|
'UPDATE bills SET minimum_payment = ROUND(minimum_payment / 100.0, 2) WHERE minimum_payment IS NOT NULL',
|
|
'UPDATE payments SET amount = ROUND(amount / 100.0, 2) WHERE amount IS NOT NULL',
|
|
'UPDATE payments SET balance_delta = ROUND(balance_delta / 100.0, 2) WHERE balance_delta IS NOT NULL',
|
|
'UPDATE payments SET interest_delta = ROUND(interest_delta / 100.0, 2) WHERE interest_delta IS NOT NULL',
|
|
'UPDATE monthly_bill_state SET actual_amount = ROUND(actual_amount / 100.0, 2) WHERE actual_amount IS NOT NULL',
|
|
'UPDATE monthly_starting_amounts SET first_amount = ROUND(first_amount / 100.0, 2) WHERE first_amount IS NOT NULL',
|
|
'UPDATE monthly_starting_amounts SET fifteenth_amount = ROUND(fifteenth_amount / 100.0, 2) WHERE fifteenth_amount IS NOT NULL',
|
|
'UPDATE monthly_starting_amounts SET other_amount = ROUND(other_amount / 100.0, 2) WHERE other_amount IS NOT NULL',
|
|
'UPDATE monthly_income SET amount = ROUND(amount / 100.0, 2) WHERE amount IS NOT NULL',
|
|
'UPDATE spending_budgets SET amount = ROUND(amount / 100.0, 2) WHERE amount IS NOT NULL',
|
|
'UPDATE snowball_plans SET extra_payment = ROUND(extra_payment / 100.0, 2) WHERE extra_payment IS NOT NULL',
|
|
'UPDATE users SET snowball_extra_payment = ROUND(snowball_extra_payment / 100.0, 2) WHERE snowball_extra_payment IS NOT NULL',
|
|
]
|
|
},
|
|
'v0.98': {
|
|
description: 'payments: bank override metadata for provisional manual payments',
|
|
sql: [
|
|
'DROP INDEX IF EXISTS idx_payments_overridden_by',
|
|
'DROP INDEX IF EXISTS idx_payments_accounting_active',
|
|
'ALTER TABLE payments DROP COLUMN IF EXISTS overridden_by_payment_id',
|
|
'ALTER TABLE payments DROP COLUMN IF EXISTS excluded_at',
|
|
'ALTER TABLE payments DROP COLUMN IF EXISTS exclusion_reason',
|
|
'ALTER TABLE payments DROP COLUMN IF EXISTS accounting_excluded',
|
|
]
|
|
},
|
|
'v0.97': {
|
|
description: 'subscription recommendation feedback: per-user learning signals',
|
|
sql: [
|
|
'DROP TABLE IF EXISTS subscription_recommendation_feedback',
|
|
]
|
|
},
|
|
'v0.96': {
|
|
description: 'bills: catalog_id FK; user_catalog_descriptors',
|
|
sql: [
|
|
'DROP TABLE IF EXISTS user_catalog_descriptors',
|
|
'ALTER TABLE bills DROP COLUMN IF EXISTS catalog_id',
|
|
]
|
|
},
|
|
'v0.95': {
|
|
description: 'subscription_catalog: bank descriptors + pricing',
|
|
sql: [
|
|
'DROP TABLE IF EXISTS subscription_catalog_descriptors',
|
|
]
|
|
},
|
|
'v0.94': {
|
|
description: 'security: session token hashing + geolocation opt-in setting',
|
|
sql: [
|
|
"DELETE FROM settings WHERE key = 'geolocation_enabled'",
|
|
// Sessions were cleared; rollback cannot restore them
|
|
]
|
|
},
|
|
'v0.93': {
|
|
description: 'bills: interest_accrued_month; payments: interest_delta; transactions: stable provider key',
|
|
sql: [
|
|
'ALTER TABLE bills DROP COLUMN IF EXISTS interest_accrued_month',
|
|
'ALTER TABLE payments DROP COLUMN IF EXISTS interest_delta',
|
|
// Restore the old (data_source_id, provider_transaction_id) dedupe index.
|
|
// The key format change and deleted duplicates cannot be reversed.
|
|
'DROP INDEX IF EXISTS idx_transactions_provider_dedupe',
|
|
`CREATE UNIQUE INDEX IF NOT EXISTS idx_transactions_provider_dedupe
|
|
ON transactions (data_source_id, provider_transaction_id)
|
|
WHERE provider_transaction_id IS NOT NULL`,
|
|
]
|
|
},
|
|
'v0.44': {
|
|
description: 'performance: add missing indexes for frequently queried columns',
|
|
sql: [
|
|
'DROP INDEX IF EXISTS idx_bills_user_name',
|
|
'DROP INDEX IF EXISTS idx_payments_method',
|
|
'DROP INDEX IF EXISTS idx_monthly_starting_amounts_user',
|
|
'DROP INDEX IF EXISTS idx_import_history_imported_at'
|
|
]
|
|
},
|
|
'v0.45': {
|
|
description: 'audit: add audit_log table for security event tracking',
|
|
sql: [
|
|
'DROP INDEX IF EXISTS idx_audit_log_user',
|
|
'DROP INDEX IF EXISTS idx_audit_log_action',
|
|
'DROP TABLE IF EXISTS audit_log'
|
|
]
|
|
},
|
|
'v0.46': {
|
|
description: 'billing: add cycle_type and cycle_day columns to bills',
|
|
sql: [
|
|
'ALTER TABLE bills DROP COLUMN cycle_day',
|
|
'ALTER TABLE bills DROP COLUMN cycle_type'
|
|
]
|
|
},
|
|
'v0.47': {
|
|
description: 'settings: reset backup_schedule_retention_count default from 14 to 2',
|
|
sql: [
|
|
"UPDATE settings SET value = '14' WHERE key = 'backup_schedule_retention_count' AND value = '2'"
|
|
]
|
|
},
|
|
'v0.48': {
|
|
description: 'bills: debt snowball fields',
|
|
sql: [
|
|
'ALTER TABLE bills DROP COLUMN snowball_include',
|
|
'ALTER TABLE bills DROP COLUMN snowball_order',
|
|
'ALTER TABLE bills DROP COLUMN minimum_payment',
|
|
'ALTER TABLE bills DROP COLUMN current_balance',
|
|
]
|
|
},
|
|
'v0.49': {
|
|
description: 'users: snowball extra payment field',
|
|
sql: ['ALTER TABLE users DROP COLUMN snowball_extra_payment']
|
|
},
|
|
'v0.50': {
|
|
description: 'payments: balance_delta column',
|
|
sql: ['ALTER TABLE payments DROP COLUMN balance_delta']
|
|
},
|
|
'v0.59': {
|
|
description: 'payments: source metadata columns',
|
|
sql: [
|
|
'ALTER TABLE payments DROP COLUMN transaction_id',
|
|
'ALTER TABLE payments DROP COLUMN payment_source',
|
|
]
|
|
},
|
|
'v0.60': {
|
|
description: 'transactions: shared transaction foundation tables',
|
|
sql: [
|
|
'DROP INDEX IF EXISTS idx_transactions_provider_dedupe',
|
|
'DROP INDEX IF EXISTS idx_transactions_matched_bill',
|
|
'DROP INDEX IF EXISTS idx_transactions_account',
|
|
'DROP INDEX IF EXISTS idx_transactions_user_match',
|
|
'DROP INDEX IF EXISTS idx_transactions_user_date',
|
|
'DROP INDEX IF EXISTS idx_financial_accounts_user_source',
|
|
'DROP INDEX IF EXISTS idx_data_sources_user_manual',
|
|
'DROP INDEX IF EXISTS idx_data_sources_user_type',
|
|
'DROP TABLE IF EXISTS transactions',
|
|
'DROP TABLE IF EXISTS financial_accounts',
|
|
'DROP TABLE IF EXISTS data_sources',
|
|
]
|
|
},
|
|
'v0.61': {
|
|
description: 'payments: one active payment per linked transaction',
|
|
sql: ['DROP INDEX IF EXISTS idx_payments_transaction_active']
|
|
},
|
|
'v0.62': {
|
|
description: 'matches: rejected transaction match suggestions',
|
|
sql: [
|
|
'DROP INDEX IF EXISTS idx_match_suggestion_rejections_user',
|
|
'DROP TABLE IF EXISTS match_suggestion_rejections',
|
|
]
|
|
},
|
|
'v0.63': {
|
|
description: 'bills: subscription metadata fields',
|
|
sql: [
|
|
'DROP INDEX IF EXISTS idx_bills_user_subscription',
|
|
'ALTER TABLE bills DROP COLUMN subscription_detected_at',
|
|
'ALTER TABLE bills DROP COLUMN subscription_source',
|
|
'ALTER TABLE bills DROP COLUMN reminder_days_before',
|
|
'ALTER TABLE bills DROP COLUMN subscription_type',
|
|
'ALTER TABLE bills DROP COLUMN is_subscription',
|
|
]
|
|
},
|
|
'v0.72': {
|
|
description: 'bills: persistent tracker sort order',
|
|
sql: [
|
|
'DROP INDEX IF EXISTS idx_bills_user_sort',
|
|
'ALTER TABLE bills DROP COLUMN sort_order',
|
|
]
|
|
},
|
|
'v0.75': {
|
|
description: 'categories: persistent sort order',
|
|
sql: [
|
|
'DROP INDEX IF EXISTS idx_categories_user_sort',
|
|
'ALTER TABLE categories DROP COLUMN sort_order',
|
|
]
|
|
},
|
|
'v0.76': {
|
|
description: 'bills: canonical billing schedule cleanup',
|
|
sql: [
|
|
`UPDATE bills
|
|
SET billing_cycle = CASE
|
|
WHEN cycle_type = 'quarterly' THEN 'quarterly'
|
|
WHEN cycle_type = 'annual' THEN 'annually'
|
|
WHEN cycle_type IN ('weekly', 'biweekly') THEN 'irregular'
|
|
ELSE 'monthly'
|
|
END`,
|
|
]
|
|
},
|
|
'v0.51': {
|
|
description: 'bills: snowball_exempt column',
|
|
sql: ['ALTER TABLE bills DROP COLUMN snowball_exempt']
|
|
},
|
|
'v0.52': {
|
|
description: 'users: last_seen_version column',
|
|
sql: ['ALTER TABLE users DROP COLUMN last_seen_version']
|
|
},
|
|
'v0.53': {
|
|
description: 'user_login_history table',
|
|
sql: ['DROP TABLE IF EXISTS user_login_history']
|
|
},
|
|
'v0.54': {
|
|
description: 'user_settings table',
|
|
sql: ['DROP TABLE IF EXISTS user_settings']
|
|
},
|
|
'v0.55': {
|
|
description: 'user_login_history device metadata columns',
|
|
sql: [
|
|
'ALTER TABLE user_login_history DROP COLUMN device_fingerprint',
|
|
'ALTER TABLE user_login_history DROP COLUMN device_type',
|
|
'ALTER TABLE user_login_history DROP COLUMN os',
|
|
'ALTER TABLE user_login_history DROP COLUMN browser',
|
|
]
|
|
},
|
|
'v0.56': {
|
|
description: 'bills/categories soft-delete columns',
|
|
sql: [
|
|
'DROP INDEX IF EXISTS idx_categories_deleted',
|
|
'DROP INDEX IF EXISTS idx_bills_deleted',
|
|
'ALTER TABLE categories DROP COLUMN deleted_at',
|
|
'ALTER TABLE bills DROP COLUMN deleted_at',
|
|
]
|
|
},
|
|
'v0.57': {
|
|
description: 'autopay suggestions and auto-mark paid',
|
|
sql: [
|
|
'DROP INDEX IF EXISTS idx_autopay_suggestion_dismissals_user_month',
|
|
'DROP TABLE IF EXISTS autopay_suggestion_dismissals',
|
|
'ALTER TABLE bills DROP COLUMN auto_mark_paid',
|
|
]
|
|
},
|
|
'v0.58': {
|
|
description: 'saved bill templates',
|
|
sql: [
|
|
'DROP INDEX IF EXISTS idx_bill_templates_user_name',
|
|
'DROP TABLE IF EXISTS bill_templates',
|
|
]
|
|
}
|
|
};
|
|
|
|
function rollbackMigration(version) {
|
|
if (!db) throw new Error('Database not initialized');
|
|
|
|
// Check the migration was actually applied
|
|
const applied = db.prepare('SELECT 1 FROM schema_migrations WHERE version = ?').get(version);
|
|
if (!applied) {
|
|
const err = new Error(`Migration ${version} has not been applied — cannot rollback`);
|
|
err.code = 'NOT_APPLIED';
|
|
throw err;
|
|
}
|
|
|
|
const rollback = ROLLBACK_SQL_MAP[version];
|
|
if (!rollback) {
|
|
const err = new Error(`Migration ${version} does not support rollback`);
|
|
err.code = 'ROLLBACK_NOT_SUPPORTED';
|
|
throw err;
|
|
}
|
|
|
|
console.log(`[rollback] Rolling back ${version}: ${rollback.description}`);
|
|
const startTime = Date.now();
|
|
|
|
try {
|
|
db.exec('BEGIN');
|
|
console.log(`[rollback] Transaction BEGIN for ${version}`);
|
|
|
|
for (const stmt of rollback.sql) {
|
|
console.log(`[rollback] Executing: ${stmt}`);
|
|
db.exec(stmt);
|
|
}
|
|
|
|
// Remove migration record
|
|
db.prepare('DELETE FROM schema_migrations WHERE version = ?').run(version);
|
|
console.log(`[rollback] Removed ${version} from schema_migrations`);
|
|
|
|
db.exec('COMMIT');
|
|
console.log(`[rollback] Transaction COMMIT for ${version}`);
|
|
|
|
const elapsed = Date.now() - startTime;
|
|
console.log(`[rollback] ${version} rolled back in ${elapsed}ms`);
|
|
|
|
// Audit log
|
|
try {
|
|
getLogAudit()({
|
|
action: 'migration.rollback',
|
|
entity_type: 'migration',
|
|
entity_id: null,
|
|
details: { version, description: rollback.description, elapsed_ms: elapsed }
|
|
});
|
|
} catch (auditErr) {
|
|
console.error(`[audit-error] Failed to log rollback to audit log: ${auditErr.message}`);
|
|
}
|
|
|
|
return { success: true, version, description: rollback.description, elapsed_ms: elapsed };
|
|
} catch (err) {
|
|
db.exec('ROLLBACK');
|
|
const elapsed = Date.now() - startTime;
|
|
console.error(`[rollback-error] ${version} failed after ${elapsed}ms: ${err.message}`);
|
|
|
|
// Audit log
|
|
try {
|
|
getLogAudit()({
|
|
action: 'migration.rollback.failure',
|
|
entity_type: 'migration',
|
|
entity_id: null,
|
|
details: { version, description: rollback.description, error: err.message, elapsed_ms: elapsed }
|
|
});
|
|
} catch (auditErr) {
|
|
console.error(`[audit-error] Failed to log rollback failure to audit log: ${auditErr.message}`);
|
|
}
|
|
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Cleanup expired sessions from the database
|
|
* @returns {Object} Result object with changes count
|
|
*/
|
|
function cleanupExpiredSessions() {
|
|
const result = db.prepare("DELETE FROM sessions WHERE expires_at < datetime('now')").run();
|
|
console.log(`[cleanup] Purged ${result.changes} expired sessions`);
|
|
return result;
|
|
}
|
|
|
|
module.exports = { getDb, getSetting, setSetting, closeDb, getDbPath, ensureUserDefaultCategories, cleanupExpiredSessions, rollbackMigration };
|