'use strict'; const { billingCycleForCycleType, insertBill, validateBillData } = require('./billsService'); const SUBSCRIPTION_TYPES = [ 'streaming', 'software', 'cloud', 'music', 'news', 'fitness', 'gaming', 'utilities', 'insurance', 'food', 'education', 'shopping', 'security', 'other', ]; const MONTHLY_FACTORS = { weekly: 52 / 12, biweekly: 26 / 12, monthly: 1, quarterly: 1 / 3, annual: 1 / 12, annually: 1 / 12, irregular: 1, }; // Transactions that are clearly not subscriptions — skip before grouping const SKIP_MERCHANT_RE = /\b(atm|withdrawal|transfer|deposit|zelle|venmo|wire|refund|rebate|interest charge)\b/; // Fallback keyword list used when catalog lookup finds no match const TYPE_KEYWORDS = [ ['streaming', ['netflix', 'hulu', 'disney', 'max', 'paramount', 'peacock', 'youtube tv', 'sling', 'espn', 'fubo', 'starz', 'crunchyroll', 'dazn']], ['music', ['spotify', 'apple music', 'tidal', 'pandora', 'siriusxm', 'soundcloud', 'deezer', 'iheart']], ['software', ['adobe', 'microsoft', 'github', 'notion', 'figma', 'canva', 'openai', 'chatgpt', 'grammarly', 'zoom', 'slack', 'cursor', 'ynab']], ['cloud', ['dropbox', 'icloud', 'google one', 'google storage', 'backblaze', 'box storage']], ['news', ['nyt', 'new york times', 'economist', 'athletic', 'washington post', 'wsj', 'bloomberg', 'substack', 'patreon', 'medium']], ['fitness', ['peloton', 'planet fitness', 'gym', 'fitbit', 'strava', 'headspace', 'calm', 'noom', 'classpass', 'whoop']], ['gaming', ['xbox', 'playstation', 'steam', 'nintendo', 'roblox', 'discord nitro', 'ea play', 'ubisoft']], ['utilities', ['verizon', 'at t', 'att', 'comcast', 'xfinity', 'spectrum', 'tmobile', 't mobile']], ['insurance', ['insurance', 'geico', 'progressive', 'state farm', 'allstate']], ['food', ['hellofresh', 'blue apron', 'doordash', 'instacart', 'uber eats', 'grubhub', 'factor', 'hungryroot']], ['education', ['duolingo', 'masterclass', 'coursera', 'skillshare', 'audible', 'kindle unlimited', 'blinkist']], ['shopping', ['amazon prime', 'walmart plus', 'costco', 'target circle', 'chewy']], ['security', ['nordvpn', 'expressvpn', '1password', 'dashlane', 'norton', 'mcafee', 'surfshark']], ]; // ── Catalog ─────────────────────────────────────────────────────────────────── function loadCatalog(db) { try { return db.prepare('SELECT id, rank, name, category, subscription_type, domain, website FROM subscription_catalog ORDER BY rank ASC').all(); } catch { return []; } } // Build a normalized-name → subscription_type map from the full catalog so // inferType can use all 290 known services, not just the hardcoded keyword list. function buildCatalogTypeMap(catalog) { const map = new Map(); for (const entry of catalog) { if (!entry.subscription_type || entry.subscription_type === 'other') continue; const key = normalizeCatalogName(entry.name); if (key.length >= 3 && !map.has(key)) map.set(key, entry.subscription_type); } return map; } function compactCatalogKey(value) { return normalizeCatalogName(value).replace(/\s+/g, ''); } function hostFromUrl(value) { if (!value) return ''; try { return new URL(String(value).startsWith('http') ? String(value) : `https://${value}`).hostname; } catch { return String(value || ''); } } function catalogDomainKeys(entry) { const keys = new Set(); const candidates = [entry.domain, hostFromUrl(entry.website)].filter(Boolean); for (const candidate of candidates) { const host = String(candidate).toLowerCase().replace(/^www\./, '').replace(/\/.*$/, ''); const labels = host.split('.').filter(Boolean); if (labels.length >= 2) { keys.add(labels.join(' ')); keys.add(labels.slice(-2).join(' ')); if (labels[0].length >= 5) keys.add(labels[0]); } } return [...keys].filter(key => key.length >= 4); } function normalizeCatalogName(value) { return String(value || '') .toLowerCase() .replace(/\+/g, ' plus ') // "Walmart+" → "walmart plus" so it only matches "walmart plus" transactions .replace(/[^a-z0-9]+/g, ' ') .replace(/\s+/g, ' ') .trim(); } // Given a normalized merchant string, find the best matching catalog entry. // Matches on service name (normalized) or domain (dot replaced with space). function lookupCatalog(catalog, merchantText) { if (!catalog.length || !merchantText) return null; let best = null; let bestScore = 0; const merchantCompact = compactCatalogKey(merchantText); for (const entry of catalog) { const nameKey = normalizeCatalogName(entry.name); const nameCompact = compactCatalogKey(entry.name); const nameScore = 1000 + nameKey.length; if ( nameKey.length >= 3 && (merchantText.includes(nameKey) || (nameCompact.length >= 5 && merchantCompact.includes(nameCompact))) && nameScore > bestScore ) { best = entry; bestScore = nameScore; } for (const domainKey of catalogDomainKeys(entry)) { const domainCompact = domainKey.replace(/\s+/g, ''); const domainScore = 500 + domainKey.length; if ( (merchantText.includes(domainKey) || (domainCompact.length >= 5 && merchantCompact.includes(domainCompact))) && domainScore > bestScore ) { best = entry; bestScore = domainScore; } } } return best; } // ── Helpers ─────────────────────────────────────────────────────────────────── function normalizeMerchant(value) { return String(value || '') .toLowerCase() .replace(/\+/g, ' plus ') // preserve "+" so "WALMART+" matches catalog "Walmart+" → "walmart plus" .replace(/&/g, '') // "&" joins words — "AT&T" → "att", not "at t" .replace(/['’]/g, '') // collapse apostrophes — "Sam's Club" → "sams club", "McDonald's" → "mcdonalds" .replace(/[^a-z0-9\s]/g, ' ') .replace(/\b(pos|debit|card|payment|purchase|recurring|online|inc|llc|co|www)\b/g, ' ') .replace(/\s+/g, ' ') .trim(); } function titleCase(value) { return String(value || 'Subscription') .split(/\s+/) .filter(Boolean) .map(word => word.charAt(0).toUpperCase() + word.slice(1)) .join(' '); } function inferType(merchantText, catalogEntry, catalogTypeMap = null) { if (catalogEntry?.subscription_type) return catalogEntry.subscription_type; const haystack = normalizeMerchant(merchantText); if (catalogTypeMap) { for (const [nameKey, type] of catalogTypeMap.entries()) { if (haystack.includes(nameKey)) return type; } } for (const [type, words] of TYPE_KEYWORDS) { if (words.some(word => haystack.includes(word))) return type; } return 'other'; } function catalogMatchPayload(catalogEntry) { return catalogEntry ? { id: catalogEntry.id, name: catalogEntry.name, category: catalogEntry.category, subscription_type: catalogEntry.subscription_type || 'other', website: catalogEntry.website || null, } : null; } function monthlyEquivalent(amount, cycleType, billingCycle) { const key = String(cycleType || billingCycle || 'monthly').toLowerCase(); const fallback = String(billingCycle || '').toLowerCase() === 'quarterly' ? 'quarterly' : String(billingCycle || '').toLowerCase() === 'annually' ? 'annual' : key; const factor = MONTHLY_FACTORS[key] ?? MONTHLY_FACTORS[fallback] ?? 1; return Math.round(Number(amount || 0) * factor * 100) / 100; } function nextDueDate(bill, now = new Date()) { const dueDay = Math.min(Math.max(Number(bill.due_day) || 1, 1), 31); const cycle = String(bill.cycle_type || bill.billing_cycle || 'monthly').toLowerCase(); let date = new Date(now.getFullYear(), now.getMonth(), dueDay); if (date < new Date(now.getFullYear(), now.getMonth(), now.getDate())) { date = new Date(now.getFullYear(), now.getMonth() + 1, dueDay); } if (cycle === 'quarterly' || cycle === 'annual') { const startMonth = Math.min(Math.max(Number(bill.cycle_day) || 1, 1), 12) - 1; const step = cycle === 'quarterly' ? 3 : 12; date = new Date(now.getFullYear(), startMonth, dueDay); while (date < new Date(now.getFullYear(), now.getMonth(), now.getDate())) { date = new Date(date.getFullYear(), date.getMonth() + step, dueDay); } } return date.toISOString().slice(0, 10); } function decorateSubscription(bill) { const monthly = monthlyEquivalent(bill.expected_amount, bill.cycle_type, bill.billing_cycle); return { ...bill, is_subscription: !!bill.is_subscription, active: !!bill.active, monthly_equivalent: monthly, yearly_equivalent: Math.round(monthly * 12 * 100) / 100, next_due_date: nextDueDate(bill), subscription_type: bill.subscription_type || inferType(`${bill.name} ${bill.category_name || ''}`, null), }; } function getSubscriptions(db, userId) { return db.prepare(` SELECT b.*, c.name AS category_name, CASE WHEN EXISTS( SELECT 1 FROM bill_merchant_rules WHERE bill_id = b.id AND user_id = b.user_id ) THEN 1 ELSE 0 END AS has_merchant_rule FROM bills b LEFT JOIN categories c ON c.id = b.category_id AND c.user_id = b.user_id AND c.deleted_at IS NULL WHERE b.user_id = ? AND b.deleted_at IS NULL AND b.is_subscription = 1 ORDER BY b.active DESC, CASE WHEN b.sort_order IS NULL THEN 1 ELSE 0 END, b.sort_order ASC, b.due_day ASC, b.name COLLATE NOCASE ASC `).all(userId).map(decorateSubscription); } function getSubscriptionSummary(subscriptions) { const active = subscriptions.filter(item => item.active); const monthlyTotal = active.reduce((sum, item) => sum + Number(item.monthly_equivalent || 0), 0); const typeTotals = new Map(); for (const item of active) { const type = item.subscription_type || 'other'; typeTotals.set(type, (typeTotals.get(type) || 0) + Number(item.monthly_equivalent || 0)); } const topType = [...typeTotals.entries()].sort((a, b) => b[1] - a[1])[0] || null; return { active_count: active.length, paused_count: subscriptions.length - active.length, monthly_total: Math.round(monthlyTotal * 100) / 100, yearly_total: Math.round(monthlyTotal * 12 * 100) / 100, top_type: topType ? { type: topType[0], monthly_total: Math.round(topType[1] * 100) / 100 } : null, }; } function existingBillNames(db, userId) { return db.prepare('SELECT name FROM bills WHERE user_id = ? AND deleted_at IS NULL') .all(userId) .map(row => normalizeMerchant(row.name)) .filter(Boolean); } function dollarsFromTransactionAmount(amount) { return Math.round((Math.abs(Number(amount || 0)) / 100) * 100) / 100; } // ── Decline store ───────────────────────────────────────────────────────────── function getDeclinedKeys(db, userId) { try { const rows = db.prepare('SELECT decline_key FROM declined_subscription_hints WHERE user_id = ?').all(userId); return new Set(rows.map(r => r.decline_key)); } catch { return new Set(); } } function declineRecommendation(db, userId, declineKey) { db.prepare(` INSERT INTO declined_subscription_hints (user_id, decline_key) VALUES (?, ?) ON CONFLICT(user_id, decline_key) DO NOTHING `).run(userId, declineKey); } // ── Recommendations ─────────────────────────────────────────────────────────── function getSubscriptionRecommendations(db, userId) { const catalog = loadCatalog(db); const catalogTypeMap = buildCatalogTypeMap(catalog); const existingNames = existingBillNames(db, userId); const declined = getDeclinedKeys(db, userId); const rows = db.prepare(` SELECT t.id, t.amount, t.currency, t.description, t.payee, t.memo, t.category, COALESCE(t.posted_date, substr(t.transacted_at, 1, 10)) AS tx_date, ds.provider AS data_source_provider, ds.name AS data_source_name, fa.name AS account_name, fa.org_name AS account_org_name FROM transactions t LEFT JOIN data_sources ds ON ds.id = t.data_source_id AND ds.user_id = t.user_id LEFT JOIN financial_accounts fa ON fa.id = t.account_id AND fa.user_id = t.user_id WHERE t.user_id = ? AND t.ignored = 0 AND t.match_status = 'unmatched' AND t.amount < 0 AND (t.account_id IS NULL OR fa.id IS NULL OR fa.monitored = 1) AND COALESCE(t.posted_date, substr(t.transacted_at, 1, 10)) >= date('now', '-420 days') ORDER BY tx_date ASC `).all(userId); // Group by merchant + amount bucket — consistent amounts are the foundation of // subscription detection. Catalog lookup names the service and boosts confidence // but does not change the grouping; deduplication at the end ensures one entry // per known service. const groups = new Map(); for (const tx of rows) { const merchant = normalizeMerchant(tx.payee || tx.description || tx.memo); if (!merchant || merchant.length < 3) continue; if (SKIP_MERCHANT_RE.test(merchant)) continue; const amount = dollarsFromTransactionAmount(tx.amount); if (amount < 1) continue; const key = `${merchant}:${Math.round(amount)}`; if (!groups.has(key)) { groups.set(key, { merchant, items: [], catalogEntry: null }); } const group = groups.get(key); group.items.push({ ...tx, amount_dollars: amount }); if (!group.catalogEntry) group.catalogEntry = lookupCatalog(catalog, merchant); } const recommendations = []; for (const group of groups.values()) { const { merchant, catalogEntry } = group; const declineKey = catalogEntry ? `catalog:${catalogEntry.id}` : `merchant:${merchant}`; if (declined.has(declineKey)) continue; if (existingNames.some(n => n.includes(merchant) || merchant.includes(n))) continue; const sorted = group.items .filter(item => item.tx_date) .sort((a, b) => String(a.tx_date).localeCompare(String(b.tx_date))); if (sorted.length === 0) continue; const averageAmount = sorted.reduce((sum, item) => sum + item.amount_dollars, 0) / sorted.length; const maxDelta = sorted.length > 1 ? Math.max(...sorted.map(item => Math.abs(item.amount_dollars - averageAmount))) : 0; const last = sorted[sorted.length - 1]; // Tier 1: catalog match with 1 occurrence if (catalogEntry && sorted.length === 1) { recommendations.push(buildRecommendation({ merchant, catalogEntry, sorted, averageAmount, maxDelta, last, cycleType: 'monthly', avgGap: 30, confidence: 90, tier: 'known_service', declineKey, catalogTypeMap, })); continue; } if (sorted.length < 2) continue; const gaps = []; for (let i = 1; i < sorted.length; i++) { gaps.push(Math.round( (new Date(`${sorted[i].tx_date}T00:00:00`) - new Date(`${sorted[i - 1].tx_date}T00:00:00`)) / 86400000 )); } const avgGap = gaps.reduce((sum, g) => sum + g, 0) / gaps.length; const cycleType = avgGap >= 320 ? 'annual' : avgGap >= 75 ? 'quarterly' : avgGap >= 10 && avgGap <= 18 ? 'biweekly' : avgGap <= 9 ? 'weekly' : 'monthly'; if (cycleType === 'monthly' && (avgGap < 24 || avgGap > 38)) continue; if (cycleType === 'quarterly' && (avgGap < 75 || avgGap > 105)) continue; if (cycleType === 'weekly') continue; if (maxDelta > Math.max(3, averageAmount * 0.18)) continue; let confidence; if (catalogEntry) { confidence = Math.min(99, 68 + sorted.length * 8 + (maxDelta <= 1 ? 8 : 0)); } else { confidence = Math.min(96, 58 + sorted.length * 9 + (maxDelta <= 1 ? 10 : 0)); } const tier = catalogEntry ? 'confirmed' : 'pattern'; recommendations.push(buildRecommendation({ merchant, catalogEntry, sorted, averageAmount, maxDelta, last, cycleType, avgGap, confidence, tier, declineKey, catalogTypeMap, })); } // Deduplicate by catalog entry — if multiple amount buckets matched the same // known service, keep only the highest-confidence one. const seen = new Map(); const deduped = []; for (const rec of recommendations.sort((a, b) => b.confidence - a.confidence || b.occurrence_count - a.occurrence_count)) { const key = rec.catalog_match ? `catalog:${rec.catalog_match.id}` : `merchant:${rec.merchant}`; if (!seen.has(key)) { seen.set(key, true); deduped.push(rec); } } return deduped.slice(0, 20); } function buildRecommendation({ merchant, catalogEntry, sorted, averageAmount, maxDelta, last, cycleType, avgGap, confidence, tier, declineKey, catalogTypeMap }) { const name = catalogEntry ? catalogEntry.name : titleCase(merchant); const subscriptionType = inferType(merchant, catalogEntry, catalogTypeMap); const accounts = Array.from(new Set(sorted .map(item => { const accountName = item.account_name || ''; const orgName = item.account_org_name || ''; if (accountName && orgName && accountName !== orgName) return `${orgName} · ${accountName}`; return accountName || orgName || item.data_source_name || ''; }) .filter(Boolean))); const reasons = []; if (catalogEntry) reasons.push(`Matches known service: ${catalogEntry.name}`); if (sorted.length > 1) reasons.push(`${sorted.length} similar charges`); if (sorted.length > 1) reasons.push(`About ${Math.round(avgGap)} days apart`); reasons.push(`${last.currency || 'USD'} ${averageAmount.toFixed(2)} average`); return { id: Buffer.from(`${merchant}:${Math.round(averageAmount)}:${last.tx_date}`).toString('base64url'), name, subscription_type: subscriptionType, expected_amount: Math.round(averageAmount * 100) / 100, monthly_equivalent: monthlyEquivalent(averageAmount, cycleType, cycleType), cycle_type: cycleType, billing_cycle: billingCycleForCycleType(cycleType), due_day: Number(String(last.tx_date).slice(8, 10)) || 1, last_seen_date: last.tx_date, occurrence_count: sorted.length, confidence, tier, catalog_match: catalogMatchPayload(catalogEntry), transaction_ids: sorted.map(item => item.id), merchant, decline_key: declineKey, source: last.data_source_name || 'Transaction history', accounts, reasons, }; } function searchSubscriptionTransactions(db, userId, query = {}) { const q = String(query.q || '').trim(); if (q.length < 2) return []; const limit = Math.max(1, Math.min(parseInt(query.limit || '50', 10) || 50, 100)); const like = `%${q}%`; const catalog = loadCatalog(db); const rows = db.prepare(` SELECT t.id, t.user_id, t.data_source_id, t.account_id, t.provider_transaction_id, t.source_type, t.transaction_type, t.posted_date, t.transacted_at, t.amount, t.currency, t.description, t.payee, t.memo, t.category, t.matched_bill_id, t.match_status, t.ignored, t.created_at, t.updated_at, ds.type AS data_source_type, ds.provider AS data_source_provider, ds.name AS data_source_name, ds.status AS data_source_status, fa.name AS account_name, fa.org_name AS account_org_name, fa.account_type AS account_type, b.name AS matched_bill_name FROM transactions t LEFT JOIN data_sources ds ON ds.id = t.data_source_id AND ds.user_id = t.user_id LEFT JOIN financial_accounts fa ON fa.id = t.account_id AND fa.user_id = t.user_id LEFT JOIN bills b ON b.id = t.matched_bill_id AND b.user_id = t.user_id AND b.deleted_at IS NULL WHERE t.user_id = ? AND t.ignored = 0 AND t.amount < 0 AND (t.account_id IS NULL OR fa.id IS NULL OR fa.monitored = 1) AND (t.description LIKE ? OR t.payee LIKE ? OR t.memo LIKE ? OR t.category LIKE ?) ORDER BY CASE WHEN t.match_status = 'unmatched' THEN 0 ELSE 1 END, COALESCE(t.posted_date, substr(t.transacted_at, 1, 10), t.created_at) DESC, t.id DESC LIMIT ? `).all(userId, like, like, like, like, limit); return rows.map(row => { const merchant = normalizeMerchant(row.payee || row.description || row.memo); const catalogEntry = lookupCatalog(catalog, merchant); return { ...row, amount_dollars: dollarsFromTransactionAmount(row.amount), merchant, is_known_subscription: !!catalogEntry, catalog_match: catalogMatchPayload(catalogEntry), }; }).sort((a, b) => Number(b.is_known_subscription) - Number(a.is_known_subscription)); } function createSubscriptionFromRecommendation(db, userId, payload = {}) { const seenDate = payload.last_seen_date || new Date().toISOString().slice(0, 10); const source = payload.catalog_match ? 'catalog_match' : 'simplefin_recommendation'; const draft = { name: payload.name, category_id: payload.category_id || null, due_day: payload.due_day, expected_amount: payload.expected_amount, billing_cycle: billingCycleForCycleType(payload.cycle_type || 'monthly'), cycle_type: payload.cycle_type || 'monthly', cycle_day: (payload.cycle_type === 'annual' || payload.cycle_type === 'quarterly') ? String(new Date(`${seenDate}T00:00:00`).getMonth() + 1) : (payload.cycle_type === 'weekly' || payload.cycle_type === 'biweekly') ? 'monday' : String(payload.due_day || 1), is_subscription: 1, subscription_type: SUBSCRIPTION_TYPES.includes(payload.subscription_type) ? payload.subscription_type : 'other', reminder_days_before: 3, subscription_source: source, subscription_detected_at: new Date().toISOString(), notes: payload.merchant ? `Detected from recurring merchant: ${payload.merchant}` : null, }; const validation = validateBillData(draft); if (validation.errors.length > 0) { const err = new Error(validation.errors[0].message); err.field = validation.errors[0].field; err.status = 400; throw err; } const created = insertBill(db, userId, validation.normalized); const ids = Array.isArray(payload.transaction_ids) ? payload.transaction_ids.map(id => Number(id)).filter(Number.isInteger).slice(0, 50) : []; if (ids.length > 0) { const placeholders = ids.map(() => '?').join(','); const txRows = db.prepare(` SELECT id, amount, posted_date, transacted_at FROM transactions WHERE user_id = ? AND id IN (${placeholders}) AND ignored = 0 `).all(userId, ...ids); const updateTx = db.prepare(` UPDATE transactions SET matched_bill_id = ?, match_status = 'matched', updated_at = CURRENT_TIMESTAMP WHERE id = ? AND user_id = ? AND ignored = 0 AND match_status != 'matched' `); const insertPayment = db.prepare(` INSERT OR IGNORE INTO payments (bill_id, amount, paid_date, payment_source, transaction_id) VALUES (?, ?, ?, 'auto_match', ?) `); db.transaction(() => { for (const tx of txRows) { const paidDate = tx.posted_date || (tx.transacted_at ? String(tx.transacted_at).slice(0, 10) : null); const amount = Math.round(Math.abs(tx.amount)) / 100; updateTx.run(created.id, tx.id, userId); if (paidDate) insertPayment.run(created.id, amount, paidDate, tx.id); } })(); } return decorateSubscription(created); } module.exports = { SUBSCRIPTION_TYPES, createSubscriptionFromRecommendation, declineRecommendation, decorateSubscription, getSubscriptionRecommendations, getSubscriptionSummary, getSubscriptions, lookupCatalog, loadCatalog, monthlyEquivalent, normalizeMerchant, searchSubscriptionTransactions, };