390 lines
16 KiB
JavaScript
390 lines
16 KiB
JavaScript
'use strict';
|
|
|
|
const { insertBill, validateBillData } = require('./billsService');
|
|
|
|
const SUBSCRIPTION_TYPES = [
|
|
'streaming', 'software', 'cloud', 'music', 'news',
|
|
'fitness', 'gaming', 'utilities', 'insurance',
|
|
'food', 'education', 'shopping', 'security', 'other',
|
|
];
|
|
|
|
const MONTHLY_FACTORS = {
|
|
weekly: 52 / 12,
|
|
biweekly: 26 / 12,
|
|
monthly: 1,
|
|
quarterly: 1 / 3,
|
|
annual: 1 / 12,
|
|
annually: 1 / 12,
|
|
irregular: 1,
|
|
};
|
|
|
|
// Fallback keyword list used when catalog lookup finds no match
|
|
const TYPE_KEYWORDS = [
|
|
['streaming', ['netflix', 'hulu', 'disney', 'max', 'paramount', 'peacock', 'youtube tv', 'sling', 'espn', 'fubo', 'starz', 'crunchyroll', 'dazn']],
|
|
['music', ['spotify', 'apple music', 'tidal', 'pandora', 'siriusxm', 'soundcloud', 'deezer', 'iheart']],
|
|
['software', ['adobe', 'microsoft', 'github', 'notion', 'figma', 'canva', 'openai', 'chatgpt', 'grammarly', 'zoom', 'slack', 'cursor', 'ynab']],
|
|
['cloud', ['dropbox', 'icloud', 'google one', 'google storage', 'backblaze', 'box storage']],
|
|
['news', ['nyt', 'new york times', 'economist', 'athletic', 'washington post', 'wsj', 'bloomberg', 'substack', 'patreon', 'medium']],
|
|
['fitness', ['peloton', 'planet fitness', 'gym', 'fitbit', 'strava', 'headspace', 'calm', 'noom', 'classpass', 'whoop']],
|
|
['gaming', ['xbox', 'playstation', 'steam', 'nintendo', 'roblox', 'discord nitro', 'ea play', 'ubisoft']],
|
|
['utilities', ['verizon', 'at t', 'att', 'comcast', 'xfinity', 'spectrum', 'tmobile', 't mobile']],
|
|
['insurance', ['insurance', 'geico', 'progressive', 'state farm', 'allstate']],
|
|
['food', ['hellofresh', 'blue apron', 'doordash', 'instacart', 'uber eats', 'grubhub', 'factor', 'hungryroot']],
|
|
['education', ['duolingo', 'masterclass', 'coursera', 'skillshare', 'audible', 'kindle unlimited', 'blinkist']],
|
|
['shopping', ['amazon prime', 'walmart plus', 'costco', 'target circle', 'chewy']],
|
|
['security', ['nordvpn', 'expressvpn', '1password', 'dashlane', 'norton', 'mcafee', 'surfshark']],
|
|
];
|
|
|
|
// ── Catalog ───────────────────────────────────────────────────────────────────
|
|
|
|
function loadCatalog(db) {
|
|
try {
|
|
return db.prepare('SELECT id, rank, name, category, subscription_type, domain FROM subscription_catalog ORDER BY rank ASC').all();
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
function normalizeCatalogName(value) {
|
|
return String(value || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
|
|
}
|
|
|
|
// Given a normalized merchant string, find the best matching catalog entry.
|
|
// Matches on service name (normalized) or domain (dot replaced with space).
|
|
function lookupCatalog(catalog, merchantText) {
|
|
if (!catalog.length || !merchantText) return null;
|
|
let best = null;
|
|
let bestLen = 0;
|
|
for (const entry of catalog) {
|
|
const nameKey = normalizeCatalogName(entry.name);
|
|
const domainKey = entry.domain ? entry.domain.replace(/\./g, ' ') : '';
|
|
if (nameKey.length >= 3 && merchantText.includes(nameKey) && nameKey.length > bestLen) {
|
|
best = entry;
|
|
bestLen = nameKey.length;
|
|
}
|
|
if (domainKey.length >= 4 && merchantText.includes(domainKey) && domainKey.length > bestLen) {
|
|
best = entry;
|
|
bestLen = domainKey.length;
|
|
}
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// ── Helpers ───────────────────────────────────────────────────────────────────
|
|
|
|
function normalizeMerchant(value) {
|
|
return String(value || '')
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9\s]/g, ' ')
|
|
.replace(/\b(pos|debit|card|payment|purchase|recurring|online|inc|llc|co|www)\b/g, ' ')
|
|
.replace(/\s+/g, ' ')
|
|
.trim();
|
|
}
|
|
|
|
function titleCase(value) {
|
|
return String(value || 'Subscription')
|
|
.split(/\s+/)
|
|
.filter(Boolean)
|
|
.map(word => word.charAt(0).toUpperCase() + word.slice(1))
|
|
.join(' ');
|
|
}
|
|
|
|
function inferType(merchantText, catalogEntry) {
|
|
if (catalogEntry?.subscription_type) return catalogEntry.subscription_type;
|
|
const haystack = normalizeMerchant(merchantText);
|
|
for (const [type, words] of TYPE_KEYWORDS) {
|
|
if (words.some(word => haystack.includes(word))) return type;
|
|
}
|
|
return 'other';
|
|
}
|
|
|
|
function monthlyEquivalent(amount, cycleType, billingCycle) {
|
|
const key = String(cycleType || billingCycle || 'monthly').toLowerCase();
|
|
const fallback = String(billingCycle || '').toLowerCase() === 'quarterly'
|
|
? 'quarterly'
|
|
: String(billingCycle || '').toLowerCase() === 'annually'
|
|
? 'annual'
|
|
: key;
|
|
const factor = MONTHLY_FACTORS[key] ?? MONTHLY_FACTORS[fallback] ?? 1;
|
|
return Math.round(Number(amount || 0) * factor * 100) / 100;
|
|
}
|
|
|
|
function nextDueDate(bill, now = new Date()) {
|
|
const dueDay = Math.min(Math.max(Number(bill.due_day) || 1, 1), 31);
|
|
const cycle = String(bill.cycle_type || bill.billing_cycle || 'monthly').toLowerCase();
|
|
let date = new Date(now.getFullYear(), now.getMonth(), dueDay);
|
|
if (date < new Date(now.getFullYear(), now.getMonth(), now.getDate())) {
|
|
date = new Date(now.getFullYear(), now.getMonth() + 1, dueDay);
|
|
}
|
|
if (cycle === 'quarterly' || cycle === 'annual') {
|
|
const startMonth = Math.min(Math.max(Number(bill.cycle_day) || 1, 1), 12) - 1;
|
|
const step = cycle === 'quarterly' ? 3 : 12;
|
|
date = new Date(now.getFullYear(), startMonth, dueDay);
|
|
while (date < new Date(now.getFullYear(), now.getMonth(), now.getDate())) {
|
|
date = new Date(date.getFullYear(), date.getMonth() + step, dueDay);
|
|
}
|
|
}
|
|
return date.toISOString().slice(0, 10);
|
|
}
|
|
|
|
function decorateSubscription(bill) {
|
|
const monthly = monthlyEquivalent(bill.expected_amount, bill.cycle_type, bill.billing_cycle);
|
|
return {
|
|
...bill,
|
|
is_subscription: !!bill.is_subscription,
|
|
active: !!bill.active,
|
|
monthly_equivalent: monthly,
|
|
yearly_equivalent: Math.round(monthly * 12 * 100) / 100,
|
|
next_due_date: nextDueDate(bill),
|
|
subscription_type: bill.subscription_type || inferType(`${bill.name} ${bill.category_name || ''}`, null),
|
|
};
|
|
}
|
|
|
|
function getSubscriptions(db, userId) {
|
|
return db.prepare(`
|
|
SELECT b.*, c.name AS category_name
|
|
FROM bills b
|
|
LEFT JOIN categories c ON c.id = b.category_id AND c.user_id = b.user_id AND c.deleted_at IS NULL
|
|
WHERE b.user_id = ?
|
|
AND b.deleted_at IS NULL
|
|
AND b.is_subscription = 1
|
|
ORDER BY b.active DESC, b.due_day ASC, b.name COLLATE NOCASE ASC
|
|
`).all(userId).map(decorateSubscription);
|
|
}
|
|
|
|
function getSubscriptionSummary(subscriptions) {
|
|
const active = subscriptions.filter(item => item.active);
|
|
const monthlyTotal = active.reduce((sum, item) => sum + Number(item.monthly_equivalent || 0), 0);
|
|
const typeTotals = new Map();
|
|
for (const item of active) {
|
|
const type = item.subscription_type || 'other';
|
|
typeTotals.set(type, (typeTotals.get(type) || 0) + Number(item.monthly_equivalent || 0));
|
|
}
|
|
const topType = [...typeTotals.entries()].sort((a, b) => b[1] - a[1])[0] || null;
|
|
return {
|
|
active_count: active.length,
|
|
paused_count: subscriptions.length - active.length,
|
|
monthly_total: Math.round(monthlyTotal * 100) / 100,
|
|
yearly_total: Math.round(monthlyTotal * 12 * 100) / 100,
|
|
top_type: topType ? { type: topType[0], monthly_total: Math.round(topType[1] * 100) / 100 } : null,
|
|
};
|
|
}
|
|
|
|
function existingBillNames(db, userId) {
|
|
return db.prepare('SELECT name FROM bills WHERE user_id = ? AND deleted_at IS NULL')
|
|
.all(userId)
|
|
.map(row => normalizeMerchant(row.name))
|
|
.filter(Boolean);
|
|
}
|
|
|
|
function dollarsFromTransactionAmount(amount) {
|
|
return Math.round((Math.abs(Number(amount || 0)) / 100) * 100) / 100;
|
|
}
|
|
|
|
function billingCycleForCycleType(cycleType) {
|
|
if (cycleType === 'quarterly') return 'quarterly';
|
|
if (cycleType === 'annual') return 'annually';
|
|
if (cycleType === 'monthly') return 'monthly';
|
|
return 'irregular';
|
|
}
|
|
|
|
// ── Recommendations ───────────────────────────────────────────────────────────
|
|
|
|
function getSubscriptionRecommendations(db, userId) {
|
|
const catalog = loadCatalog(db);
|
|
const existingNames = existingBillNames(db, userId);
|
|
|
|
// Scan all transaction sources, not just SimpleFIN
|
|
const rows = db.prepare(`
|
|
SELECT
|
|
t.id, t.amount, t.currency, t.description, t.payee, t.memo, t.category,
|
|
COALESCE(t.posted_date, substr(t.transacted_at, 1, 10)) AS tx_date,
|
|
ds.provider AS data_source_provider,
|
|
ds.name AS data_source_name
|
|
FROM transactions t
|
|
LEFT JOIN data_sources ds ON ds.id = t.data_source_id AND ds.user_id = t.user_id
|
|
WHERE t.user_id = ?
|
|
AND t.ignored = 0
|
|
AND t.match_status = 'unmatched'
|
|
AND t.amount < 0
|
|
AND COALESCE(t.posted_date, substr(t.transacted_at, 1, 10)) >= date('now', '-420 days')
|
|
ORDER BY tx_date ASC
|
|
`).all(userId);
|
|
|
|
// Group by merchant + amount bucket
|
|
const groups = new Map();
|
|
for (const tx of rows) {
|
|
const merchant = normalizeMerchant(tx.payee || tx.description || tx.memo);
|
|
if (!merchant || merchant.length < 3) continue;
|
|
const amount = dollarsFromTransactionAmount(tx.amount);
|
|
if (amount < 1) continue;
|
|
const key = `${merchant}:${Math.round(amount)}`;
|
|
if (!groups.has(key)) {
|
|
groups.set(key, { merchant, amountBucket: Math.round(amount), items: [], catalogEntry: null });
|
|
}
|
|
const group = groups.get(key);
|
|
group.items.push({ ...tx, amount_dollars: amount });
|
|
if (!group.catalogEntry) group.catalogEntry = lookupCatalog(catalog, merchant);
|
|
}
|
|
|
|
const recommendations = [];
|
|
|
|
for (const group of groups.values()) {
|
|
const { merchant, catalogEntry } = group;
|
|
|
|
// Skip if already a known bill
|
|
if (existingNames.some(name => name.includes(merchant) || merchant.includes(name))) continue;
|
|
|
|
const sorted = group.items
|
|
.filter(item => item.tx_date)
|
|
.sort((a, b) => String(a.tx_date).localeCompare(String(b.tx_date)));
|
|
|
|
if (sorted.length === 0) continue;
|
|
|
|
const averageAmount = sorted.reduce((sum, item) => sum + item.amount_dollars, 0) / sorted.length;
|
|
const maxDelta = sorted.length > 1
|
|
? Math.max(...sorted.map(item => Math.abs(item.amount_dollars - averageAmount)))
|
|
: 0;
|
|
const last = sorted[sorted.length - 1];
|
|
|
|
// ── Tier 1: catalog match with 1 occurrence (possible subscription) ──────
|
|
if (catalogEntry && sorted.length === 1) {
|
|
const confidence = 62;
|
|
recommendations.push(buildRecommendation({
|
|
merchant, catalogEntry, sorted, averageAmount, maxDelta, last,
|
|
cycleType: 'monthly', avgGap: 30, confidence, tier: 'possible',
|
|
}));
|
|
continue;
|
|
}
|
|
|
|
// ── Tier 2: 2+ occurrences — pattern detection ────────────────────────────
|
|
if (sorted.length < 2) continue;
|
|
|
|
const gaps = [];
|
|
for (let i = 1; i < sorted.length; i++) {
|
|
gaps.push(Math.round(
|
|
(new Date(`${sorted[i].tx_date}T00:00:00`) - new Date(`${sorted[i - 1].tx_date}T00:00:00`)) / 86400000
|
|
));
|
|
}
|
|
const avgGap = gaps.reduce((sum, g) => sum + g, 0) / gaps.length;
|
|
const cycleType = avgGap >= 320 ? 'annual'
|
|
: avgGap >= 75 ? 'quarterly'
|
|
: avgGap >= 10 && avgGap <= 18 ? 'biweekly'
|
|
: avgGap <= 9 ? 'weekly'
|
|
: 'monthly';
|
|
|
|
if (cycleType === 'monthly' && (avgGap < 24 || avgGap > 38)) continue;
|
|
if (cycleType === 'quarterly' && (avgGap < 75 || avgGap > 105)) continue;
|
|
if (maxDelta > Math.max(3, averageAmount * 0.18)) continue;
|
|
|
|
// Confidence: catalog match raises the floor and ceiling
|
|
let confidence;
|
|
if (catalogEntry) {
|
|
confidence = Math.min(99, 68 + sorted.length * 8 + (maxDelta <= 1 ? 8 : 0));
|
|
} else {
|
|
confidence = Math.min(96, 58 + sorted.length * 9 + (maxDelta <= 1 ? 10 : 0));
|
|
}
|
|
|
|
const tier = catalogEntry ? 'confirmed' : 'pattern';
|
|
recommendations.push(buildRecommendation({
|
|
merchant, catalogEntry, sorted, averageAmount, maxDelta, last,
|
|
cycleType, avgGap, confidence, tier,
|
|
}));
|
|
}
|
|
|
|
return recommendations
|
|
.sort((a, b) => b.confidence - a.confidence || b.occurrence_count - a.occurrence_count)
|
|
.slice(0, 20);
|
|
}
|
|
|
|
function buildRecommendation({ merchant, catalogEntry, sorted, averageAmount, maxDelta, last, cycleType, avgGap, confidence, tier }) {
|
|
const name = catalogEntry ? catalogEntry.name : titleCase(merchant);
|
|
const subscriptionType = inferType(merchant, catalogEntry);
|
|
|
|
const reasons = [];
|
|
if (catalogEntry) reasons.push(`Matches known service: ${catalogEntry.name}`);
|
|
if (sorted.length > 1) reasons.push(`${sorted.length} similar charges`);
|
|
if (sorted.length > 1) reasons.push(`About ${Math.round(avgGap)} days apart`);
|
|
reasons.push(`${last.currency || 'USD'} ${averageAmount.toFixed(2)} average`);
|
|
|
|
return {
|
|
id: Buffer.from(`${merchant}:${Math.round(averageAmount)}:${last.tx_date}`).toString('base64url'),
|
|
name,
|
|
subscription_type: subscriptionType,
|
|
expected_amount: Math.round(averageAmount * 100) / 100,
|
|
monthly_equivalent: monthlyEquivalent(averageAmount, cycleType, cycleType),
|
|
cycle_type: cycleType,
|
|
billing_cycle: billingCycleForCycleType(cycleType),
|
|
due_day: Number(String(last.tx_date).slice(8, 10)) || 1,
|
|
last_seen_date: last.tx_date,
|
|
occurrence_count: sorted.length,
|
|
confidence,
|
|
tier,
|
|
catalog_match: catalogEntry ? { id: catalogEntry.id, name: catalogEntry.name, category: catalogEntry.category } : null,
|
|
transaction_ids: sorted.map(item => item.id),
|
|
merchant,
|
|
source: last.data_source_name || 'Transaction history',
|
|
reasons,
|
|
};
|
|
}
|
|
|
|
function createSubscriptionFromRecommendation(db, userId, payload = {}) {
|
|
const seenDate = payload.last_seen_date || new Date().toISOString().slice(0, 10);
|
|
const source = payload.catalog_match
|
|
? 'catalog_match'
|
|
: 'simplefin_recommendation';
|
|
|
|
const draft = {
|
|
name: payload.name,
|
|
category_id: payload.category_id || null,
|
|
due_day: payload.due_day,
|
|
expected_amount: payload.expected_amount,
|
|
billing_cycle: billingCycleForCycleType(payload.cycle_type || 'monthly'),
|
|
cycle_type: payload.cycle_type || 'monthly',
|
|
cycle_day: payload.cycle_type === 'annual' || payload.cycle_type === 'quarterly'
|
|
? String(new Date(`${seenDate}T00:00:00`).getMonth() + 1)
|
|
: String(payload.due_day || 1),
|
|
is_subscription: 1,
|
|
subscription_type: SUBSCRIPTION_TYPES.includes(payload.subscription_type) ? payload.subscription_type : 'other',
|
|
reminder_days_before: 3,
|
|
subscription_source: source,
|
|
subscription_detected_at: new Date().toISOString(),
|
|
notes: payload.merchant ? `Detected from recurring merchant: ${payload.merchant}` : null,
|
|
};
|
|
|
|
const validation = validateBillData(draft);
|
|
if (validation.errors.length > 0) {
|
|
const err = new Error(validation.errors[0].message);
|
|
err.field = validation.errors[0].field;
|
|
err.status = 400;
|
|
throw err;
|
|
}
|
|
|
|
const created = insertBill(db, userId, validation.normalized);
|
|
const ids = Array.isArray(payload.transaction_ids)
|
|
? payload.transaction_ids.map(id => Number(id)).filter(Number.isInteger).slice(0, 50)
|
|
: [];
|
|
if (ids.length > 0) {
|
|
const update = db.prepare(`
|
|
UPDATE transactions
|
|
SET matched_bill_id = ?, match_status = 'matched', updated_at = CURRENT_TIMESTAMP
|
|
WHERE user_id = ? AND id = ? AND ignored = 0
|
|
`);
|
|
for (const id of ids) update.run(created.id, userId, id);
|
|
}
|
|
|
|
return decorateSubscription(created);
|
|
}
|
|
|
|
module.exports = {
|
|
SUBSCRIPTION_TYPES,
|
|
createSubscriptionFromRecommendation,
|
|
decorateSubscription,
|
|
getSubscriptionRecommendations,
|
|
getSubscriptionSummary,
|
|
getSubscriptions,
|
|
lookupCatalog,
|
|
loadCatalog,
|
|
monthlyEquivalent,
|
|
};
|