fix: subscription recommendation dedup and amount-bucket grouping

- Amount-bucket grouping ensures consistent charges are grouped together
- Catalog lookup names and boosts the result
- Deduplication ensures one recommendation per known service
- Removed catalog-first rewrite
This commit is contained in:
null 2026-05-29 02:51:30 -05:00
parent 1d8ae4f511
commit c43c476ae9
7 changed files with 155 additions and 32 deletions

View File

@ -184,6 +184,7 @@ export const api = {
subscriptionRecommendations: () => get('/subscriptions/recommendations'), subscriptionRecommendations: () => get('/subscriptions/recommendations'),
updateSubscription: (id, data) => _fetch('PATCH', `/subscriptions/${id}`, data), updateSubscription: (id, data) => _fetch('PATCH', `/subscriptions/${id}`, data),
createSubscriptionFromRecommendation: (data) => post('/subscriptions/recommendations/create', data), createSubscriptionFromRecommendation: (data) => post('/subscriptions/recommendations/create', data),
declineRecommendation: (declineKey) => post('/subscriptions/recommendations/decline', { decline_key: declineKey }),
// Payments // Payments
quickPay: (data) => post('/payments/quick', data), quickPay: (data) => post('/payments/quick', data),

View File

@ -11,6 +11,7 @@ import {
RefreshCw, RefreshCw,
Repeat, Repeat,
Sparkles, Sparkles,
X,
} from 'lucide-react'; } from 'lucide-react';
import { api } from '@/api'; import { api } from '@/api';
import { cn, fmt, fmtDate } from '@/lib/utils'; import { cn, fmt, fmtDate } from '@/lib/utils';
@ -111,7 +112,7 @@ function SubscriptionRow({ item, onEdit, onToggle }) {
); );
} }
function RecommendationCard({ recommendation, categoryId, onAccept, busy }) { function RecommendationCard({ recommendation, categoryId, onAccept, onDecline, busy }) {
return ( return (
<div className="rounded-lg border border-primary/20 bg-primary/[0.05] p-4"> <div className="rounded-lg border border-primary/20 bg-primary/[0.05] p-4">
<div className="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between"> <div className="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
@ -138,16 +139,29 @@ function RecommendationCard({ recommendation, categoryId, onAccept, busy }) {
<p className="tracker-number text-xs font-semibold text-emerald-600 dark:text-emerald-300"> <p className="tracker-number text-xs font-semibold text-emerald-600 dark:text-emerald-300">
{fmt(recommendation.monthly_equivalent)} / mo {fmt(recommendation.monthly_equivalent)} / mo
</p> </p>
<Button <div className="mt-3 flex gap-2 sm:justify-end">
type="button" <Button
size="sm" type="button"
className="mt-3 gap-2" size="sm"
disabled={busy} variant="ghost"
onClick={() => onAccept({ ...recommendation, category_id: categoryId })} className="gap-1.5 text-muted-foreground hover:text-destructive"
> disabled={busy}
{busy ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <CheckCircle2 className="h-3.5 w-3.5" />} onClick={() => onDecline(recommendation)}
Track >
</Button> {busy ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <X className="h-3.5 w-3.5" />}
Decline
</Button>
<Button
type="button"
size="sm"
className="gap-2"
disabled={busy}
onClick={() => onAccept({ ...recommendation, category_id: categoryId })}
>
{busy ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <CheckCircle2 className="h-3.5 w-3.5" />}
Track
</Button>
</div>
</div> </div>
</div> </div>
</div> </div>
@ -231,6 +245,19 @@ export default function SubscriptionsPage() {
} }
} }
async function declineRecommendation(recommendation) {
if (!recommendation.decline_key) return;
setBusyId(`dec-${recommendation.id}`);
try {
await api.declineRecommendation(recommendation.decline_key);
setRecommendations(prev => prev.filter(r => r.id !== recommendation.id));
} catch (err) {
toast.error(err.message || 'Could not dismiss recommendation.');
} finally {
setBusyId(null);
}
}
function openManualSubscription() { function openManualSubscription() {
setModal({ setModal({
bill: null, bill: null,
@ -342,8 +369,9 @@ export default function SubscriptionsPage() {
key={recommendation.id} key={recommendation.id}
recommendation={recommendation} recommendation={recommendation}
categoryId={subscriptionCategoryId} categoryId={subscriptionCategoryId}
busy={busyId === `rec-${recommendation.id}`} busy={busyId === `rec-${recommendation.id}` || busyId === `dec-${recommendation.id}`}
onAccept={acceptRecommendation} onAccept={acceptRecommendation}
onDecline={declineRecommendation}
/> />
)) ))
)} )}

View File

@ -1346,6 +1346,26 @@ function reconcileLegacyMigrations() {
run: function() { run: function() {
runSubscriptionCatalogMigration(db); runSubscriptionCatalogMigration(db);
} }
},
{
version: 'v0.66',
description: 'declined_subscription_hints: per-user dismissed recommendation store',
check: function() {
return !!db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='declined_subscription_hints'").get();
},
run: function() {
db.exec(`
CREATE TABLE IF NOT EXISTS declined_subscription_hints (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
decline_key TEXT NOT NULL,
declined_at TEXT NOT NULL DEFAULT (datetime('now')),
UNIQUE(user_id, decline_key)
);
CREATE INDEX IF NOT EXISTS idx_declined_hints_user
ON declined_subscription_hints(user_id);
`);
}
} }
]; ];
@ -2129,6 +2149,24 @@ function runMigrations() {
run: function() { run: function() {
runSubscriptionCatalogMigration(db); runSubscriptionCatalogMigration(db);
} }
},
{
version: 'v0.66',
description: 'declined_subscription_hints: per-user dismissed recommendation store',
dependsOn: ['v0.65'],
run: function() {
db.exec(`
CREATE TABLE IF NOT EXISTS declined_subscription_hints (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
decline_key TEXT NOT NULL,
declined_at TEXT NOT NULL DEFAULT (datetime('now')),
UNIQUE(user_id, decline_key)
);
CREATE INDEX IF NOT EXISTS idx_declined_hints_user
ON declined_subscription_hints(user_id);
`);
}
} }
]; ];

View File

@ -1,6 +1,6 @@
{ {
"name": "bill-tracker", "name": "bill-tracker",
"version": "0.33.3", "version": "0.33.4",
"description": "Monthly bill tracking system", "description": "Monthly bill tracking system",
"main": "server.js", "main": "server.js",
"scripts": { "scripts": {

View File

@ -4,6 +4,7 @@ const { getDb, ensureUserDefaultCategories } = require('../db/database');
const { standardizeError } = require('../middleware/errorFormatter'); const { standardizeError } = require('../middleware/errorFormatter');
const { const {
createSubscriptionFromRecommendation, createSubscriptionFromRecommendation,
declineRecommendation,
decorateSubscription, decorateSubscription,
getSubscriptionRecommendations, getSubscriptionRecommendations,
getSubscriptionSummary, getSubscriptionSummary,
@ -27,6 +28,19 @@ router.get('/recommendations', (req, res) => {
}); });
}); });
router.post('/recommendations/decline', (req, res) => {
const { decline_key } = req.body || {};
if (!decline_key || typeof decline_key !== 'string' || decline_key.length > 200) {
return res.status(400).json(standardizeError('decline_key is required', 'VALIDATION_ERROR', 'decline_key'));
}
try {
declineRecommendation(getDb(), req.user.id, decline_key);
res.json({ ok: true });
} catch (err) {
res.status(500).json(standardizeError(err.message || 'Failed to decline recommendation', 'DECLINE_ERROR'));
}
});
router.post('/recommendations/create', (req, res) => { router.post('/recommendations/create', (req, res) => {
const db = getDb(); const db = getDb();
ensureUserDefaultCategories(req.user.id); ensureUserDefaultCategories(req.user.id);

View File

@ -2,7 +2,8 @@
const { getSetting, setSetting } = require('../db/database'); const { getSetting, setSetting } = require('../db/database');
const SYNC_DAYS_MAX = 90; // SimpleFIN Bridge hard limit const SYNC_DAYS_MAX = 90; // SimpleFIN Bridge advertised limit
const SYNC_DAYS_EFFECTIVE = 89; // 1-day buffer to avoid bridge-side capping due to request latency
const SYNC_DAYS_DEFAULT = 90; const SYNC_DAYS_DEFAULT = 90;
const SYNC_INTERVAL_DEFAULT = 4; // hours const SYNC_INTERVAL_DEFAULT = 4; // hours
@ -26,7 +27,7 @@ function getBankSyncConfig() {
: Number.isFinite(syncDaysEnv) && syncDaysEnv > 0 : Number.isFinite(syncDaysEnv) && syncDaysEnv > 0
? syncDaysEnv ? syncDaysEnv
: SYNC_DAYS_DEFAULT; : SYNC_DAYS_DEFAULT;
const syncDays = Math.min(rawSyncDays, SYNC_DAYS_MAX); const syncDays = Math.min(rawSyncDays, SYNC_DAYS_EFFECTIVE);
const intervalDb = parseFloat(getSetting('simplefin_sync_interval_hours') || ''); const intervalDb = parseFloat(getSetting('simplefin_sync_interval_hours') || '');
const intervalEnv = parseFloat(process.env.SIMPLEFIN_SYNC_INTERVAL_HOURS || ''); const intervalEnv = parseFloat(process.env.SIMPLEFIN_SYNC_INTERVAL_HOURS || '');

View File

@ -18,6 +18,9 @@ const MONTHLY_FACTORS = {
irregular: 1, irregular: 1,
}; };
// Transactions that are clearly not subscriptions — skip before grouping
const SKIP_MERCHANT_RE = /\b(atm|withdrawal|transfer|deposit|zelle|venmo|wire|refund|rebate|interest charge)\b/;
// Fallback keyword list used when catalog lookup finds no match // Fallback keyword list used when catalog lookup finds no match
const TYPE_KEYWORDS = [ const TYPE_KEYWORDS = [
['streaming', ['netflix', 'hulu', 'disney', 'max', 'paramount', 'peacock', 'youtube tv', 'sling', 'espn', 'fubo', 'starz', 'crunchyroll', 'dazn']], ['streaming', ['netflix', 'hulu', 'disney', 'max', 'paramount', 'peacock', 'youtube tv', 'sling', 'espn', 'fubo', 'starz', 'crunchyroll', 'dazn']],
@ -46,7 +49,12 @@ function loadCatalog(db) {
} }
function normalizeCatalogName(value) { function normalizeCatalogName(value) {
return String(value || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim(); return String(value || '')
.toLowerCase()
.replace(/\+/g, ' plus ') // "Walmart+" → "walmart plus" so it only matches "walmart plus" transactions
.replace(/[^a-z0-9]+/g, ' ')
.replace(/\s+/g, ' ')
.trim();
} }
// Given a normalized merchant string, find the best matching catalog entry. // Given a normalized merchant string, find the best matching catalog entry.
@ -75,6 +83,7 @@ function lookupCatalog(catalog, merchantText) {
function normalizeMerchant(value) { function normalizeMerchant(value) {
return String(value || '') return String(value || '')
.toLowerCase() .toLowerCase()
.replace(/\+/g, ' plus ') // preserve "+" so "WALMART+" matches catalog "Walmart+" → "walmart plus"
.replace(/[^a-z0-9\s]/g, ' ') .replace(/[^a-z0-9\s]/g, ' ')
.replace(/\b(pos|debit|card|payment|purchase|recurring|online|inc|llc|co|www)\b/g, ' ') .replace(/\b(pos|debit|card|payment|purchase|recurring|online|inc|llc|co|www)\b/g, ' ')
.replace(/\s+/g, ' ') .replace(/\s+/g, ' ')
@ -188,13 +197,32 @@ function billingCycleForCycleType(cycleType) {
return 'irregular'; return 'irregular';
} }
// ── Decline store ─────────────────────────────────────────────────────────────
function getDeclinedKeys(db, userId) {
try {
const rows = db.prepare('SELECT decline_key FROM declined_subscription_hints WHERE user_id = ?').all(userId);
return new Set(rows.map(r => r.decline_key));
} catch {
return new Set();
}
}
function declineRecommendation(db, userId, declineKey) {
db.prepare(`
INSERT INTO declined_subscription_hints (user_id, decline_key)
VALUES (?, ?)
ON CONFLICT(user_id, decline_key) DO NOTHING
`).run(userId, declineKey);
}
// ── Recommendations ─────────────────────────────────────────────────────────── // ── Recommendations ───────────────────────────────────────────────────────────
function getSubscriptionRecommendations(db, userId) { function getSubscriptionRecommendations(db, userId) {
const catalog = loadCatalog(db); const catalog = loadCatalog(db);
const existingNames = existingBillNames(db, userId); const existingNames = existingBillNames(db, userId);
const declined = getDeclinedKeys(db, userId);
// Scan all transaction sources, not just SimpleFIN
const rows = db.prepare(` const rows = db.prepare(`
SELECT SELECT
t.id, t.amount, t.currency, t.description, t.payee, t.memo, t.category, t.id, t.amount, t.currency, t.description, t.payee, t.memo, t.category,
@ -211,16 +239,20 @@ function getSubscriptionRecommendations(db, userId) {
ORDER BY tx_date ASC ORDER BY tx_date ASC
`).all(userId); `).all(userId);
// Group by merchant + amount bucket // Group by merchant + amount bucket — consistent amounts are the foundation of
// subscription detection. Catalog lookup names the service and boosts confidence
// but does not change the grouping; deduplication at the end ensures one entry
// per known service.
const groups = new Map(); const groups = new Map();
for (const tx of rows) { for (const tx of rows) {
const merchant = normalizeMerchant(tx.payee || tx.description || tx.memo); const merchant = normalizeMerchant(tx.payee || tx.description || tx.memo);
if (!merchant || merchant.length < 3) continue; if (!merchant || merchant.length < 3) continue;
if (SKIP_MERCHANT_RE.test(merchant)) continue;
const amount = dollarsFromTransactionAmount(tx.amount); const amount = dollarsFromTransactionAmount(tx.amount);
if (amount < 1) continue; if (amount < 1) continue;
const key = `${merchant}:${Math.round(amount)}`; const key = `${merchant}:${Math.round(amount)}`;
if (!groups.has(key)) { if (!groups.has(key)) {
groups.set(key, { merchant, amountBucket: Math.round(amount), items: [], catalogEntry: null }); groups.set(key, { merchant, items: [], catalogEntry: null });
} }
const group = groups.get(key); const group = groups.get(key);
group.items.push({ ...tx, amount_dollars: amount }); group.items.push({ ...tx, amount_dollars: amount });
@ -231,14 +263,14 @@ function getSubscriptionRecommendations(db, userId) {
for (const group of groups.values()) { for (const group of groups.values()) {
const { merchant, catalogEntry } = group; const { merchant, catalogEntry } = group;
const declineKey = catalogEntry ? `catalog:${catalogEntry.id}` : `merchant:${merchant}`;
// Skip if already a known bill if (declined.has(declineKey)) continue;
if (existingNames.some(name => name.includes(merchant) || merchant.includes(name))) continue; if (existingNames.some(n => n.includes(merchant) || merchant.includes(n))) continue;
const sorted = group.items const sorted = group.items
.filter(item => item.tx_date) .filter(item => item.tx_date)
.sort((a, b) => String(a.tx_date).localeCompare(String(b.tx_date))); .sort((a, b) => String(a.tx_date).localeCompare(String(b.tx_date)));
if (sorted.length === 0) continue; if (sorted.length === 0) continue;
const averageAmount = sorted.reduce((sum, item) => sum + item.amount_dollars, 0) / sorted.length; const averageAmount = sorted.reduce((sum, item) => sum + item.amount_dollars, 0) / sorted.length;
@ -247,17 +279,15 @@ function getSubscriptionRecommendations(db, userId) {
: 0; : 0;
const last = sorted[sorted.length - 1]; const last = sorted[sorted.length - 1];
// ── Tier 1: catalog match with 1 occurrence (possible subscription) ────── // Tier 1: catalog match with 1 occurrence
if (catalogEntry && sorted.length === 1) { if (catalogEntry && sorted.length === 1) {
const confidence = 62;
recommendations.push(buildRecommendation({ recommendations.push(buildRecommendation({
merchant, catalogEntry, sorted, averageAmount, maxDelta, last, merchant, catalogEntry, sorted, averageAmount, maxDelta, last,
cycleType: 'monthly', avgGap: 30, confidence, tier: 'possible', cycleType: 'monthly', avgGap: 30, confidence: 62, tier: 'possible', declineKey,
})); }));
continue; continue;
} }
// ── Tier 2: 2+ occurrences — pattern detection ────────────────────────────
if (sorted.length < 2) continue; if (sorted.length < 2) continue;
const gaps = []; const gaps = [];
@ -275,9 +305,9 @@ function getSubscriptionRecommendations(db, userId) {
if (cycleType === 'monthly' && (avgGap < 24 || avgGap > 38)) continue; if (cycleType === 'monthly' && (avgGap < 24 || avgGap > 38)) continue;
if (cycleType === 'quarterly' && (avgGap < 75 || avgGap > 105)) continue; if (cycleType === 'quarterly' && (avgGap < 75 || avgGap > 105)) continue;
if (cycleType === 'weekly') continue;
if (maxDelta > Math.max(3, averageAmount * 0.18)) continue; if (maxDelta > Math.max(3, averageAmount * 0.18)) continue;
// Confidence: catalog match raises the floor and ceiling
let confidence; let confidence;
if (catalogEntry) { if (catalogEntry) {
confidence = Math.min(99, 68 + sorted.length * 8 + (maxDelta <= 1 ? 8 : 0)); confidence = Math.min(99, 68 + sorted.length * 8 + (maxDelta <= 1 ? 8 : 0));
@ -288,16 +318,25 @@ function getSubscriptionRecommendations(db, userId) {
const tier = catalogEntry ? 'confirmed' : 'pattern'; const tier = catalogEntry ? 'confirmed' : 'pattern';
recommendations.push(buildRecommendation({ recommendations.push(buildRecommendation({
merchant, catalogEntry, sorted, averageAmount, maxDelta, last, merchant, catalogEntry, sorted, averageAmount, maxDelta, last,
cycleType, avgGap, confidence, tier, cycleType, avgGap, confidence, tier, declineKey,
})); }));
} }
return recommendations // Deduplicate by catalog entry — if multiple amount buckets matched the same
.sort((a, b) => b.confidence - a.confidence || b.occurrence_count - a.occurrence_count) // known service, keep only the highest-confidence one.
.slice(0, 20); const seen = new Map();
const deduped = [];
for (const rec of recommendations.sort((a, b) => b.confidence - a.confidence || b.occurrence_count - a.occurrence_count)) {
const key = rec.catalog_match ? `catalog:${rec.catalog_match.id}` : `merchant:${rec.merchant}`;
if (!seen.has(key)) {
seen.set(key, true);
deduped.push(rec);
}
}
return deduped.slice(0, 20);
} }
function buildRecommendation({ merchant, catalogEntry, sorted, averageAmount, maxDelta, last, cycleType, avgGap, confidence, tier }) { function buildRecommendation({ merchant, catalogEntry, sorted, averageAmount, maxDelta, last, cycleType, avgGap, confidence, tier, declineKey }) {
const name = catalogEntry ? catalogEntry.name : titleCase(merchant); const name = catalogEntry ? catalogEntry.name : titleCase(merchant);
const subscriptionType = inferType(merchant, catalogEntry); const subscriptionType = inferType(merchant, catalogEntry);
@ -323,6 +362,7 @@ function buildRecommendation({ merchant, catalogEntry, sorted, averageAmount, ma
catalog_match: catalogEntry ? { id: catalogEntry.id, name: catalogEntry.name, category: catalogEntry.category } : null, catalog_match: catalogEntry ? { id: catalogEntry.id, name: catalogEntry.name, category: catalogEntry.category } : null,
transaction_ids: sorted.map(item => item.id), transaction_ids: sorted.map(item => item.id),
merchant, merchant,
decline_key: declineKey,
source: last.data_source_name || 'Transaction history', source: last.data_source_name || 'Transaction history',
reasons, reasons,
}; };
@ -379,6 +419,7 @@ function createSubscriptionFromRecommendation(db, userId, payload = {}) {
module.exports = { module.exports = {
SUBSCRIPTION_TYPES, SUBSCRIPTION_TYPES,
createSubscriptionFromRecommendation, createSubscriptionFromRecommendation,
declineRecommendation,
decorateSubscription, decorateSubscription,
getSubscriptionRecommendations, getSubscriptionRecommendations,
getSubscriptionSummary, getSubscriptionSummary,