fix: subscription recommendation dedup and amount-bucket grouping
- Amount-bucket grouping ensures consistent charges are grouped together - Catalog lookup names and boosts the result - Deduplication ensures one recommendation per known service - Removed catalog-first rewrite
This commit is contained in:
null
parent
1d8ae4f511
commit
c43c476ae9
|
|
@ -184,6 +184,7 @@ export const api = {
|
|||
subscriptionRecommendations: () => get('/subscriptions/recommendations'),
|
||||
updateSubscription: (id, data) => _fetch('PATCH', `/subscriptions/${id}`, data),
|
||||
createSubscriptionFromRecommendation: (data) => post('/subscriptions/recommendations/create', data),
|
||||
declineRecommendation: (declineKey) => post('/subscriptions/recommendations/decline', { decline_key: declineKey }),
|
||||
|
||||
// Payments
|
||||
quickPay: (data) => post('/payments/quick', data),
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import {
|
|||
RefreshCw,
|
||||
Repeat,
|
||||
Sparkles,
|
||||
X,
|
||||
} from 'lucide-react';
|
||||
import { api } from '@/api';
|
||||
import { cn, fmt, fmtDate } from '@/lib/utils';
|
||||
|
|
@ -111,7 +112,7 @@ function SubscriptionRow({ item, onEdit, onToggle }) {
|
|||
);
|
||||
}
|
||||
|
||||
function RecommendationCard({ recommendation, categoryId, onAccept, busy }) {
|
||||
function RecommendationCard({ recommendation, categoryId, onAccept, onDecline, busy }) {
|
||||
return (
|
||||
<div className="rounded-lg border border-primary/20 bg-primary/[0.05] p-4">
|
||||
<div className="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
|
||||
|
|
@ -138,16 +139,29 @@ function RecommendationCard({ recommendation, categoryId, onAccept, busy }) {
|
|||
<p className="tracker-number text-xs font-semibold text-emerald-600 dark:text-emerald-300">
|
||||
{fmt(recommendation.monthly_equivalent)} / mo
|
||||
</p>
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
className="mt-3 gap-2"
|
||||
disabled={busy}
|
||||
onClick={() => onAccept({ ...recommendation, category_id: categoryId })}
|
||||
>
|
||||
{busy ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <CheckCircle2 className="h-3.5 w-3.5" />}
|
||||
Track
|
||||
</Button>
|
||||
<div className="mt-3 flex gap-2 sm:justify-end">
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
className="gap-1.5 text-muted-foreground hover:text-destructive"
|
||||
disabled={busy}
|
||||
onClick={() => onDecline(recommendation)}
|
||||
>
|
||||
{busy ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <X className="h-3.5 w-3.5" />}
|
||||
Decline
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
className="gap-2"
|
||||
disabled={busy}
|
||||
onClick={() => onAccept({ ...recommendation, category_id: categoryId })}
|
||||
>
|
||||
{busy ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <CheckCircle2 className="h-3.5 w-3.5" />}
|
||||
Track
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -231,6 +245,19 @@ export default function SubscriptionsPage() {
|
|||
}
|
||||
}
|
||||
|
||||
async function declineRecommendation(recommendation) {
|
||||
if (!recommendation.decline_key) return;
|
||||
setBusyId(`dec-${recommendation.id}`);
|
||||
try {
|
||||
await api.declineRecommendation(recommendation.decline_key);
|
||||
setRecommendations(prev => prev.filter(r => r.id !== recommendation.id));
|
||||
} catch (err) {
|
||||
toast.error(err.message || 'Could not dismiss recommendation.');
|
||||
} finally {
|
||||
setBusyId(null);
|
||||
}
|
||||
}
|
||||
|
||||
function openManualSubscription() {
|
||||
setModal({
|
||||
bill: null,
|
||||
|
|
@ -342,8 +369,9 @@ export default function SubscriptionsPage() {
|
|||
key={recommendation.id}
|
||||
recommendation={recommendation}
|
||||
categoryId={subscriptionCategoryId}
|
||||
busy={busyId === `rec-${recommendation.id}`}
|
||||
busy={busyId === `rec-${recommendation.id}` || busyId === `dec-${recommendation.id}`}
|
||||
onAccept={acceptRecommendation}
|
||||
onDecline={declineRecommendation}
|
||||
/>
|
||||
))
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -1346,6 +1346,26 @@ function reconcileLegacyMigrations() {
|
|||
run: function() {
|
||||
runSubscriptionCatalogMigration(db);
|
||||
}
|
||||
},
|
||||
{
|
||||
version: 'v0.66',
|
||||
description: 'declined_subscription_hints: per-user dismissed recommendation store',
|
||||
check: function() {
|
||||
return !!db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='declined_subscription_hints'").get();
|
||||
},
|
||||
run: function() {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS declined_subscription_hints (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
decline_key TEXT NOT NULL,
|
||||
declined_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE(user_id, decline_key)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_declined_hints_user
|
||||
ON declined_subscription_hints(user_id);
|
||||
`);
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
|
|
@ -2129,6 +2149,24 @@ function runMigrations() {
|
|||
run: function() {
|
||||
runSubscriptionCatalogMigration(db);
|
||||
}
|
||||
},
|
||||
{
|
||||
version: 'v0.66',
|
||||
description: 'declined_subscription_hints: per-user dismissed recommendation store',
|
||||
dependsOn: ['v0.65'],
|
||||
run: function() {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS declined_subscription_hints (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
decline_key TEXT NOT NULL,
|
||||
declined_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE(user_id, decline_key)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_declined_hints_user
|
||||
ON declined_subscription_hints(user_id);
|
||||
`);
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "bill-tracker",
|
||||
"version": "0.33.3",
|
||||
"version": "0.33.4",
|
||||
"description": "Monthly bill tracking system",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ const { getDb, ensureUserDefaultCategories } = require('../db/database');
|
|||
const { standardizeError } = require('../middleware/errorFormatter');
|
||||
const {
|
||||
createSubscriptionFromRecommendation,
|
||||
declineRecommendation,
|
||||
decorateSubscription,
|
||||
getSubscriptionRecommendations,
|
||||
getSubscriptionSummary,
|
||||
|
|
@ -27,6 +28,19 @@ router.get('/recommendations', (req, res) => {
|
|||
});
|
||||
});
|
||||
|
||||
router.post('/recommendations/decline', (req, res) => {
|
||||
const { decline_key } = req.body || {};
|
||||
if (!decline_key || typeof decline_key !== 'string' || decline_key.length > 200) {
|
||||
return res.status(400).json(standardizeError('decline_key is required', 'VALIDATION_ERROR', 'decline_key'));
|
||||
}
|
||||
try {
|
||||
declineRecommendation(getDb(), req.user.id, decline_key);
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
res.status(500).json(standardizeError(err.message || 'Failed to decline recommendation', 'DECLINE_ERROR'));
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/recommendations/create', (req, res) => {
|
||||
const db = getDb();
|
||||
ensureUserDefaultCategories(req.user.id);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@
|
|||
|
||||
const { getSetting, setSetting } = require('../db/database');
|
||||
|
||||
const SYNC_DAYS_MAX = 90; // SimpleFIN Bridge hard limit
|
||||
const SYNC_DAYS_MAX = 90; // SimpleFIN Bridge advertised limit
|
||||
const SYNC_DAYS_EFFECTIVE = 89; // 1-day buffer to avoid bridge-side capping due to request latency
|
||||
const SYNC_DAYS_DEFAULT = 90;
|
||||
const SYNC_INTERVAL_DEFAULT = 4; // hours
|
||||
|
||||
|
|
@ -26,7 +27,7 @@ function getBankSyncConfig() {
|
|||
: Number.isFinite(syncDaysEnv) && syncDaysEnv > 0
|
||||
? syncDaysEnv
|
||||
: SYNC_DAYS_DEFAULT;
|
||||
const syncDays = Math.min(rawSyncDays, SYNC_DAYS_MAX);
|
||||
const syncDays = Math.min(rawSyncDays, SYNC_DAYS_EFFECTIVE);
|
||||
|
||||
const intervalDb = parseFloat(getSetting('simplefin_sync_interval_hours') || '');
|
||||
const intervalEnv = parseFloat(process.env.SIMPLEFIN_SYNC_INTERVAL_HOURS || '');
|
||||
|
|
|
|||
|
|
@ -18,6 +18,9 @@ const MONTHLY_FACTORS = {
|
|||
irregular: 1,
|
||||
};
|
||||
|
||||
// Transactions that are clearly not subscriptions — skip before grouping
|
||||
const SKIP_MERCHANT_RE = /\b(atm|withdrawal|transfer|deposit|zelle|venmo|wire|refund|rebate|interest charge)\b/;
|
||||
|
||||
// Fallback keyword list used when catalog lookup finds no match
|
||||
const TYPE_KEYWORDS = [
|
||||
['streaming', ['netflix', 'hulu', 'disney', 'max', 'paramount', 'peacock', 'youtube tv', 'sling', 'espn', 'fubo', 'starz', 'crunchyroll', 'dazn']],
|
||||
|
|
@ -46,7 +49,12 @@ function loadCatalog(db) {
|
|||
}
|
||||
|
||||
function normalizeCatalogName(value) {
|
||||
return String(value || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
|
||||
return String(value || '')
|
||||
.toLowerCase()
|
||||
.replace(/\+/g, ' plus ') // "Walmart+" → "walmart plus" so it only matches "walmart plus" transactions
|
||||
.replace(/[^a-z0-9]+/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// Given a normalized merchant string, find the best matching catalog entry.
|
||||
|
|
@ -75,6 +83,7 @@ function lookupCatalog(catalog, merchantText) {
|
|||
function normalizeMerchant(value) {
|
||||
return String(value || '')
|
||||
.toLowerCase()
|
||||
.replace(/\+/g, ' plus ') // preserve "+" so "WALMART+" matches catalog "Walmart+" → "walmart plus"
|
||||
.replace(/[^a-z0-9\s]/g, ' ')
|
||||
.replace(/\b(pos|debit|card|payment|purchase|recurring|online|inc|llc|co|www)\b/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
|
|
@ -188,13 +197,32 @@ function billingCycleForCycleType(cycleType) {
|
|||
return 'irregular';
|
||||
}
|
||||
|
||||
// ── Decline store ─────────────────────────────────────────────────────────────
|
||||
|
||||
function getDeclinedKeys(db, userId) {
|
||||
try {
|
||||
const rows = db.prepare('SELECT decline_key FROM declined_subscription_hints WHERE user_id = ?').all(userId);
|
||||
return new Set(rows.map(r => r.decline_key));
|
||||
} catch {
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
|
||||
function declineRecommendation(db, userId, declineKey) {
|
||||
db.prepare(`
|
||||
INSERT INTO declined_subscription_hints (user_id, decline_key)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT(user_id, decline_key) DO NOTHING
|
||||
`).run(userId, declineKey);
|
||||
}
|
||||
|
||||
// ── Recommendations ───────────────────────────────────────────────────────────
|
||||
|
||||
function getSubscriptionRecommendations(db, userId) {
|
||||
const catalog = loadCatalog(db);
|
||||
const existingNames = existingBillNames(db, userId);
|
||||
const declined = getDeclinedKeys(db, userId);
|
||||
|
||||
// Scan all transaction sources, not just SimpleFIN
|
||||
const rows = db.prepare(`
|
||||
SELECT
|
||||
t.id, t.amount, t.currency, t.description, t.payee, t.memo, t.category,
|
||||
|
|
@ -211,16 +239,20 @@ function getSubscriptionRecommendations(db, userId) {
|
|||
ORDER BY tx_date ASC
|
||||
`).all(userId);
|
||||
|
||||
// Group by merchant + amount bucket
|
||||
// Group by merchant + amount bucket — consistent amounts are the foundation of
|
||||
// subscription detection. Catalog lookup names the service and boosts confidence
|
||||
// but does not change the grouping; deduplication at the end ensures one entry
|
||||
// per known service.
|
||||
const groups = new Map();
|
||||
for (const tx of rows) {
|
||||
const merchant = normalizeMerchant(tx.payee || tx.description || tx.memo);
|
||||
if (!merchant || merchant.length < 3) continue;
|
||||
if (SKIP_MERCHANT_RE.test(merchant)) continue;
|
||||
const amount = dollarsFromTransactionAmount(tx.amount);
|
||||
if (amount < 1) continue;
|
||||
const key = `${merchant}:${Math.round(amount)}`;
|
||||
if (!groups.has(key)) {
|
||||
groups.set(key, { merchant, amountBucket: Math.round(amount), items: [], catalogEntry: null });
|
||||
groups.set(key, { merchant, items: [], catalogEntry: null });
|
||||
}
|
||||
const group = groups.get(key);
|
||||
group.items.push({ ...tx, amount_dollars: amount });
|
||||
|
|
@ -231,14 +263,14 @@ function getSubscriptionRecommendations(db, userId) {
|
|||
|
||||
for (const group of groups.values()) {
|
||||
const { merchant, catalogEntry } = group;
|
||||
const declineKey = catalogEntry ? `catalog:${catalogEntry.id}` : `merchant:${merchant}`;
|
||||
|
||||
// Skip if already a known bill
|
||||
if (existingNames.some(name => name.includes(merchant) || merchant.includes(name))) continue;
|
||||
if (declined.has(declineKey)) continue;
|
||||
if (existingNames.some(n => n.includes(merchant) || merchant.includes(n))) continue;
|
||||
|
||||
const sorted = group.items
|
||||
.filter(item => item.tx_date)
|
||||
.sort((a, b) => String(a.tx_date).localeCompare(String(b.tx_date)));
|
||||
|
||||
if (sorted.length === 0) continue;
|
||||
|
||||
const averageAmount = sorted.reduce((sum, item) => sum + item.amount_dollars, 0) / sorted.length;
|
||||
|
|
@ -247,17 +279,15 @@ function getSubscriptionRecommendations(db, userId) {
|
|||
: 0;
|
||||
const last = sorted[sorted.length - 1];
|
||||
|
||||
// ── Tier 1: catalog match with 1 occurrence (possible subscription) ──────
|
||||
// Tier 1: catalog match with 1 occurrence
|
||||
if (catalogEntry && sorted.length === 1) {
|
||||
const confidence = 62;
|
||||
recommendations.push(buildRecommendation({
|
||||
merchant, catalogEntry, sorted, averageAmount, maxDelta, last,
|
||||
cycleType: 'monthly', avgGap: 30, confidence, tier: 'possible',
|
||||
cycleType: 'monthly', avgGap: 30, confidence: 62, tier: 'possible', declineKey,
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
|
||||
// ── Tier 2: 2+ occurrences — pattern detection ────────────────────────────
|
||||
if (sorted.length < 2) continue;
|
||||
|
||||
const gaps = [];
|
||||
|
|
@ -275,9 +305,9 @@ function getSubscriptionRecommendations(db, userId) {
|
|||
|
||||
if (cycleType === 'monthly' && (avgGap < 24 || avgGap > 38)) continue;
|
||||
if (cycleType === 'quarterly' && (avgGap < 75 || avgGap > 105)) continue;
|
||||
if (cycleType === 'weekly') continue;
|
||||
if (maxDelta > Math.max(3, averageAmount * 0.18)) continue;
|
||||
|
||||
// Confidence: catalog match raises the floor and ceiling
|
||||
let confidence;
|
||||
if (catalogEntry) {
|
||||
confidence = Math.min(99, 68 + sorted.length * 8 + (maxDelta <= 1 ? 8 : 0));
|
||||
|
|
@ -288,16 +318,25 @@ function getSubscriptionRecommendations(db, userId) {
|
|||
const tier = catalogEntry ? 'confirmed' : 'pattern';
|
||||
recommendations.push(buildRecommendation({
|
||||
merchant, catalogEntry, sorted, averageAmount, maxDelta, last,
|
||||
cycleType, avgGap, confidence, tier,
|
||||
cycleType, avgGap, confidence, tier, declineKey,
|
||||
}));
|
||||
}
|
||||
|
||||
return recommendations
|
||||
.sort((a, b) => b.confidence - a.confidence || b.occurrence_count - a.occurrence_count)
|
||||
.slice(0, 20);
|
||||
// Deduplicate by catalog entry — if multiple amount buckets matched the same
|
||||
// known service, keep only the highest-confidence one.
|
||||
const seen = new Map();
|
||||
const deduped = [];
|
||||
for (const rec of recommendations.sort((a, b) => b.confidence - a.confidence || b.occurrence_count - a.occurrence_count)) {
|
||||
const key = rec.catalog_match ? `catalog:${rec.catalog_match.id}` : `merchant:${rec.merchant}`;
|
||||
if (!seen.has(key)) {
|
||||
seen.set(key, true);
|
||||
deduped.push(rec);
|
||||
}
|
||||
}
|
||||
return deduped.slice(0, 20);
|
||||
}
|
||||
|
||||
function buildRecommendation({ merchant, catalogEntry, sorted, averageAmount, maxDelta, last, cycleType, avgGap, confidence, tier }) {
|
||||
function buildRecommendation({ merchant, catalogEntry, sorted, averageAmount, maxDelta, last, cycleType, avgGap, confidence, tier, declineKey }) {
|
||||
const name = catalogEntry ? catalogEntry.name : titleCase(merchant);
|
||||
const subscriptionType = inferType(merchant, catalogEntry);
|
||||
|
||||
|
|
@ -323,6 +362,7 @@ function buildRecommendation({ merchant, catalogEntry, sorted, averageAmount, ma
|
|||
catalog_match: catalogEntry ? { id: catalogEntry.id, name: catalogEntry.name, category: catalogEntry.category } : null,
|
||||
transaction_ids: sorted.map(item => item.id),
|
||||
merchant,
|
||||
decline_key: declineKey,
|
||||
source: last.data_source_name || 'Transaction history',
|
||||
reasons,
|
||||
};
|
||||
|
|
@ -379,6 +419,7 @@ function createSubscriptionFromRecommendation(db, userId, payload = {}) {
|
|||
module.exports = {
|
||||
SUBSCRIPTION_TYPES,
|
||||
createSubscriptionFromRecommendation,
|
||||
declineRecommendation,
|
||||
decorateSubscription,
|
||||
getSubscriptionRecommendations,
|
||||
getSubscriptionSummary,
|
||||
|
|
|
|||
Loading…
Reference in New Issue