feat: auto-sync worker for SimpleFIN bank sync
New:
services/bankSyncWorker.js — interval-based worker running every 4h (configurable via SIMPLEFIN_SYNC_INTERVAL_HOURS)
- Checks bank sync enabled, fetches oldest-synced sources, skips <1h old
- Staggers syncs 3s apart, writes last_error on failure, timer.unref() for clean shutdown
Modified:
server.js — starts worker inside app.listen callback
routes/admin.js — GET bank-sync-config includes worker status (running, interval, last/next run)
client/components/admin/BankSyncAdminCard.jsx — shows auto-sync worker status panel when enabled
.env.example — SIMPLEFIN_SYNC_INTERVAL_HOURS
This commit is contained in:
parent
858f65b66b
commit
22df64e5e7
|
|
@ -49,6 +49,9 @@ NODE_ENV=production
|
||||||
#
|
#
|
||||||
# How many days back to fetch transactions on first sync (default: 90).
|
# How many days back to fetch transactions on first sync (default: 90).
|
||||||
# SIMPLEFIN_SYNC_DAYS=90
|
# SIMPLEFIN_SYNC_DAYS=90
|
||||||
|
#
|
||||||
|
# How often the background auto-sync worker runs (default: 4 hours, minimum: 0.5).
|
||||||
|
# SIMPLEFIN_SYNC_INTERVAL_HOURS=4
|
||||||
|
|
||||||
# ── First-run admin account ────────────────────────────────────────────────────
|
# ── First-run admin account ────────────────────────────────────────────────────
|
||||||
# Set BOTH on first start to create the admin account automatically.
|
# Set BOTH on first start to create the admin account automatically.
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,24 @@ import { Button } from '@/components/ui/button';
|
||||||
import { Card, CardHeader, CardTitle, CardContent } from '@/components/ui/card';
|
import { Card, CardHeader, CardTitle, CardContent } from '@/components/ui/card';
|
||||||
import { Toggle } from './adminShared';
|
import { Toggle } from './adminShared';
|
||||||
|
|
||||||
|
function timeAgo(iso) {
|
||||||
|
if (!iso) return null;
|
||||||
|
const secs = Math.floor((Date.now() - new Date(iso).getTime()) / 1000);
|
||||||
|
if (secs < 60) return 'just now';
|
||||||
|
if (secs < 3600) return `${Math.floor(secs / 60)}m ago`;
|
||||||
|
if (secs < 86400) return `${Math.floor(secs / 3600)}h ago`;
|
||||||
|
return `${Math.floor(secs / 86400)}d ago`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function timeUntil(iso) {
|
||||||
|
if (!iso) return null;
|
||||||
|
const secs = Math.floor((new Date(iso).getTime() - Date.now()) / 1000);
|
||||||
|
if (secs <= 0) return 'soon';
|
||||||
|
if (secs < 60) return `${secs}s`;
|
||||||
|
if (secs < 3600) return `${Math.floor(secs / 60)}m`;
|
||||||
|
return `${Math.floor(secs / 3600)}h`;
|
||||||
|
}
|
||||||
|
|
||||||
export default function BankSyncAdminCard() {
|
export default function BankSyncAdminCard() {
|
||||||
const [config, setConfig] = useState(null);
|
const [config, setConfig] = useState(null);
|
||||||
const [loading, setLoading] = useState(true);
|
const [loading, setLoading] = useState(true);
|
||||||
|
|
@ -46,6 +64,7 @@ export default function BankSyncAdminCard() {
|
||||||
}
|
}
|
||||||
|
|
||||||
const changed = enabled !== !!config?.enabled;
|
const changed = enabled !== !!config?.enabled;
|
||||||
|
const worker = config?.worker;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Card>
|
<Card>
|
||||||
|
|
@ -74,6 +93,33 @@ export default function BankSyncAdminCard() {
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Auto-sync worker status */}
|
||||||
|
{config?.enabled && worker && (
|
||||||
|
<div className="rounded-lg border border-border/60 bg-muted/20 px-4 py-3 space-y-2">
|
||||||
|
<p className="text-xs font-semibold uppercase tracking-widest text-muted-foreground">Auto-sync worker</p>
|
||||||
|
<div className="grid grid-cols-2 gap-x-6 gap-y-1 text-sm sm:grid-cols-3">
|
||||||
|
<div>
|
||||||
|
<p className="text-xs text-muted-foreground">Status</p>
|
||||||
|
<p className="font-medium flex items-center gap-1.5">
|
||||||
|
<span className={`h-1.5 w-1.5 rounded-full ${worker.running ? 'bg-amber-500 animate-pulse' : 'bg-emerald-500'}`} />
|
||||||
|
{worker.running ? 'Running' : 'Idle'}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<p className="text-xs text-muted-foreground">Last run</p>
|
||||||
|
<p className="font-medium">{worker.last_run_at ? timeAgo(worker.last_run_at) : '—'}</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<p className="text-xs text-muted-foreground">Next run</p>
|
||||||
|
<p className="font-medium">{worker.next_run_at ? `in ${timeUntil(worker.next_run_at)}` : '—'}</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<p className="text-xs text-muted-foreground">
|
||||||
|
Syncs every {worker.interval_hours}h. Set <code className="font-mono">SIMPLEFIN_SYNC_INTERVAL_HOURS</code> to adjust.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<div className="flex justify-end pt-2">
|
<div className="flex justify-end pt-2">
|
||||||
<Button onClick={handleSave} disabled={saving || !changed}>
|
<Button onClick={handleSave} disabled={saving || !changed}>
|
||||||
{saving ? 'Saving…' : 'Save'}
|
{saving ? 'Saving…' : 'Save'}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "bill-tracker",
|
"name": "bill-tracker",
|
||||||
"version": "0.29.2",
|
"version": "0.29.3",
|
||||||
"description": "Monthly bill tracking system",
|
"description": "Monthly bill tracking system",
|
||||||
"main": "server.js",
|
"main": "server.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ const express = require('express');
|
||||||
const router = express.Router();
|
const router = express.Router();
|
||||||
const { getDb, rollbackMigration } = require('../db/database');
|
const { getDb, rollbackMigration } = require('../db/database');
|
||||||
const { getBankSyncConfig, setBankSyncEnabled } = require('../services/bankSyncConfigService');
|
const { getBankSyncConfig, setBankSyncEnabled } = require('../services/bankSyncConfigService');
|
||||||
|
const { getStatus: getBankSyncWorkerStatus } = require('../services/bankSyncWorker');
|
||||||
const { hashPassword } = require('../services/authService');
|
const { hashPassword } = require('../services/authService');
|
||||||
const { logAudit } = require('../services/auditService');
|
const { logAudit } = require('../services/auditService');
|
||||||
const {
|
const {
|
||||||
|
|
@ -401,7 +402,7 @@ router.put('/auth-mode', (req, res) => {
|
||||||
|
|
||||||
// GET /api/admin/bank-sync-config
|
// GET /api/admin/bank-sync-config
|
||||||
router.get('/bank-sync-config', (req, res) => {
|
router.get('/bank-sync-config', (req, res) => {
|
||||||
res.json(getBankSyncConfig());
|
res.json({ ...getBankSyncConfig(), worker: getBankSyncWorkerStatus() });
|
||||||
});
|
});
|
||||||
|
|
||||||
// PUT /api/admin/bank-sync-config
|
// PUT /api/admin/bank-sync-config
|
||||||
|
|
|
||||||
|
|
@ -230,6 +230,13 @@ async function main() {
|
||||||
}, CLEANUP_INTERVAL_MS);
|
}, CLEANUP_INTERVAL_MS);
|
||||||
|
|
||||||
console.log(`[cleanup] Scheduled periodic cleanup every ${CLEANUP_INTERVAL_MS}ms`);
|
console.log(`[cleanup] Scheduled periodic cleanup every ${CLEANUP_INTERVAL_MS}ms`);
|
||||||
|
|
||||||
|
// Start SimpleFIN auto-sync worker (no-op when bank sync is disabled)
|
||||||
|
try {
|
||||||
|
require('./services/bankSyncWorker').start();
|
||||||
|
} catch (err) {
|
||||||
|
console.error('[bankSync] Failed to start auto-sync worker:', err.message);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,128 @@
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
const { getDb } = require('../db/database');
|
||||||
|
const { getBankSyncConfig } = require('./bankSyncConfigService');
|
||||||
|
const { syncDataSource } = require('./bankSyncService');
|
||||||
|
|
||||||
|
const DEFAULT_INTERVAL_HOURS = 4;
|
||||||
|
// Skip a source if it was synced less than this long ago (catches recent manual syncs)
|
||||||
|
const MIN_SYNC_AGE_MS = 60 * 60 * 1000; // 1 hour
|
||||||
|
// Pause between each source to avoid hammering SimpleFIN
|
||||||
|
const STAGGER_DELAY_MS = 3000;
|
||||||
|
|
||||||
|
let timer = null;
|
||||||
|
let running = false;
|
||||||
|
let lastRunAt = null;
|
||||||
|
let nextRunAt = null;
|
||||||
|
|
||||||
|
function intervalMs() {
|
||||||
|
const hours = parseFloat(process.env.SIMPLEFIN_SYNC_INTERVAL_HOURS);
|
||||||
|
return Number.isFinite(hours) && hours >= 0.5
|
||||||
|
? Math.round(hours * 3600000)
|
||||||
|
: DEFAULT_INTERVAL_HOURS * 3600000;
|
||||||
|
}
|
||||||
|
|
||||||
|
function needsSync(source) {
|
||||||
|
if (!source.last_sync_at) return true;
|
||||||
|
const age = Date.now() - new Date(source.last_sync_at).getTime();
|
||||||
|
return age >= MIN_SYNC_AGE_MS;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms) {
|
||||||
|
return new Promise(resolve => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runCycle() {
|
||||||
|
if (running) return;
|
||||||
|
|
||||||
|
const { enabled } = getBankSyncConfig();
|
||||||
|
if (!enabled) return;
|
||||||
|
|
||||||
|
let db, sources;
|
||||||
|
try {
|
||||||
|
db = getDb();
|
||||||
|
sources = db.prepare(`
|
||||||
|
SELECT * FROM data_sources
|
||||||
|
WHERE type = 'provider_sync' AND provider = 'simplefin'
|
||||||
|
ORDER BY last_sync_at ASC
|
||||||
|
`).all();
|
||||||
|
} catch (err) {
|
||||||
|
console.error('[bankSync] Worker failed to load sources:', err.message);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sources.length === 0) return;
|
||||||
|
|
||||||
|
running = true;
|
||||||
|
lastRunAt = new Date().toISOString();
|
||||||
|
|
||||||
|
let synced = 0;
|
||||||
|
let skipped = 0;
|
||||||
|
let failed = 0;
|
||||||
|
|
||||||
|
for (let i = 0; i < sources.length; i++) {
|
||||||
|
const source = sources[i];
|
||||||
|
|
||||||
|
if (!needsSync(source)) {
|
||||||
|
skipped++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await syncDataSource(db, source.user_id, source.id);
|
||||||
|
synced++;
|
||||||
|
} catch {
|
||||||
|
// syncDataSource already writes last_error to the data_sources row
|
||||||
|
failed++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stagger requests — don't fire them all simultaneously
|
||||||
|
if (i < sources.length - 1) await sleep(STAGGER_DELAY_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (synced > 0 || failed > 0) {
|
||||||
|
console.log(`[bankSync] Auto-sync complete: ${synced} synced, ${failed} failed, ${skipped} skipped`);
|
||||||
|
}
|
||||||
|
|
||||||
|
running = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scheduleNext() {
|
||||||
|
const ms = intervalMs();
|
||||||
|
nextRunAt = new Date(Date.now() + ms).toISOString();
|
||||||
|
|
||||||
|
timer = setTimeout(() => {
|
||||||
|
runCycle()
|
||||||
|
.catch(err => {
|
||||||
|
console.error('[bankSync] Worker cycle error:', err.message);
|
||||||
|
running = false;
|
||||||
|
})
|
||||||
|
.finally(scheduleNext);
|
||||||
|
}, ms);
|
||||||
|
|
||||||
|
// Don't hold the event loop open if the server is shutting down
|
||||||
|
if (timer.unref) timer.unref();
|
||||||
|
}
|
||||||
|
|
||||||
|
function start() {
|
||||||
|
if (timer) return;
|
||||||
|
scheduleNext();
|
||||||
|
const hours = intervalMs() / 3600000;
|
||||||
|
console.log(`[bankSync] Auto-sync worker started (interval: ${hours}h)`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function stop() {
|
||||||
|
clearTimeout(timer);
|
||||||
|
timer = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getStatus() {
|
||||||
|
return {
|
||||||
|
running,
|
||||||
|
interval_hours: intervalMs() / 3600000,
|
||||||
|
last_run_at: lastRunAt,
|
||||||
|
next_run_at: nextRunAt,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { start, stop, getStatus };
|
||||||
Loading…
Reference in New Issue