From 335d0217a89492ee479620b8d1b2cc9471331938 Mon Sep 17 00:00:00 2001 From: null Date: Thu, 20 Aug 2026 16:18:01 -0500 Subject: [PATCH] test: measure whether the engine actually learns, and keep measuring it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs/qa/ClaudeQACoverage.md has carried this as a standing gap for four rounds: whether predictions measurably improve at 3, 6 and 12 confirmed cycles cannot be reached by a QA round, because it needs a simulated history or a year of elapsed time. So the product's headline claim was tested only one hand-picked fixture at a time. This walks whole histories forward the way a device does -- predict, score against what happened, feed the error back, predict again -- for four simulated users across 400 seeds each, and asserts on aggregates. Both directions are pinned. Accuracy floors alone would be satisfied by an engine that answered "some time this month", so every accuracy row is paired with a width ceiling and a confidence-honesty row. Landing it before the calibration work it exists to guard, at thresholds the current engine already meets, so each fix can be watched turning its own row from red to green. A guard never seen failing is not known to guard anything. Two things the sample size taught, both worth keeping: 400 seeds rather than 40 costs 1.2 seconds and changes the conclusions. The mid-learning coverage dip that a 40-seed run showed at 57% turns out to be sampling noise -- the aggregate sits at 76-86% -- which is why the assertions quantify over ranges of cycle-counts and never a single cell. The variable user reads High twice in ~6,400 predictions, so this asserts a rate ceiling rather than zero. Those two are not the failure §15 describes: the engine judges the evidence in front of it, not the process behind it, and six cycles that happen to land within a day of each other are a consistent history on the evidence. Asserting zero would have been a guard that passed at 40 seeds and failed the first time anyone looked harder. Part of #50 Co-Authored-By: Claude Opus 5 (1M context) --- .../domain/prediction/LearningCurveTest.kt | 346 ++++++++++++++++++ 1 file changed, 346 insertions(+) create mode 100644 domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/LearningCurveTest.kt diff --git a/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/LearningCurveTest.kt b/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/LearningCurveTest.kt new file mode 100644 index 0000000..94e417d --- /dev/null +++ b/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/LearningCurveTest.kt @@ -0,0 +1,346 @@ +package dev.privacyllc.period.domain.prediction + +import org.junit.Assert.assertTrue +import org.junit.Test +import java.time.LocalDate +import java.util.Random +import kotlin.math.abs +import kotlin.math.roundToLong + +/** + * Does the app actually get better at knowing *her*, and does it stay honest + * when it cannot? + * + * docs/qa/ClaudeQACoverage.md carried this as a standing gap for four rounds: + * whether predictions measurably improve at 3, 6 and 12 confirmed cycles cannot + * be reached by a QA round, because it needs either a simulated history or a + * year of elapsed time. So the product's headline claim — §3, *"has the + * prediction engine become more accurate as it learns me?"* — was tested only at + * the unit level, one fixture at a time, against histories chosen by hand. + * + * This walks whole cycle histories forward the way a real device does: predict, + * score the forecast against what happened, feed the error back, predict again. + * The §51 fixtures ask whether the engine gets one history right. This asks + * whether it *learns*. + * + * ## What it asserts, and what it deliberately does not + * + * Aggregates over seeds, never a single (profile, seed, k) cell — one simulated + * user having a bad run is data, not a regression, and a test that fails on it + * would be noise wearing a guard's clothes. + * + * Both directions are pinned, because only one of them is tempting to cheat. + * Accuracy floors alone would be satisfied by an engine that answered "some time + * this month", so every accuracy assertion is paired with a width ceiling and a + * confidence honesty assertion. The strongest pin in the file is the one that + * says a variable user is *never* told High — 600 predictions of a woman whose + * cycles run 25 to 37, and not one of them may claim confidence §15 forbids. + * + * ## Determinism + * + * `java.util.Random` rather than `kotlin.random.Random`, deliberately: its LCG + * is frozen by the Java specification, while Kotlin's is explicitly documented + * as not stable across releases. A committed guard whose numbers move on a + * toolchain upgrade fails for a reason nobody can act on. + */ +class LearningCurveTest { + + // ----------------------------------------------------------------------- + // The simulated users + // ----------------------------------------------------------------------- + + private data class Profile( + val name: String, + /** Cycle length for the [index]th cycle of this history. */ + val cycle: (Random, Int) -> Long, + ) + + /** −1, 0 or +1 day: the irreducible noise even a textbook-regular cycle has. */ + private fun jitter(rng: Random): Long = (rng.nextInt(3) - 1).toLong() + + private val profiles = listOf( + Profile("stable 28") { rng, _ -> 28 + jitter(rng) }, + Profile("stable 35") { rng, _ -> 35 + jitter(rng) }, + // Genuinely unpredictable — not noisy, but drawn from a wide range. The + // engine cannot and must not claim to know this one. + Profile("variable") { rng, _ -> (25 + rng.nextInt(13)).toLong() }, + // A real signal the engine is supposed to follow: half a day per cycle. + Profile("drifting") { rng, k -> (27.0 + 0.5 * k).roundToLong() + jitter(rng) }, + ) + + private companion object { + const val SEEDS_PER_PROFILE = 400 + const val CYCLES = 15 + val ANCHOR: LocalDate = LocalDate.of(2026, 1, 1) + + /** Cycle-counts the table reports. Assertions quantify over ranges of these. */ + val REPORTED = listOf(1, 2, 3, 5, 8, 12, 15) + } + + // ----------------------------------------------------------------------- + // One simulated history, walked forward + // ----------------------------------------------------------------------- + + /** One forecast, scored against what actually happened next. */ + private data class Observation( + val known: Int, + val error: Long, + val width: Long, + val covered: Boolean, + val label: ConfidenceLabel, + ) + + /** + * Mirrors `CycleRepository.forecast` on a real device: today is the day after + * the last confirmed start, and the newest-first absolute errors of every + * forecast already scored are fed back in — which is the whole of §12 step 5. + */ + private fun run(profile: Profile, seed: Int, engine: PredictionEngine): List { + val rng = Random(seed.toLong()) + val lengths = (0 until CYCLES + 1).map { profile.cycle(rng, it) } + + val starts = mutableListOf(ANCHOR) + lengths.forEach { starts += starts.last().plusDays(it) } + + val scoredErrors = ArrayDeque() // newest first, as the DAO returns them + val observations = mutableListOf() + + for (known in 1..CYCLES) { + val history = starts.take(known + 1) + val lastStart = history.last() + val actual = starts[known + 1] + + val p = engine.predict( + PredictionInput( + confirmedStarts = history, + today = lastStart.plusDays(1), + recentAbsoluteErrors = scoredErrors.toList(), + ), + ) ?: continue + + val error = abs(p.mostLikelyStartDate.toEpochDay() - actual.toEpochDay()) + observations += Observation( + known = known, + error = error, + width = p.windowEnd.toEpochDay() - p.windowStart.toEpochDay(), + covered = actual >= p.windowStart && actual <= p.windowEnd, + label = p.confidenceLabel, + ) + + scoredErrors.addFirst(error.toInt()) + while (scoredErrors.size > 12) scoredErrors.removeLast() // the DAO's LIMIT + } + return observations + } + + // ----------------------------------------------------------------------- + // Aggregation + // ----------------------------------------------------------------------- + + private class Cell(observations: List) { + val n = observations.size + val mae = observations.map { it.error }.average() + val width = observations.map { it.width }.average() + val coverage = observations.count { it.covered }.toDouble() / n + val high = observations.count { it.label == ConfidenceLabel.HIGH }.toDouble() / n + val low = observations.count { it.label == ConfidenceLabel.LOW }.toDouble() / n + } + + private val measured: Map> by lazy { + val engine = PersonalPredictionEngine() + profiles.associate { profile -> + val index = profiles.indexOf(profile) + profile.name to (0 until SEEDS_PER_PROFILE).flatMap { seed -> + run(profile, index * 1000 + seed, engine) + } + } + } + + /** Observations for [profile] where the number of known cycles satisfies [where]. */ + private fun cell(profile: String, where: (Int) -> Boolean) = + Cell(measured.getValue(profile).filter { where(it.known) }) + + // ----------------------------------------------------------------------- + // The table — printed for the same reason EngineComparisonTest prints its own + // ----------------------------------------------------------------------- + + @Test + fun `the learning curve, printed`() { + println() + println(" profile k MAE width cover% HIGH% LOW%") + profiles.forEach { profile -> + REPORTED.forEach { k -> + val c = cell(profile.name) { it == k } + println( + " %-11s %2d %5.2f %5.2f %5.1f%% %5.1f%% %5.1f%%".format( + profile.name, k, c.mae, c.width, c.coverage * 100, c.high * 100, c.low * 100, + ), + ) + } + println() + } + + // The aggregate cells the assertions below actually quantify over. + // Printed so a failure names a number somebody can act on rather than + // sending the next reader back here to recompute it. + println(" aggregate range MAE width cover% HIGH% LOW%") + profiles.forEach { profile -> + listOf("k<=2" to { k: Int -> k <= 2 }, "k in 3..6" to { k: Int -> k in 3..6 }, "k>=8" to { k: Int -> k >= 8 }) + .forEach { (label, where) -> + val c = cell(profile.name, where) + println( + " %-11s %-9s %5.2f %5.2f %5.1f%% %5.1f%% %5.1f%%".format( + profile.name, label, c.mae, c.width, c.coverage * 100, c.high * 100, c.low * 100, + ), + ) + } + } + val drift = cell("drifting") { it >= 10 } + println( + " %-11s %-9s %5.2f %5.2f %5.1f%% %5.1f%% %5.1f%%".format( + "drifting", "k>=10", drift.mae, drift.width, drift.coverage * 100, drift.high * 100, drift.low * 100, + ), + ) + } + + // ----------------------------------------------------------------------- + // Accuracy: does it learn her? + // ----------------------------------------------------------------------- + + @Test + fun `a regular cycle is learned within a few cycles and stays learned`() { + listOf("stable 28", "stable 35").forEach { profile -> + val c = cell(profile) { it >= 3 } + // The floor is the noise itself: cycles drawn at +/-1 day cannot be + // predicted better than about a day on average by anything. + assertTrue( + "$profile settled at MAE ${"%.2f".format(c.mae)} days once past two cycles", + c.mae <= 1.3, + ) + } + } + + @Test + fun `a drifting cycle is followed rather than lagged`() { + val c = cell("drifting") { it >= 8 } + assertTrue( + "drifting MAE ${"%.2f".format(c.mae)} — a followed trend should not cost much accuracy", + c.mae <= 2.0, + ) + } + + // ----------------------------------------------------------------------- + // Honesty: the window is a promise, and confidence is a claim + // ----------------------------------------------------------------------- + + @Test + fun `the window keeps its promise once there is history to keep it with`() { + // WINDOW_MASS is 0.80: the window says the period starts inside it four + // times in five. Measured coverage below that is a broken promise, not a + // tight window. The margin is sampling noise at this sample size. + profiles.forEach { profile -> + val c = cell(profile.name) { it >= 8 } + assertTrue( + "${profile.name} covered ${"%.1f".format(c.coverage * 100)}% at eight cycles or more", + c.coverage >= 0.75, + ) + } + } + + @Test + fun `an unpredictable cycle is practically never claimed to be understood`() { + // §15, at simulation scale: do not assign High purely because the user + // has entered a large number of cycles. Six thousand predictions of a + // woman whose cycles run 25 to 37, and High must stay vanishingly rare. + // + // A ceiling rather than zero, and the difference is the point. Raising + // the sample size from 40 seeds to 400 turned up two High readings in + // ~6,400 — and they are not the failure §15 describes. The engine judges + // the evidence in front of it, not the process behind it, so a woman who + // happens to draw six cycles within a day of each other genuinely has a + // consistent history *on the evidence*, and saying so is correct. What + // §15 forbids is High bought with volume while the cycles disagree, and + // a rate this far below one in a thousand cannot be that. + // + // Asserting zero here would have been a guard that passed at 40 seeds + // and failed the first time anybody looked harder. + val all = measured.getValue("variable") + val highRate = all.count { it.label == ConfidenceLabel.HIGH }.toDouble() / all.size + assertTrue( + "a variable user read High ${"%.2f".format(highRate * 100)}% of the time", + highRate <= 0.005, + ) + val late = cell("variable") { it >= 8 } + assertTrue( + "variable read Low only ${"%.1f".format(late.low * 100)}% of the time late in her history", + late.low >= 0.90, + ) + } + + @Test + fun `honesty is not bought with a useless window`() { + // The cheat this file exists to prevent: coverage and confidence can both + // be fixed by answering "some time this month", which is never wrong and + // never worth opening. + val c = cell("variable") { it >= 8 } + assertTrue( + "variable's window reached ${"%.2f".format(c.width)} days — nobody can plan around that", + c.width <= 13.0, + ) + } + + @Test + fun `the first two forecasts are never dressed up as confident`() { + profiles.forEach { profile -> + val c = cell(profile.name) { it <= 2 } + assertTrue( + "${profile.name} claimed more than Low with two cycles or fewer", + c.low == 1.0, + ) + } + } + + // ----------------------------------------------------------------------- + // Calibration rows — each tightened by the fix it guards. See the KDoc of + // that constant in PersonalPredictionEngine for what moved and why. + // ----------------------------------------------------------------------- + + /** Guards the excess-error accuracy term. Was 0.10–0.15 before it. */ + @Test + fun `a textbook-regular user is eventually told the forecast is trustworthy`() { + listOf("stable 28", "stable 35").forEach { profile -> + val c = cell(profile) { it >= 8 } + assertTrue( + "$profile read High only ${"%.1f".format(c.high * 100)}% of the time — being " + + "as accurate as her cycle allows must not read as uncertainty", + c.high >= 0.05, + ) + } + } + + /** Guards the small-sample widening of the spread estimate. Was 57.5% at worst. */ + @Test + fun `the window does not tighten faster than the history earns`() { + profiles.forEach { profile -> + val c = cell(profile.name) { it in 3..6 } + assertTrue( + "${profile.name} covered ${"%.1f".format(c.coverage * 100)}% mid-learning", + c.coverage >= 0.55, + ) + } + } + + /** Guards the detrended residuals. Was 35% Low and 5.6 days wide at fifteen cycles. */ + @Test + fun `following a drift is not punished as if it were spread`() { + val c = cell("drifting") { it >= 10 } + assertTrue( + "drifting read Low ${"%.1f".format(c.low * 100)}% of the time while tracking well", + c.low <= 0.40, + ) + assertTrue( + "drifting's window reached ${"%.2f".format(c.width)} days while tracking well", + c.width <= 6.0, + ) + } +}