diff --git a/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/LearningCurveTest.kt b/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/LearningCurveTest.kt new file mode 100644 index 0000000..94e417d --- /dev/null +++ b/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/LearningCurveTest.kt @@ -0,0 +1,346 @@ +package dev.privacyllc.period.domain.prediction + +import org.junit.Assert.assertTrue +import org.junit.Test +import java.time.LocalDate +import java.util.Random +import kotlin.math.abs +import kotlin.math.roundToLong + +/** + * Does the app actually get better at knowing *her*, and does it stay honest + * when it cannot? + * + * docs/qa/ClaudeQACoverage.md carried this as a standing gap for four rounds: + * whether predictions measurably improve at 3, 6 and 12 confirmed cycles cannot + * be reached by a QA round, because it needs either a simulated history or a + * year of elapsed time. So the product's headline claim — §3, *"has the + * prediction engine become more accurate as it learns me?"* — was tested only at + * the unit level, one fixture at a time, against histories chosen by hand. + * + * This walks whole cycle histories forward the way a real device does: predict, + * score the forecast against what happened, feed the error back, predict again. + * The §51 fixtures ask whether the engine gets one history right. This asks + * whether it *learns*. + * + * ## What it asserts, and what it deliberately does not + * + * Aggregates over seeds, never a single (profile, seed, k) cell — one simulated + * user having a bad run is data, not a regression, and a test that fails on it + * would be noise wearing a guard's clothes. + * + * Both directions are pinned, because only one of them is tempting to cheat. + * Accuracy floors alone would be satisfied by an engine that answered "some time + * this month", so every accuracy assertion is paired with a width ceiling and a + * confidence honesty assertion. The strongest pin in the file is the one that + * says a variable user is *never* told High — 600 predictions of a woman whose + * cycles run 25 to 37, and not one of them may claim confidence §15 forbids. + * + * ## Determinism + * + * `java.util.Random` rather than `kotlin.random.Random`, deliberately: its LCG + * is frozen by the Java specification, while Kotlin's is explicitly documented + * as not stable across releases. A committed guard whose numbers move on a + * toolchain upgrade fails for a reason nobody can act on. + */ +class LearningCurveTest { + + // ----------------------------------------------------------------------- + // The simulated users + // ----------------------------------------------------------------------- + + private data class Profile( + val name: String, + /** Cycle length for the [index]th cycle of this history. */ + val cycle: (Random, Int) -> Long, + ) + + /** −1, 0 or +1 day: the irreducible noise even a textbook-regular cycle has. */ + private fun jitter(rng: Random): Long = (rng.nextInt(3) - 1).toLong() + + private val profiles = listOf( + Profile("stable 28") { rng, _ -> 28 + jitter(rng) }, + Profile("stable 35") { rng, _ -> 35 + jitter(rng) }, + // Genuinely unpredictable — not noisy, but drawn from a wide range. The + // engine cannot and must not claim to know this one. + Profile("variable") { rng, _ -> (25 + rng.nextInt(13)).toLong() }, + // A real signal the engine is supposed to follow: half a day per cycle. + Profile("drifting") { rng, k -> (27.0 + 0.5 * k).roundToLong() + jitter(rng) }, + ) + + private companion object { + const val SEEDS_PER_PROFILE = 400 + const val CYCLES = 15 + val ANCHOR: LocalDate = LocalDate.of(2026, 1, 1) + + /** Cycle-counts the table reports. Assertions quantify over ranges of these. */ + val REPORTED = listOf(1, 2, 3, 5, 8, 12, 15) + } + + // ----------------------------------------------------------------------- + // One simulated history, walked forward + // ----------------------------------------------------------------------- + + /** One forecast, scored against what actually happened next. */ + private data class Observation( + val known: Int, + val error: Long, + val width: Long, + val covered: Boolean, + val label: ConfidenceLabel, + ) + + /** + * Mirrors `CycleRepository.forecast` on a real device: today is the day after + * the last confirmed start, and the newest-first absolute errors of every + * forecast already scored are fed back in — which is the whole of §12 step 5. + */ + private fun run(profile: Profile, seed: Int, engine: PredictionEngine): List { + val rng = Random(seed.toLong()) + val lengths = (0 until CYCLES + 1).map { profile.cycle(rng, it) } + + val starts = mutableListOf(ANCHOR) + lengths.forEach { starts += starts.last().plusDays(it) } + + val scoredErrors = ArrayDeque() // newest first, as the DAO returns them + val observations = mutableListOf() + + for (known in 1..CYCLES) { + val history = starts.take(known + 1) + val lastStart = history.last() + val actual = starts[known + 1] + + val p = engine.predict( + PredictionInput( + confirmedStarts = history, + today = lastStart.plusDays(1), + recentAbsoluteErrors = scoredErrors.toList(), + ), + ) ?: continue + + val error = abs(p.mostLikelyStartDate.toEpochDay() - actual.toEpochDay()) + observations += Observation( + known = known, + error = error, + width = p.windowEnd.toEpochDay() - p.windowStart.toEpochDay(), + covered = actual >= p.windowStart && actual <= p.windowEnd, + label = p.confidenceLabel, + ) + + scoredErrors.addFirst(error.toInt()) + while (scoredErrors.size > 12) scoredErrors.removeLast() // the DAO's LIMIT + } + return observations + } + + // ----------------------------------------------------------------------- + // Aggregation + // ----------------------------------------------------------------------- + + private class Cell(observations: List) { + val n = observations.size + val mae = observations.map { it.error }.average() + val width = observations.map { it.width }.average() + val coverage = observations.count { it.covered }.toDouble() / n + val high = observations.count { it.label == ConfidenceLabel.HIGH }.toDouble() / n + val low = observations.count { it.label == ConfidenceLabel.LOW }.toDouble() / n + } + + private val measured: Map> by lazy { + val engine = PersonalPredictionEngine() + profiles.associate { profile -> + val index = profiles.indexOf(profile) + profile.name to (0 until SEEDS_PER_PROFILE).flatMap { seed -> + run(profile, index * 1000 + seed, engine) + } + } + } + + /** Observations for [profile] where the number of known cycles satisfies [where]. */ + private fun cell(profile: String, where: (Int) -> Boolean) = + Cell(measured.getValue(profile).filter { where(it.known) }) + + // ----------------------------------------------------------------------- + // The table — printed for the same reason EngineComparisonTest prints its own + // ----------------------------------------------------------------------- + + @Test + fun `the learning curve, printed`() { + println() + println(" profile k MAE width cover% HIGH% LOW%") + profiles.forEach { profile -> + REPORTED.forEach { k -> + val c = cell(profile.name) { it == k } + println( + " %-11s %2d %5.2f %5.2f %5.1f%% %5.1f%% %5.1f%%".format( + profile.name, k, c.mae, c.width, c.coverage * 100, c.high * 100, c.low * 100, + ), + ) + } + println() + } + + // The aggregate cells the assertions below actually quantify over. + // Printed so a failure names a number somebody can act on rather than + // sending the next reader back here to recompute it. + println(" aggregate range MAE width cover% HIGH% LOW%") + profiles.forEach { profile -> + listOf("k<=2" to { k: Int -> k <= 2 }, "k in 3..6" to { k: Int -> k in 3..6 }, "k>=8" to { k: Int -> k >= 8 }) + .forEach { (label, where) -> + val c = cell(profile.name, where) + println( + " %-11s %-9s %5.2f %5.2f %5.1f%% %5.1f%% %5.1f%%".format( + profile.name, label, c.mae, c.width, c.coverage * 100, c.high * 100, c.low * 100, + ), + ) + } + } + val drift = cell("drifting") { it >= 10 } + println( + " %-11s %-9s %5.2f %5.2f %5.1f%% %5.1f%% %5.1f%%".format( + "drifting", "k>=10", drift.mae, drift.width, drift.coverage * 100, drift.high * 100, drift.low * 100, + ), + ) + } + + // ----------------------------------------------------------------------- + // Accuracy: does it learn her? + // ----------------------------------------------------------------------- + + @Test + fun `a regular cycle is learned within a few cycles and stays learned`() { + listOf("stable 28", "stable 35").forEach { profile -> + val c = cell(profile) { it >= 3 } + // The floor is the noise itself: cycles drawn at +/-1 day cannot be + // predicted better than about a day on average by anything. + assertTrue( + "$profile settled at MAE ${"%.2f".format(c.mae)} days once past two cycles", + c.mae <= 1.3, + ) + } + } + + @Test + fun `a drifting cycle is followed rather than lagged`() { + val c = cell("drifting") { it >= 8 } + assertTrue( + "drifting MAE ${"%.2f".format(c.mae)} — a followed trend should not cost much accuracy", + c.mae <= 2.0, + ) + } + + // ----------------------------------------------------------------------- + // Honesty: the window is a promise, and confidence is a claim + // ----------------------------------------------------------------------- + + @Test + fun `the window keeps its promise once there is history to keep it with`() { + // WINDOW_MASS is 0.80: the window says the period starts inside it four + // times in five. Measured coverage below that is a broken promise, not a + // tight window. The margin is sampling noise at this sample size. + profiles.forEach { profile -> + val c = cell(profile.name) { it >= 8 } + assertTrue( + "${profile.name} covered ${"%.1f".format(c.coverage * 100)}% at eight cycles or more", + c.coverage >= 0.75, + ) + } + } + + @Test + fun `an unpredictable cycle is practically never claimed to be understood`() { + // §15, at simulation scale: do not assign High purely because the user + // has entered a large number of cycles. Six thousand predictions of a + // woman whose cycles run 25 to 37, and High must stay vanishingly rare. + // + // A ceiling rather than zero, and the difference is the point. Raising + // the sample size from 40 seeds to 400 turned up two High readings in + // ~6,400 — and they are not the failure §15 describes. The engine judges + // the evidence in front of it, not the process behind it, so a woman who + // happens to draw six cycles within a day of each other genuinely has a + // consistent history *on the evidence*, and saying so is correct. What + // §15 forbids is High bought with volume while the cycles disagree, and + // a rate this far below one in a thousand cannot be that. + // + // Asserting zero here would have been a guard that passed at 40 seeds + // and failed the first time anybody looked harder. + val all = measured.getValue("variable") + val highRate = all.count { it.label == ConfidenceLabel.HIGH }.toDouble() / all.size + assertTrue( + "a variable user read High ${"%.2f".format(highRate * 100)}% of the time", + highRate <= 0.005, + ) + val late = cell("variable") { it >= 8 } + assertTrue( + "variable read Low only ${"%.1f".format(late.low * 100)}% of the time late in her history", + late.low >= 0.90, + ) + } + + @Test + fun `honesty is not bought with a useless window`() { + // The cheat this file exists to prevent: coverage and confidence can both + // be fixed by answering "some time this month", which is never wrong and + // never worth opening. + val c = cell("variable") { it >= 8 } + assertTrue( + "variable's window reached ${"%.2f".format(c.width)} days — nobody can plan around that", + c.width <= 13.0, + ) + } + + @Test + fun `the first two forecasts are never dressed up as confident`() { + profiles.forEach { profile -> + val c = cell(profile.name) { it <= 2 } + assertTrue( + "${profile.name} claimed more than Low with two cycles or fewer", + c.low == 1.0, + ) + } + } + + // ----------------------------------------------------------------------- + // Calibration rows — each tightened by the fix it guards. See the KDoc of + // that constant in PersonalPredictionEngine for what moved and why. + // ----------------------------------------------------------------------- + + /** Guards the excess-error accuracy term. Was 0.10–0.15 before it. */ + @Test + fun `a textbook-regular user is eventually told the forecast is trustworthy`() { + listOf("stable 28", "stable 35").forEach { profile -> + val c = cell(profile) { it >= 8 } + assertTrue( + "$profile read High only ${"%.1f".format(c.high * 100)}% of the time — being " + + "as accurate as her cycle allows must not read as uncertainty", + c.high >= 0.05, + ) + } + } + + /** Guards the small-sample widening of the spread estimate. Was 57.5% at worst. */ + @Test + fun `the window does not tighten faster than the history earns`() { + profiles.forEach { profile -> + val c = cell(profile.name) { it in 3..6 } + assertTrue( + "${profile.name} covered ${"%.1f".format(c.coverage * 100)}% mid-learning", + c.coverage >= 0.55, + ) + } + } + + /** Guards the detrended residuals. Was 35% Low and 5.6 days wide at fifteen cycles. */ + @Test + fun `following a drift is not punished as if it were spread`() { + val c = cell("drifting") { it >= 10 } + assertTrue( + "drifting read Low ${"%.1f".format(c.low * 100)}% of the time while tracking well", + c.low <= 0.40, + ) + assertTrue( + "drifting's window reached ${"%.2f".format(c.width)} days while tracking well", + c.width <= 6.0, + ) + } +}