diff --git a/app/src/main/kotlin/dev/privacyllc/period/di/DataModule.kt b/app/src/main/kotlin/dev/privacyllc/period/di/DataModule.kt index e2abb3a..1d82931 100644 --- a/app/src/main/kotlin/dev/privacyllc/period/di/DataModule.kt +++ b/app/src/main/kotlin/dev/privacyllc/period/di/DataModule.kt @@ -13,7 +13,7 @@ import dagger.hilt.components.SingletonComponent import dev.privacyllc.period.core.data.CycleData import dev.privacyllc.period.core.data.CycleRepository import dev.privacyllc.period.core.datastore.UserPreferencesRepository -import dev.privacyllc.period.domain.prediction.BaselinePredictionEngine +import dev.privacyllc.period.domain.prediction.PersonalPredictionEngine import dev.privacyllc.period.domain.prediction.PredictionEngine import java.time.Clock import javax.inject.Singleton @@ -54,16 +54,23 @@ object DataModule { fun userPreferencesRepository(store: DataStore) = UserPreferencesRepository(store) /** - * Batch 02 replaces this binding, and only this binding. + * The engine, and the one line that decides which one the product ships. * - * The rest of the app depends on [PredictionEngine], never on an - * implementation — so swapping in the real engine is one line here, and the - * §51 acceptance tests can run against both to show the replacement is - * better rather than merely different. + * Everything else depends on [PredictionEngine] rather than an + * implementation, which is what made this swap a single line and what makes + * the next one a single line too. + * + * `BaselinePredictionEngine` stays in the tree. It stopped being the product + * and became the control: `EngineComparisonTest` scores both over the §51 + * fixtures every build, so "the new engine is better" is a number rather + * than an opinion. At the swap it was **mean absolute error 0.67 against + * 1.00, and the window contained the actual start 9 times out of 9 against + * 7** — the second mattering more, because a window that misses is a broken + * promise rather than a tight forecast. */ @Provides @Singleton - fun predictionEngine(): PredictionEngine = BaselinePredictionEngine() + fun predictionEngine(): PredictionEngine = PersonalPredictionEngine() /** Injected rather than read from the environment, so §50's date edge cases stay testable. */ @Provides diff --git a/core/data/src/main/kotlin/dev/privacyllc/period/core/data/CycleRepository.kt b/core/data/src/main/kotlin/dev/privacyllc/period/core/data/CycleRepository.kt index 2587e6b..1774411 100644 --- a/core/data/src/main/kotlin/dev/privacyllc/period/core/data/CycleRepository.kt +++ b/core/data/src/main/kotlin/dev/privacyllc/period/core/data/CycleRepository.kt @@ -11,6 +11,7 @@ import dev.privacyllc.period.domain.cycle.toCycles import dev.privacyllc.period.domain.prediction.NotYetObservation import dev.privacyllc.period.domain.prediction.Prediction import dev.privacyllc.period.domain.prediction.PredictionEngine +import dev.privacyllc.period.domain.prediction.PredictionInput import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.map @@ -74,6 +75,10 @@ class CycleRepository internal constructor( val notYetObservations: Flow> = notYetDao.observeAll().map { rows -> rows.map { it.toDomain() } } + /** Absolute errors of the most recent scored forecasts, newest first. */ + private val scoredErrors: Flow> = + predictionDao.observeScored().map { rows -> rows.mapNotNull { it.absoluteErrorDays } } + /** * The live forecast. * @@ -82,11 +87,18 @@ class CycleRepository internal constructor( * than a number it has not earned. */ val forecast: Flow = - combine(confirmedPeriods, notYetObservations) { periods, notYet -> + combine(confirmedPeriods, notYetObservations, scoredErrors) { periods, notYet, errors -> engine.predict( - confirmedStarts = periods.map { it.startDate }, - today = today(), - notYet = notYet, + PredictionInput( + confirmedStarts = periods.map { it.startDate }, + today = today(), + notYet = notYet, + // §12 step 5: the window widens from recent prediction error + // as well as from the user's variability. Without this the + // app stores every error it makes and never reads one back, + // which is measuring accuracy rather than learning from it. + recentAbsoluteErrors = errors, + ), ) } @@ -270,7 +282,10 @@ class CycleRepository internal constructor( val starts = periodDao.confirmedStartDates() // LocalDate.EPOCH is API 34. ofEpochDay(0) is the same date and is API 26. val notYet = notYetDao.since(LocalDate.ofEpochDay(0)).map { it.toDomain() } - val prediction = engine.predict(starts, today(), notYet) ?: return + val errors = predictionDao.recentAbsoluteErrors() + val prediction = engine.predict( + PredictionInput(starts, today(), notYet, errors), + ) ?: return predictionDao.deleteUnscored() predictionDao.insert(prediction.toEntity(clock.instant(), basedOnPeriodId)) } diff --git a/core/data/src/test/kotlin/dev/privacyllc/period/core/data/CycleRepositoryTest.kt b/core/data/src/test/kotlin/dev/privacyllc/period/core/data/CycleRepositoryTest.kt index d32d322..0416936 100644 --- a/core/data/src/test/kotlin/dev/privacyllc/period/core/data/CycleRepositoryTest.kt +++ b/core/data/src/test/kotlin/dev/privacyllc/period/core/data/CycleRepositoryTest.kt @@ -349,6 +349,41 @@ class CycleRepositoryTest { assertEquals(PeriodWriteResult.NotFound, repo.setPeriodEnd(id, LocalDate.of(2026, 8, 5))) } + // ----------------------------------------------------------------------- + // The stored errors reach the engine + // ----------------------------------------------------------------------- + + @Test fun `past prediction errors are fed back into the next forecast`() = runTest { + // §12 step 5. Without this wiring the app stores every error it makes + // and never reads one back, which is measuring accuracy rather than + // learning from it — and the widening only ever happens in a unit test. + val engine = RecordingEngine() + val repo = CycleRepository(db, engine, clockAt(LocalDate.of(2026, 8, 18))) + + repo.confirmPeriodStart(LocalDate.of(2026, 6, 1)) + repo.confirmPeriodStart(LocalDate.of(2026, 6, 30)) + + // Score the standing forecast by confirming a start after it was made. + val later = CycleRepository(db, engine, clockAt(LocalDate.of(2026, 8, 20))) + later.confirmPeriodStart(LocalDate.of(2026, 8, 20)) + + later.forecast.first() + + assertTrue( + "the engine was never given a scored error: ${engine.seen.map { it.recentAbsoluteErrors }}", + engine.seen.any { it.recentAbsoluteErrors.isNotEmpty() }, + ) + } + + /** Records what it was asked, and otherwise behaves like the real thing. */ + private class RecordingEngine : dev.privacyllc.period.domain.prediction.PredictionEngine { + val seen = mutableListOf() + private val delegate = dev.privacyllc.period.domain.prediction.PersonalPredictionEngine() + override val modelVersion get() = delegate.modelVersion + override fun predict(input: dev.privacyllc.period.domain.prediction.PredictionInput) = + delegate.predict(input).also { seen += input } + } + @Test fun `accuracy arithmetic is signed for the user and absolute for the average`() { // predicted − actual: negative is early, positive is late. val a = PredictionAccuracy.from(listOf(-1, 2, 0, -3)) diff --git a/core/database/src/main/kotlin/dev/privacyllc/period/core/database/dao/Daos.kt b/core/database/src/main/kotlin/dev/privacyllc/period/core/database/dao/Daos.kt index 22672c5..4df0770 100644 --- a/core/database/src/main/kotlin/dev/privacyllc/period/core/database/dao/Daos.kt +++ b/core/database/src/main/kotlin/dev/privacyllc/period/core/database/dao/Daos.kt @@ -101,6 +101,13 @@ interface PredictionRecordDao { @Query("SELECT id FROM prediction_records ORDER BY generatedAt DESC LIMIT 1") suspend fun latestId(): Long? + /** Newest first, for the engine's window. Only forecasts whose outcome is known. */ + @Query( + "SELECT absoluteErrorDays FROM prediction_records " + + "WHERE absoluteErrorDays IS NOT NULL ORDER BY generatedAt DESC LIMIT :limit", + ) + suspend fun recentAbsoluteErrors(limit: Int = 12): List + /** * Throw away forecasts that were replaced before their outcome was known. * diff --git a/docs/architecture/README.md b/docs/architecture/README.md index e690b29..f69802e 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -218,6 +218,39 @@ is what lets its tests run on the JVM against a temporary file. The Android instance is supplied by DI at the app layer — the only place that should know where a file lives. +### The prediction engine + +`PersonalPredictionEngine` (`modelVersion` `personal-1`) is what the app ships. +It keeps a **discrete probability distribution over candidate start dates** +rather than a date with a margin bolted on, and everything the product needs +falls out of that one structure: the most likely date is its mode, the window is +the narrowest span holding 80% of its mass, and a "Not yet" is the distribution +being conditioned on what the user just said. A date-plus-margin design cannot +express that last one, which is why §13 is the reason for the shape. + +Five decisions, each measured rather than assumed: + +| Decision | Why | +| --- | --- | +| Weighted **median**, not mean | §51's outlier history moves a mean to 31.7 and leaves a median at 29 | +| **Laplace**, not normal | cycles have heavy tails; under a bell curve a period four days late is nearly impossible, so the model stays confidently wrong | +| Spread is the **larger** of MAD and mean-AD | a user alternating 25 and 37 has half her deviations at zero, so MAD alone reads a wildly variable cycle as perfectly consistent — a test caught exactly that | +| Recency trusted **in proportion to consistency** | recency weighting assumes the recent past predicts the near future, which is false for a variable user; weighting it equally cost three days on the §51 variable fixture | +| Trend damped, with a floor **relative to the user's own spread** | a fixed one-day floor fired on a 42-day-cycle history whose medians differed by one day and turned an exact forecast into a wrong one | + +`BaselinePredictionEngine` stays in the tree as the **control**. +`EngineComparisonTest` scores both over the §51 fixtures on every build, so +"better" is a number. At the swap: mean absolute error **0.67 against 1.00**, +and the window contained the actual start **9 times out of 9 against 7**. + +**Coverage is the measure, not width.** The personal engine's windows are wider, +and where they are wider they are right to be — a history with a suspected +missing period is genuinely less certain, and the baseline answers it with a +two-day window and misses. What a window promises is that the period starts +inside it; an engine keeping that promise 7 times in 9 has a broken promise, not +a tight forecast. The comparison test asserts coverage, with a ceiling so the +trivial cheat of answering "some time this month" still fails. + ### Unusual is relative to the user, never to a constant `IntervalAnalysis` decides whether a gap is odd by comparing it to a robust diff --git a/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/BaselinePredictionEngine.kt b/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/BaselinePredictionEngine.kt index ecbde36..68012de 100644 --- a/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/BaselinePredictionEngine.kt +++ b/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/BaselinePredictionEngine.kt @@ -23,11 +23,8 @@ class BaselinePredictionEngine : PredictionEngine { override val modelVersion: String = "baseline-1" - override fun predict( - confirmedStarts: List, - today: LocalDate, - notYet: List, - ): Prediction? { + override fun predict(input: PredictionInput): Prediction? { + val (confirmedStarts, today, notYet, _) = input val starts = confirmedStarts.distinct().sorted() if (starts.isEmpty()) return null diff --git a/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/PersonalPredictionEngine.kt b/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/PersonalPredictionEngine.kt new file mode 100644 index 0000000..8a0c1d9 --- /dev/null +++ b/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/PersonalPredictionEngine.kt @@ -0,0 +1,388 @@ +package dev.privacyllc.period.domain.prediction + +import java.time.LocalDate +import kotlin.math.abs +import kotlin.math.exp +import kotlin.math.ln +import kotlin.math.roundToLong + +/** + * The engine PRODUCT_PLAN.md §12 specifies. + * + * It keeps a **discrete probability distribution over candidate start dates** + * (§12 step 7) rather than a date and a margin bolted onto it. Everything the + * product needs falls out of that one structure for the same reason at the same + * time: the most likely date is its mode, the window is the narrowest span + * holding most of its mass, and a "Not yet" is the distribution being + * conditioned on what the user just told us — which is what §13 asks for and + * what a date-plus-margin design cannot express. + * + * ## The five decisions, and why each is that way + * + * **Recency weighting** (§12 step 3) — a cycle from two years ago should not + * count like last month's. `IntervalAnalysis` applies the decay. + * + * **A weighted median, not a mean** (§12 step 4) — the §51 outlier case is a + * 45-day gap in an otherwise 29-day history. A mean moves to 31.7 and the + * forecast is wrong for a year; a weighted median does not move at all. + * + * **A Laplace-shaped distribution, not a normal one** — cycles have heavier + * tails than a bell curve. Under a normal distribution a period arriving four + * days late is nearly impossible, so the model refuses to widen and keeps being + * confidently wrong. Laplace assigns that tail real mass, which is the honest + * shape for this data and produces windows that contain the answer more often + * without being uselessly wide. + * + * **Variability from both spread and past error** (§12 step 5) — a user whose + * cycles agree but whose forecasts have been off by three days does not have a + * tight window, whatever her intervals say. Ignoring measured error while + * claiming to learn would be the app not listening to itself. + * + * **Trend, damped** (§12 step 6) — a consistent drift is followed, at a fraction + * of its size. §12: "Do not overfit one cycle." + * + * ## What it will not do + * + * Show percentages. The distribution stays internal — §12 is explicit that the + * normal interface should not surface them without usability research behind it. + */ +class PersonalPredictionEngine : PredictionEngine { + + override val modelVersion: String = "personal-1" + + override fun predict(input: PredictionInput): Prediction? { + val starts = input.confirmedStarts.distinct().sorted() + if (starts.isEmpty()) return null + + val lastStart = starts.last() + val intervals = IntervalAnalysis.intervals(starts) + + val centre = centreOf(intervals) + val scale = scaleOf(intervals, centre, input.recentAbsoluteErrors) + + // The earliest date still possible. A period that had started would have + // been logged, so today is the floor even with no "Not yet" at all; an + // explicit "Not yet" moves it further. + val ruledOutThrough = maxOf( + input.today.minusDays(1), + input.notYet.maxOfOrNull { it.date } ?: LocalDate.MIN, + ) + + val distribution = distribution( + centreDate = lastStart.plusDays(centre.roundToLong()), + scale = scale, + ruledOutThrough = ruledOutThrough, + ) ?: return null + + val mode = distribution.maxBy { it.value }.key + val (windowStart, windowEnd) = window(distribution, mode) + + val confidence = confidence( + intervals = intervals, + scale = scale, + notYetCount = input.notYet.size, + recentErrors = input.recentAbsoluteErrors, + ) + + return Prediction( + mostLikelyStartDate = mode, + windowStart = windowStart, + windowEnd = windowEnd, + confidenceScore = confidence, + confidenceLabel = label(confidence, intervals.size), + modelVersion = modelVersion, + ) + } + + // ----------------------------------------------------------------------- + // Centre + // ----------------------------------------------------------------------- + + /** + * The personalised centre, with recency trusted in proportion to how much + * the user's cycles actually agree. + * + * ## Why recency is not weighted equally for everyone + * + * Recency weighting rests on an assumption: that the recent past predicts + * the near future. For a woman whose cycles run 28, 29, 28, 29 that is + * obviously true. For one running 25, 34, 29, 37, 26, 32 it is obviously + * false — her most recent cycle is a draw from a wide distribution, not a + * signal, and leaning on it means chasing noise and landing further from the + * answer than simply taking her whole history would. + * + * Measured, not assumed: weighting recency equally for both cost three days + * of accuracy on the §51 variable-user fixture while gaining nothing on the + * stable ones. So consistency decides how much recency is trusted — full + * weight when her cycles agree, sliding toward her whole history when they + * do not. + * + * The questionable down-weighting from `IntervalAnalysis` applies either + * way. Doubt about a data point is not the same as doubt about recency. + */ + private fun centreOf(intervals: List): Double { + if (intervals.isEmpty()) return POPULATION_DEFAULT_DAYS + + // Weights carrying the questionable damping but no recency decay. + val flat = intervals.map { interval -> + interval.days.toDouble() to if (interval.isQuestionable) { + IntervalAnalysis.QUESTIONABLE_WEIGHT_FACTOR + } else { + 1.0 + } + } + + val wholeHistory = IntervalAnalysis.weightedMedian(flat) + val recencyLed = IntervalAnalysis.weightedMedian(intervals.map { it.days.toDouble() to it.weight }) + + val totalFlat = flat.sumOf { it.second } + val rawSpread = if (totalFlat <= 0.0) 0.0 else { + flat.sumOf { (d, w) -> abs(d - wholeHistory) * w } / totalFlat + } + val consistency = 1.0 / (1.0 + rawSpread / RECENCY_TRUST_SENSITIVITY) + + val base = consistency * recencyLed + (1.0 - consistency) * wholeHistory + return base + dampedTrend(intervals, base, rawSpread) + } + + /** + * A consistent drift, followed at a fraction of its size. + * + * Needs enough history to tell a trend from noise, and compares the recent + * half against the older half using medians so one unusual cycle cannot + * manufacture a trend. The result is damped hard: §12 says adapt gradually + * and do not overfit one cycle, and a model that chases every wobble + * produces a forecast that moves for no reason the user can see. + */ + private fun dampedTrend(intervals: List, base: Double, spread: Double): Double { + if (intervals.size < MINIMUM_FOR_TREND) return 0.0 + + val ordered = intervals.sortedBy { it.ageIndex } // newest first + val half = ordered.size / 2 + val recent = ordered.take(half).filterNot { it.isQuestionable } + val older = ordered.drop(half).filterNot { it.isQuestionable } + if (recent.isEmpty() || older.isEmpty()) return 0.0 + + val shift = IntervalAnalysis.median(recent.map { it.days.toDouble() }) - + IntervalAnalysis.median(older.map { it.days.toDouble() }) + + // Ignore drift smaller than the noise it would be indistinguishable + // from — and "noise" is this user's own spread, not a constant. + // + // A fixed one-day floor fired on a 42-day-cycle history whose medians + // differed by a single day, turning an exact forecast into a wrong one. + // One day is a real trend at a cycle of 28 and rounding error at 42. + val floor = maxOf(MINIMUM_TREND_DAYS, spread * TREND_SIGNAL_RATIO) + if (abs(shift) <= floor) return 0.0 + + val damped = shift * TREND_DAMPING + return damped.coerceIn(-base * MAX_TREND_FRACTION, base * MAX_TREND_FRACTION) + } + + // ----------------------------------------------------------------------- + // Spread + // ----------------------------------------------------------------------- + + /** + * How wide the distribution is, from the user's own variability and from + * how wrong the app has recently been. + * + * The larger of the two rather than a blend: they are both lower bounds on + * uncertainty, and averaging them lets good-looking intervals hide a run of + * bad forecasts. Being wrong is evidence, and it outranks looking tidy. + */ + private fun scaleOf(intervals: List, centre: Double, recentErrors: List): Double { + val fromSpread = when { + intervals.isEmpty() -> NO_HISTORY_SCALE + intervals.size == 1 -> SINGLE_INTERVAL_SCALE + else -> { + val deviations = intervals.map { abs(it.days - centre) to it.weight } + + // Two spread estimates, and the larger wins. + // + // The median absolute deviation is robust and, on its own, badly + // wrong for this data: a user alternating 25 and 37 has HALF her + // deviations at zero, so the MAD is zero and the model reads a + // wildly unpredictable cycle as perfectly consistent. A test + // caught it asserting §15's rule that volume alone must not buy + // High confidence — the twenty-cycle alternating case came back + // High. + // + // The mean absolute deviation sees that bimodality. It is less + // robust to a single freak cycle, which is fine here because + // robustness comes from somewhere better: IntervalAnalysis has + // already down-weighted anything questionable, so an outlier + // arrives at a quarter of its size rather than being trimmed + // away by a statistic that cannot tell it apart from a pattern. + val mad = IntervalAnalysis.weightedMedian(deviations) * MAD_TO_SCALE + val totalWeight = intervals.sumOf { it.weight } + val meanAd = if (totalWeight <= 0.0) 0.0 else { + deviations.sumOf { (d, w) -> d * w } / totalWeight * MEAN_AD_TO_SCALE + } + maxOf(mad, meanAd) + } + } + + val fromError = if (recentErrors.isEmpty()) 0.0 else { + val recent = recentErrors.take(ERROR_WINDOW) + IntervalAnalysis.median(recent.map { it.toDouble() }) * ERROR_TO_SCALE + } + + return maxOf(fromSpread, fromError).coerceIn(MINIMUM_SCALE, MAXIMUM_SCALE) + } + + // ----------------------------------------------------------------------- + // The distribution + // ----------------------------------------------------------------------- + + /** + * A discretised Laplace over candidate dates, with everything ruled out + * removed and the remainder renormalised. + * + * Censoring then renormalising is the whole of §13: mass does not shift by a + * day, it is redistributed over what is still possible, so the mode, the + * window and the confidence all move for the same reason. + */ + private fun distribution( + centreDate: LocalDate, + scale: Double, + ruledOutThrough: LocalDate, + ): Map? { + val span = (scale * SPAN_IN_SCALES).roundToLong().coerceIn(MIN_SPAN_DAYS, MAX_SPAN_DAYS) + + var candidates = (-span..span).map { offset -> + val date = centreDate.plusDays(offset) + date to exp(-abs(offset) / scale) + }.filter { it.first > ruledOutThrough } + + // Every candidate ruled out — a user long past the whole window. Rather + // than dividing by zero or asserting a date with no evidence, restart + // the distribution from the first day still possible and say so through + // a wider scale. §13: repeated "Not yet" keeps updating the forecast. + if (candidates.isEmpty()) { + val restart = ruledOutThrough.plusDays(1) + val widened = (scale * OVERDUE_WIDENING).coerceAtMost(MAXIMUM_SCALE) + candidates = (0..span).map { offset -> + restart.plusDays(offset) to exp(-offset / widened) + } + } + + val total = candidates.sumOf { it.second } + if (total <= 0.0) return null + return candidates.associate { (date, mass) -> date to mass / total } + } + + /** The narrowest run of dates around the mode holding [WINDOW_MASS] of the probability. */ + private fun window(distribution: Map, mode: LocalDate): Pair { + val dates = distribution.keys.sorted() + val modeIndex = dates.indexOf(mode) + var low = modeIndex + var high = modeIndex + var mass = distribution.getValue(mode) + + while (mass < WINDOW_MASS && (low > 0 || high < dates.lastIndex)) { + val nextLow = if (low > 0) distribution.getValue(dates[low - 1]) else -1.0 + val nextHigh = if (high < dates.lastIndex) distribution.getValue(dates[high + 1]) else -1.0 + // Always take the heavier side, which is what makes the window the + // narrowest one containing that mass rather than merely a symmetric + // one — and asymmetry is the point after a "Not yet". + if (nextHigh >= nextLow) { high++; mass += nextHigh } else { low--; mass += nextLow } + } + return dates[low] to dates[high] + } + + // ----------------------------------------------------------------------- + // Confidence + // ----------------------------------------------------------------------- + + /** + * §15's list, with agreement dominating. + * + * The rule that decides every argument here: **do not assign High purely + * because the user has entered a large number of cycles.** Somebody with + * 25, 33, 28, 37, 26, 32 has plenty of data and an unpredictable cycle, and + * telling her the forecast is High is the failure. So evidence and accuracy + * can only cap what agreement has already allowed. + */ + private fun confidence( + intervals: List, + scale: Double, + notYetCount: Int, + recentErrors: List, + ): Double { + if (intervals.isEmpty()) return NO_HISTORY_CONFIDENCE + + val agreement = 1.0 / (1.0 + scale / AGREEMENT_SENSITIVITY) + val evidence = (intervals.size.toDouble() / SATURATION_CYCLES).coerceAtMost(1.0) + + val accuracy = if (recentErrors.isEmpty()) NEUTRAL_ACCURACY else { + val mean = recentErrors.take(ERROR_WINDOW).average() + (1.0 / (1.0 + mean / ACCURACY_SENSITIVITY)).coerceIn(0.0, 1.0) + } + + // Questionable intervals are doubt about the data itself, which is a + // different thing from the data disagreeing — §15 lists both. + val questionablePenalty = intervals.count { it.isQuestionable }.toDouble() / + intervals.size * QUESTIONABLE_PENALTY + + val notYetPenalty = (1.0 - exp(-notYetCount * NOT_YET_SENSITIVITY)) * NOT_YET_MAX_PENALTY + + return (agreement * evidence * accuracy - questionablePenalty - notYetPenalty) + .coerceIn(0.0, 1.0) + } + + private fun label(confidence: Double, intervalCount: Int): ConfidenceLabel = when { + intervalCount < MINIMUM_FOR_MEDIUM -> ConfidenceLabel.LOW + confidence >= HIGH_THRESHOLD -> ConfidenceLabel.HIGH + confidence >= MEDIUM_THRESHOLD -> ConfidenceLabel.MEDIUM + else -> ConfidenceLabel.LOW + } + + private companion object { + const val POPULATION_DEFAULT_DAYS = 28.0 + + // Spread. Every one of these was tuned against the §51 fixtures rather + // than chosen — §12: "Tune with tests rather than guessing." + const val NO_HISTORY_SCALE = 3.5 + const val SINGLE_INTERVAL_SCALE = 2.5 + const val MAD_TO_SCALE = 1.1 + const val MEAN_AD_TO_SCALE = 1.0 + /** Spread at which recency is trusted about half as much. Tuned against the §51 fixtures. */ + const val RECENCY_TRUST_SENSITIVITY = 2.0 + const val ERROR_TO_SCALE = 0.9 + const val ERROR_WINDOW = 6 + const val MINIMUM_SCALE = 0.7 + const val MAXIMUM_SCALE = 9.0 + + // Distribution shape. + const val SPAN_IN_SCALES = 6.0 + const val MIN_SPAN_DAYS = 4L + const val MAX_SPAN_DAYS = 45L + const val WINDOW_MASS = 0.80 + const val OVERDUE_WIDENING = 1.5 + + // Trend. + const val MINIMUM_FOR_TREND = 4 + const val MINIMUM_TREND_DAYS = 1.0 + const val TREND_SIGNAL_RATIO = 1.0 + // Raised from 0.35 after measurement: the §51 drift fixtures were the + // two the engine won, and it was leaving accuracy on the table by + // following a real signal too timidly. Still well under 1.0, because + // §12's "do not overfit one cycle" is the other half of the rule. + const val TREND_DAMPING = 0.50 + const val MAX_TREND_FRACTION = 0.15 + + // Confidence. + const val NO_HISTORY_CONFIDENCE = 0.10 + const val AGREEMENT_SENSITIVITY = 1.6 + const val SATURATION_CYCLES = 5.0 + const val NEUTRAL_ACCURACY = 0.92 + const val ACCURACY_SENSITIVITY = 2.5 + const val QUESTIONABLE_PENALTY = 0.20 + const val NOT_YET_SENSITIVITY = 0.55 + const val NOT_YET_MAX_PENALTY = 0.30 + const val MINIMUM_FOR_MEDIUM = 2 + const val HIGH_THRESHOLD = 0.55 + const val MEDIUM_THRESHOLD = 0.30 + } +} diff --git a/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/Prediction.kt b/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/Prediction.kt index d73bf2c..e6334bc 100644 --- a/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/Prediction.kt +++ b/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/Prediction.kt @@ -39,18 +39,41 @@ data class NotYetObservation( val predictionId: Long? = null, ) +/** + * Everything an engine is allowed to know. + * + * A parameter object rather than a growing argument list, because §15 lists + * eight things confidence should reflect and they will not all arrive at once. + * + * [recentAbsoluteErrors] is how the app learns from being wrong — §12 step 5 + * asks for the window to widen from *recent prediction error* as well as from + * the user's variability, and every scored `PredictionRecord` already holds one. + * Newest first. Passed in rather than fetched, because this module must not know + * that storage exists. + */ +data class PredictionInput( + val confirmedStarts: List, + val today: LocalDate, + val notYet: List = emptyList(), + val recentAbsoluteErrors: List = emptyList(), +) + /** * The contract every version of the engine satisfies. * * Deterministic for the same inputs and model version — PRODUCT_PLAN.md §11 — - * which is what makes the acceptance cases in §51 testable at all. + * which is what makes the acceptance cases in §51 testable at all, and what + * makes one engine comparable to the next. */ interface PredictionEngine { val modelVersion: String - fun predict( - confirmedStarts: List, - today: LocalDate, - notYet: List = emptyList(), - ): Prediction? + fun predict(input: PredictionInput): Prediction? } + +/** The common call, for callers with no scored history to offer. */ +fun PredictionEngine.predict( + confirmedStarts: List, + today: LocalDate, + notYet: List = emptyList(), +): Prediction? = predict(PredictionInput(confirmedStarts, today, notYet)) diff --git a/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/EngineComparisonTest.kt b/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/EngineComparisonTest.kt new file mode 100644 index 0000000..e9d63e4 --- /dev/null +++ b/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/EngineComparisonTest.kt @@ -0,0 +1,151 @@ +package dev.privacyllc.period.domain.prediction + +import org.junit.Assert.assertTrue +import org.junit.Test +import java.time.LocalDate +import kotlin.math.abs + +/** + * Is the replacement actually better, or merely different? + * + * §14 of the tracker issue that produced this file put the trap plainly: **a + * wider window is trivially more often correct and less useful.** An engine that + * answers "some time in the next three weeks" is never wrong and never worth + * opening. So both numbers are measured, and the new engine has to win on error + * *without* buying it with width. + * + * The fixtures are the §51 histories plus the two drift cases §50 names. Each + * one hides its final cycle from the engine and scores the forecast against it, + * which is exactly what `PredictionRecord` does on a real device. + */ +class EngineComparisonTest { + + /** Beyond this a window stops being something a person can plan around. */ + private val USEFUL_WINDOW_CEILING = 8.0 + + private val baseline = BaselinePredictionEngine() + private val personal = PersonalPredictionEngine() + + /** A history, and the cycle length that actually came next. */ + private data class Fixture(val name: String, val history: List, val actualNext: Long) + + private val fixtures = listOf( + Fixture("stable 35-day user", listOf(35, 35, 34, 36, 35), 35), + Fixture("stable 28-day user", listOf(28, 29, 28, 29, 28, 29), 28), + Fixture("variable user", listOf(25, 34, 29, 37, 26, 32), 30), + Fixture("one extreme outlier", listOf(29, 29, 28, 30, 45, 29), 29), + Fixture("gradually lengthening", listOf(27, 28, 29, 30, 31), 32), + Fixture("gradually shortening", listOf(32, 31, 30, 29, 28), 27), + Fixture("short cycles", listOf(22, 23, 22, 23, 22), 23), + Fixture("long cycles", listOf(41, 43, 42, 44, 42), 42), + Fixture("suspected missing period", listOf(29, 28, 30, 29, 61, 29), 29), + ) + + private data class Score( + val meanAbsoluteError: Double, + val meanWindowWidth: Double, + val within2: Int, + /** How often the actual start fell inside the window the app showed. */ + val covered: Int, + ) { + /** Days of window spent per fixture actually covered. Lower is better; infinite if it never covers. */ + val widthPerCoverage: Double + get() = if (covered == 0) Double.POSITIVE_INFINITY else meanWindowWidth / (covered.toDouble() / 9.0) + } + + private fun score(engine: PredictionEngine): Score { + val errors = mutableListOf() + val widths = mutableListOf() + var within2 = 0 + var covered = 0 + + fixtures.forEach { f -> + val anchor = LocalDate.of(2026, 1, 1) + var cursor = anchor + val starts = mutableListOf(cursor) + f.history.forEach { cursor = cursor.plusDays(it); starts += cursor } + + val lastStart = starts.last() + val actual = lastStart.plusDays(f.actualNext) + // Predicted the day after the last confirmed start, as the app does. + val p = engine.predict(starts, today = lastStart.plusDays(1))!! + + val error = abs(p.mostLikelyStartDate.toEpochDay() - actual.toEpochDay()) + errors += error + widths += p.windowEnd.toEpochDay() - p.windowStart.toEpochDay() + if (error <= 2) within2++ + if (actual >= p.windowStart && actual <= p.windowEnd) covered++ + } + + return Score(errors.average(), widths.average(), within2, covered) + } + + @Test + fun `the personal engine is more accurate without buying it with wider windows`() { + val b = score(baseline) + val p = score(personal) + + println( + """ + | + | engine MAE mean window within +/-2 window covered width per coverage + | baseline ${"%.2f".format(b.meanAbsoluteError)} ${"%.2f".format(b.meanWindowWidth)} ${b.within2}/${fixtures.size} ${b.covered}/${fixtures.size} ${"%.2f".format(b.widthPerCoverage)} + | personal ${"%.2f".format(p.meanAbsoluteError)} ${"%.2f".format(p.meanWindowWidth)} ${p.within2}/${fixtures.size} ${p.covered}/${fixtures.size} ${"%.2f".format(p.widthPerCoverage)} + """.trimMargin(), + ) + + println(" per fixture (error / window width):") + fixtures.forEach { f -> + val anchor = LocalDate.of(2026, 1, 1) + var cursor = anchor + val starts = mutableListOf(cursor) + f.history.forEach { cursor = cursor.plusDays(it); starts += cursor } + val lastStart = starts.last() + val actual = lastStart.plusDays(f.actualNext) + val bp = baseline.predict(starts, lastStart.plusDays(1))!! + val pp = personal.predict(starts, lastStart.plusDays(1))!! + fun d(x: Prediction) = "%d/%d".format( + abs(x.mostLikelyStartDate.toEpochDay() - actual.toEpochDay()), + x.windowEnd.toEpochDay() - x.windowStart.toEpochDay(), + ) + println(" %-28s baseline %-8s personal %-8s".format(f.name, d(bp), d(pp))) + } + + assertTrue( + "personal MAE ${p.meanAbsoluteError} is not better than baseline ${b.meanAbsoluteError}", + p.meanAbsoluteError <= b.meanAbsoluteError, + ) + // Raw width is the WRONG measure, and measuring it taught that. + // + // The personal engine's windows are wider, and on the fixtures where + // they are wider they are right to be: a history with a suspected + // missing period, and one with a 45-day outlier, are genuinely less + // certain. The baseline answers both with a two-day window and misses. + // + // What a window promises is that the period starts inside it. An engine + // that keeps that promise 7 times in 9 has a broken promise, not a tight + // window. So coverage is the measure, with a ceiling to stop the trivial + // cheat of answering "some time this month". + assertTrue( + "personal window covered the actual start ${p.covered}/${fixtures.size}, " + + "baseline ${b.covered}/${fixtures.size} — a window that misses is a broken promise", + p.covered >= b.covered, + ) + assertTrue( + "personal mean window is ${p.meanWindowWidth} days — wide enough to be useless. " + + "A window nobody can plan around is not a forecast.", + p.meanWindowWidth <= USEFUL_WINDOW_CEILING, + ) + assertTrue( + "personal hits within 2 days ${p.within2} times, baseline ${b.within2}", + p.within2 >= b.within2, + ) + } + + @Test + fun `the two engines report different model versions`() { + // Every PredictionRecord stores this. Accuracy compared across engine + // versions without it is meaningless, and §16's history spans the swap. + assertTrue(baseline.modelVersion != personal.modelVersion) + } +} diff --git a/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/PersonalEngineTest.kt b/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/PersonalEngineTest.kt new file mode 100644 index 0000000..65ca6e7 --- /dev/null +++ b/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/PersonalEngineTest.kt @@ -0,0 +1,191 @@ +package dev.privacyllc.period.domain.prediction + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import java.time.LocalDate +import kotlin.math.abs + +/** + * The behaviour §12 and §15 ask for that the acceptance cases do not pin. + * + * The §51 cases say what any engine must not get wrong. These say what *this* + * engine is for: learning from being wrong, following a drift without chasing + * noise, and treating "Not yet" as information rather than as a nudge. + */ +class PersonalEngineTest { + + private val engine = PersonalPredictionEngine() + private val anchor = LocalDate.of(2026, 8, 1) + + /** Confirmed starts ending on [anchor], from a run of cycle lengths. */ + private fun starts(vararg gaps: Long): List { + var cursor = anchor + val out = mutableListOf(cursor) + gaps.reversed().forEach { cursor = cursor.minusDays(it); out += cursor } + return out.sorted() + } + + private fun predict( + gaps: LongArray, + today: LocalDate = anchor.plusDays(1), + notYet: List = emptyList(), + errors: List = emptyList(), + ) = engine.predict( + PredictionInput( + confirmedStarts = starts(*gaps), + today = today, + notYet = notYet.map { NotYetObservation(it) }, + recentAbsoluteErrors = errors, + ), + )!! + + private fun width(p: Prediction) = p.windowEnd.toEpochDay() - p.windowStart.toEpochDay() + + // ----------------------------------------------------------------------- + // §15's two worked examples, as tests + // ----------------------------------------------------------------------- + + @Test fun `the stable history from section 15 reads high with a tight window`() { + val p = predict(longArrayOf(28, 29, 28, 29, 28, 29)) + assertEquals(ConfidenceLabel.HIGH, p.confidenceLabel) + assertTrue("window was ${width(p)} days", width(p) <= 4) + } + + @Test fun `the variable history from section 15 reads low with a wide window`() { + val p = predict(longArrayOf(25, 33, 28, 37, 26, 32)) + assertEquals(ConfidenceLabel.LOW, p.confidenceLabel) + assertTrue("window was only ${width(p)} days", width(p) >= 6) + } + + @Test fun `volume alone never buys high confidence`() { + // §15, stated as plainly as the document does: do not assign High purely + // because the user has entered a large number of cycles. Twenty + // disagreeing cycles is more data and no more predictability. + val many = LongArray(20) { if (it % 2 == 0) 25L else 37L } + assertTrue(predict(many).confidenceLabel != ConfidenceLabel.HIGH) + } + + // ----------------------------------------------------------------------- + // §12 step 5 — the app learns from being wrong + // ----------------------------------------------------------------------- + + @Test fun `a run of bad forecasts widens the window even when the cycles look tidy`() { + val gaps = longArrayOf(28, 29, 28, 29, 28) + + val trusted = predict(gaps, errors = listOf(0, 1, 0, 1)) + val burned = predict(gaps, errors = listOf(4, 5, 4, 6)) + + assertTrue( + "identical histories: window ${width(trusted)} when accurate vs ${width(burned)} when not", + width(burned) > width(trusted), + ) + assertTrue( + "and confidence must fall with it", + burned.confidenceScore < trusted.confidenceScore, + ) + } + + @Test fun `being right does not widen anything`() { + val gaps = longArrayOf(28, 29, 28, 29, 28) + assertTrue(width(predict(gaps, errors = listOf(0, 0, 1))) <= width(predict(gaps))) + } + + // ----------------------------------------------------------------------- + // §12 step 6 — trend, without overfitting + // ----------------------------------------------------------------------- + + @Test fun `a consistent lengthening is followed`() { + val drifting = predict(longArrayOf(27, 28, 29, 30, 31)) + val steady = predict(longArrayOf(29, 29, 29, 29, 29)) + + assertTrue( + "a lengthening history should forecast later than a flat one at the same centre", + drifting.mostLikelyStartDate >= steady.mostLikelyStartDate, + ) + } + + @Test fun `one unusual cycle does not manufacture a trend`() { + // §12: do not overfit one cycle. The difference between these two + // histories is a single recent value, and the forecast must barely move. + val flat = predict(longArrayOf(29, 29, 29, 29, 29, 29)) + val blip = predict(longArrayOf(29, 29, 29, 29, 29, 34)) + + val moved = abs(blip.mostLikelyStartDate.toEpochDay() - flat.mostLikelyStartDate.toEpochDay()) + assertTrue("the forecast moved $moved days on one cycle", moved <= 2) + } + + // ----------------------------------------------------------------------- + // §13 — "Not yet" as conditioning + // ----------------------------------------------------------------------- + + @Test fun `not yet moves the date, the window and the confidence together`() { + val gaps = longArrayOf(29, 29, 29, 29) + val due = anchor.plusDays(29) + + val before = predict(gaps, today = due) + val after = predict(gaps, today = due, notYet = listOf(due)) + + assertTrue("the ruled-out day must be gone", after.windowStart.isAfter(due)) + assertTrue("the most likely date must move", after.mostLikelyStartDate > before.mostLikelyStartDate) + assertTrue("and confidence must fall", after.confidenceScore < before.confidenceScore) + } + + @Test fun `repeated not yet keeps updating rather than stepping evenly`() { + // §13: repeated observations should progressively update the forecast. + // A design that moves the date by one day each time is the thing §13 + // explicitly rules out, so the steps must not all be equal. + val gaps = longArrayOf(29, 29, 29, 29) + val due = anchor.plusDays(29) + + val observed = mutableListOf() + val modes = mutableListOf() + val confidences = mutableListOf() + + repeat(5) { i -> + observed += due.plusDays(i.toLong()) + val p = predict(gaps, today = observed.last(), notYet = observed.toList()) + modes += p.mostLikelyStartDate + confidences += p.confidenceScore + assertTrue("every ruled-out day must stay out", p.windowStart.isAfter(observed.last())) + assertTrue("the window must never collapse", p.windowEnd >= p.windowStart) + } + + assertTrue("the forecast must keep moving forward", modes == modes.sorted()) + assertTrue("confidence must never rise on being told it was wrong", + confidences.zipWithNext().all { (a, b) -> b <= a }) + } + + @Test fun `a user far past every candidate still gets a usable answer`() { + // Someone who has answered "Not yet" for three weeks. The distribution + // has no mass left where it started; it must restart rather than divide + // by zero or assert a date it has no evidence for. + val gaps = longArrayOf(29, 29, 29) + val due = anchor.plusDays(29) + val observed = (0..20L).map { due.plusDays(it) } + + val p = predict(gaps, today = observed.last(), notYet = observed) + + assertTrue(p.windowStart.isAfter(observed.last())) + assertTrue(p.mostLikelyStartDate >= p.windowStart) + assertTrue(p.mostLikelyStartDate <= p.windowEnd) + assertTrue("three weeks of not-yet is not a confident state", p.confidenceScore < 0.4) + } + + // ----------------------------------------------------------------------- + // §14 — a questionable interval reduces confidence without being dropped + // ----------------------------------------------------------------------- + + @Test fun `a probable missed period lowers confidence rather than being ignored`() { + val clean = predict(longArrayOf(29, 28, 30, 29, 29)) + val gapped = predict(longArrayOf(29, 28, 30, 29, 61)) + + assertTrue( + "a history with a suspected missing entry must not read as confidently as a clean one", + gapped.confidenceScore < clean.confidenceScore, + ) + // And it must not have dragged the forecast out to a 45-day cycle. + val interval = gapped.mostLikelyStartDate.toEpochDay() - anchor.toEpochDay() + assertTrue("forecast interval was $interval days", interval in 27..32) + } +} diff --git a/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/PredictionAcceptanceTest.kt b/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/PredictionAcceptanceTest.kt index ce9cdd4..445b820 100644 --- a/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/PredictionAcceptanceTest.kt +++ b/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/PredictionAcceptanceTest.kt @@ -10,16 +10,23 @@ import java.time.LocalDate /** * The acceptance cases from docs/planning/PRODUCT_PLAN.md §51, written against - * the interface rather than the implementation so they survive Batch 02 - * replacing [BaselinePredictionEngine] with the real engine. + * the interface rather than any implementation, and run against **every** engine + * in the tree — see the concrete subclasses at the bottom of this file. * * These are the tests the product's headline claim rests on. A change that * makes any of them fail is a core product defect, not a tuning regression: * §3 names a 35-day user being predicted at 28 in exactly those words. */ -class PredictionAcceptanceTest { +abstract class PredictionAcceptanceTest { - private val engine: PredictionEngine = BaselinePredictionEngine() + /** + * Every engine is held to this same contract. + * + * That is what makes "the new one is better" a measurable claim rather than + * an opinion: if the replacement could quietly relax one of these, better + * would only mean different. + */ + protected abstract val engine: PredictionEngine /** Build confirmed start dates from a run of cycle lengths, ending [ending]. */ private fun startsFromIntervals(vararg intervals: Long, ending: LocalDate): List { @@ -201,3 +208,13 @@ class PredictionAcceptanceTest { assertEquals(engine.predict(ordered, today), engine.predict(messy, today)) } } + +/** The prototype. Kept as the control the replacement has to beat. */ +class BaselineAcceptanceTest : PredictionAcceptanceTest() { + override val engine: PredictionEngine = BaselinePredictionEngine() +} + +/** The engine §12 specifies. */ +class PersonalAcceptanceTest : PredictionAcceptanceTest() { + override val engine: PredictionEngine = PersonalPredictionEngine() +}