diff --git a/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/PersonalPredictionEngine.kt b/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/PersonalPredictionEngine.kt index 8a0c1d9..e4c7efd 100644 --- a/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/PersonalPredictionEngine.kt +++ b/domain/prediction/src/main/kotlin/dev/privacyllc/period/domain/prediction/PersonalPredictionEngine.kt @@ -48,7 +48,14 @@ import kotlin.math.roundToLong */ class PersonalPredictionEngine : PredictionEngine { - override val modelVersion: String = "personal-1" + /** + * Bumped from `personal-1` when confidence stopped punishing measured + * accuracy that was already at the achievable floor, and when spread began + * being measured against a followed trend rather than a static centre. Every + * snapshot stores this, so §16's history spans the change honestly instead + * of comparing two engines' errors as though one engine made them. + */ + override val modelVersion: String = "personal-2" override fun predict(input: PredictionInput): Prediction? { val starts = input.confirmedStarts.distinct().sorted() @@ -315,9 +322,34 @@ class PersonalPredictionEngine : PredictionEngine { val agreement = 1.0 / (1.0 + scale / AGREEMENT_SENSITIVITY) val evidence = (intervals.size.toDouble() / SATURATION_CYCLES).coerceAtMost(1.0) + // Being measured must not cost her confidence she has earned. + // + // This term used to divide raw mean error, and the arithmetic said + // something absurd: a stable user with no scored forecasts scored 0.64 + // (High), and the moment real errors arrived — averaging 0.9 days, which + // is as close as anyone can get to a cycle that varies by a day — she + // scored 0.51 and read Medium, permanently. Being demonstrably as + // accurate as her cycle allows rated *worse* than never having been + // checked, which turned §16's "your predictions are getting better" into + // a claim the confidence label quietly contradicted. + // + // So the yardstick is what the forecast already admits it does not know. + // For a Laplace distribution with scale b, E|X| = b: the mean error of a + // perfectly calibrated forecast IS its scale. Errors inside that are the + // model working, not the model failing, and only the excess counts. + // EXPECTED_ERROR_PER_SCALE is 1.0 because of that identity — a property + // of the distribution this engine already chose, not a tuned number. + // + // Real inaccuracy is still punished, and harder than it looks here: it + // arrives through `agreement`, because ERROR_TO_SCALE feeds scored error + // into the scale itself. A run of genuinely bad forecasts widens the + // window and collapses agreement, which is the honest route — the window + // the user is shown gets wider at the same moment the confidence beside + // it drops, for the same reason. val accuracy = if (recentErrors.isEmpty()) NEUTRAL_ACCURACY else { val mean = recentErrors.take(ERROR_WINDOW).average() - (1.0 / (1.0 + mean / ACCURACY_SENSITIVITY)).coerceIn(0.0, 1.0) + val excess = (mean - scale * EXPECTED_ERROR_PER_SCALE).coerceAtLeast(0.0) + (1.0 / (1.0 + excess / ACCURACY_SENSITIVITY)).coerceIn(0.0, 1.0) } // Questionable intervals are doubt about the data itself, which is a @@ -377,12 +409,18 @@ class PersonalPredictionEngine : PredictionEngine { const val AGREEMENT_SENSITIVITY = 1.6 const val SATURATION_CYCLES = 5.0 const val NEUTRAL_ACCURACY = 0.92 + /** + * The mean absolute error a perfectly calibrated forecast still makes, + * as a multiple of its own scale. For a Laplace distribution E|X| = b, + * so this is 1.0 by identity rather than by tuning. + */ + const val EXPECTED_ERROR_PER_SCALE = 1.0 const val ACCURACY_SENSITIVITY = 2.5 const val QUESTIONABLE_PENALTY = 0.20 const val NOT_YET_SENSITIVITY = 0.55 const val NOT_YET_MAX_PENALTY = 0.30 const val MINIMUM_FOR_MEDIUM = 2 const val HIGH_THRESHOLD = 0.55 - const val MEDIUM_THRESHOLD = 0.30 + const val MEDIUM_THRESHOLD = 0.44 } } diff --git a/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/LearningCurveTest.kt b/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/LearningCurveTest.kt index 94e417d..c0c86af 100644 --- a/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/LearningCurveTest.kt +++ b/domain/prediction/src/test/kotlin/dev/privacyllc/period/domain/prediction/LearningCurveTest.kt @@ -305,7 +305,13 @@ class LearningCurveTest { // that constant in PersonalPredictionEngine for what moved and why. // ----------------------------------------------------------------------- - /** Guards the excess-error accuracy term. Was 0.10–0.15 before it. */ + /** + * Guards the excess-error accuracy term. + * + * Measured at 15.7% and 14.6% before it, which is the defect: a woman whose + * cycles run 28 +/- 1 is being predicted to within a day and told, five + * times in six, that the app is only moderately sure. + */ @Test fun `a textbook-regular user is eventually told the forecast is trustworthy`() { listOf("stable 28", "stable 35").forEach { profile -> @@ -313,7 +319,7 @@ class LearningCurveTest { assertTrue( "$profile read High only ${"%.1f".format(c.high * 100)}% of the time — being " + "as accurate as her cycle allows must not read as uncertainty", - c.high >= 0.05, + c.high >= 0.50, ) } }