sofhiaazzhr Claude Opus 5.5 commited on
Commit
50cae40
Β·
1 Parent(s): f28898b

[NOTICKET] feat(trend,forecast): row count per period and stub flag (P0-8)

Browse files

- Each trend point and forecast history point now carries `n` (rows) and `stub`
- A stub has fewer rows than half the median non-empty period (needs >= 3 periods);
empty periods are not stubs
- analyze_trend leaves stubs out of first/last, direction, slope and R^2, with a caveat
naming them; it falls back to all periods if every one is a stub
- analyze_forecast still fits every period but caveats a thin last actual period
- IPR OB LOADER (Sep 9 rows vs 63-66): sum(mtbs) "down" -> "up" (R^2 0.01 -> 0.91);
mean(pa) last 99.95 (Sep) -> 92.45 (Jul)
- Reported by Rifqi (audit P0-8, decision D4)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/tools/analytics/forecast.py CHANGED
@@ -35,6 +35,7 @@ from src.tools.analytics.temporal import (
35
  _parse_date_column,
36
  _period_label,
37
  _resample,
 
38
  )
39
  from src.tools.ml.executor import run_fit
40
  from src.tools.spec_builder import build_description
@@ -193,9 +194,9 @@ async def analyze_forecast(
193
  ordinary synchronous compute. `data` is materialised upstream (Pattern A) and
194
  arrives as `df`.
195
 
196
- Returns a dict with: freq, agg, horizon, model, history + forecast point lists,
197
- last_actual / forecast_end / change_abs / change_pct / direction, and `charts`
198
- (the reserved auto-chart key).
199
 
200
  Raises (all wrapped by the invoker's never-throw seam):
201
  ColumnNotFoundError, InvalidFrequencyError, UnsupportedAggregationError,
@@ -241,9 +242,26 @@ async def analyze_forecast(
241
  future_index = pd.date_range(
242
  start=filled.index[-1], periods=horizon + 1, freq=FREQ_MAP[freq]
243
  )[1:]
 
 
244
  history = [
245
- {"period": _period_label(ts, freq), "value": _clean(val)} for ts, val in resampled.items()
 
 
 
 
 
 
246
  ]
 
 
 
 
 
 
 
 
 
247
  forecast_pts = [
248
  {"period": _period_label(ts, freq), "value": val}
249
  for ts, val in zip(future_index, fc_values, strict=True)
@@ -272,6 +290,7 @@ async def analyze_forecast(
272
  "change_abs": change_abs,
273
  "change_pct": change_pct,
274
  "direction": direction,
 
275
  # Reserved `quality` block (conventions Β§3): the tool reports its own score and
276
  # its own verdict; CK7 surfaces `weak`. An unvalidated forecast (series too
277
  # short to hold anything out) counts as weak β€” shipping a projection nobody
 
35
  _parse_date_column,
36
  _period_label,
37
  _resample,
38
+ _stub_periods,
39
  )
40
  from src.tools.ml.executor import run_fit
41
  from src.tools.spec_builder import build_description
 
194
  ordinary synchronous compute. `data` is materialised upstream (Pattern A) and
195
  arrives as `df`.
196
 
197
+ Returns a dict with: freq, agg, horizon, model, history (with per-period `n` and
198
+ `stub`) + forecast point lists, last_actual / forecast_end / change_abs /
199
+ change_pct / direction, caveats, and `charts` (the reserved auto-chart key).
200
 
201
  Raises (all wrapped by the invoker's never-throw seam):
202
  ColumnNotFoundError, InvalidFrequencyError, UnsupportedAggregationError,
 
242
  future_index = pd.date_range(
243
  start=filled.index[-1], periods=horizon + 1, freq=FREQ_MAP[freq]
244
  )[1:]
245
+ counts = _resample(frame["_value"], freq, "count")
246
+ stubs, median_n = _stub_periods(counts)
247
  history = [
248
+ {
249
+ "period": _period_label(ts, freq),
250
+ "value": _clean(val),
251
+ "n": int(counts[ts]),
252
+ "stub": ts in stubs,
253
+ }
254
+ for ts, val in resampled.items()
255
  ]
256
+ # The fit still uses every period; a thin last period is the base the projection
257
+ # starts from, so say so rather than drop it (audit P0-8).
258
+ caveats: list[str] = []
259
+ if observed.index[-1] in stubs and median_n is not None:
260
+ caveats.append(
261
+ f"The last actual period {_period_label(observed.index[-1], freq)} rests on "
262
+ f"{int(counts[observed.index[-1]])} rows against a median of {median_n:g} β€” "
263
+ "the forecast starts from a partly covered period."
264
+ )
265
  forecast_pts = [
266
  {"period": _period_label(ts, freq), "value": val}
267
  for ts, val in zip(future_index, fc_values, strict=True)
 
290
  "change_abs": change_abs,
291
  "change_pct": change_pct,
292
  "direction": direction,
293
+ "caveats": caveats,
294
  # Reserved `quality` block (conventions Β§3): the tool reports its own score and
295
  # its own verdict; CK7 surfaces `weak`. An unvalidated forecast (series too
296
  # short to hold anything out) counts as weak β€” shipping a projection nobody
src/tools/analytics/temporal.py CHANGED
@@ -45,6 +45,13 @@ SUPPORTED_AGGS = ("sum", "mean", "count", "min", "max", "median")
45
  # reported direction is then flagged as weakly supported (D-7).
46
  _WEAK_TREND_R2 = 0.3
47
 
 
 
 
 
 
 
 
48
 
49
  class InvalidFrequencyError(ValueError):
50
  """The requested period is not in FREQ_MAP (maps to error_code INVALID_FREQUENCY)."""
@@ -141,6 +148,19 @@ def _resample(series: pd.Series, freq: str, agg: str) -> pd.Series:
141
  return resampler.agg(agg)
142
 
143
 
 
 
 
 
 
 
 
 
 
 
 
 
 
144
  def _period_label(ts: pd.Timestamp, freq: str) -> str:
145
  """Human-readable period label keyed off the friendly frequency name."""
146
  if freq == "month":
@@ -238,8 +258,10 @@ def analyze_trend(
238
  Returns:
239
  dict with:
240
  freq, agg, value_column β€” echo of the chosen settings
241
- points β€” [{"period": str, "value": number|None}, ...]
242
- first, last β€” value of the first/last non-empty period
 
 
243
  change_abs β€” last - first
244
  change_pct β€” (last - first) / first, or None if first == 0
245
  direction β€” "up" | "down" | "flat" (endpoints: last vs first)
@@ -282,14 +304,25 @@ def analyze_trend(
282
  values = pd.Series(df[value_column].to_numpy(), name="_value")
283
  s = pd.concat([dates, values], axis=1).dropna(subset=["_date"]).set_index("_date")
284
  resampled = _resample(s["_value"], freq, agg)
 
 
285
 
286
  points = [
287
- {"period": _period_label(ts, freq), "value": _clean(val)}
 
 
 
 
 
288
  for ts, val in resampled.items()
289
  ]
290
 
291
- # Summary stats are computed over non-empty periods only.
 
292
  non_null = resampled.dropna()
 
 
 
293
  first: float | None
294
  last: float | None
295
  change_abs: float | None
@@ -329,6 +362,14 @@ def analyze_trend(
329
  # On a noisy series they can disagree, and neither is wrong β€” so say so rather than
330
  # letting the assembler assert one. R^2 also lets it qualify a weak trend.
331
  caveats: list[str] = []
 
 
 
 
 
 
 
 
332
  if slope is not None and change_abs is not None and change_abs != 0 and slope != 0:
333
  if (change_abs > 0) != (slope > 0):
334
  caveats.append(
 
45
  # reported direction is then flagged as weakly supported (D-7).
46
  _WEAK_TREND_R2 = 0.3
47
 
48
+ # A period with fewer rows than this share of the median period is a stub: a month
49
+ # the data only partly covers. Audit P0-8: a September with 9 rows (other months
50
+ # 63-66) set the first/last comparison and the answer said "turun".
51
+ _STUB_SHARE = 0.5
52
+ # Below this many non-empty periods a median says too little to call one short.
53
+ _STUB_MIN_PERIODS = 3
54
+
55
 
56
  class InvalidFrequencyError(ValueError):
57
  """The requested period is not in FREQ_MAP (maps to error_code INVALID_FREQUENCY)."""
 
148
  return resampler.agg(agg)
149
 
150
 
151
+ def _stub_periods(counts: pd.Series) -> tuple[set[pd.Timestamp], float | None]:
152
+ """Periods whose row count is below `_STUB_SHARE` of the median non-empty period.
153
+
154
+ Returns (stub period timestamps, that median). Empty periods are not stubs β€” they
155
+ have no value to mislead with.
156
+ """
157
+ filled = counts[counts > 0]
158
+ if filled.shape[0] < _STUB_MIN_PERIODS:
159
+ return set(), None
160
+ median = float(filled.median())
161
+ return set(filled[filled < median * _STUB_SHARE].index), median
162
+
163
+
164
  def _period_label(ts: pd.Timestamp, freq: str) -> str:
165
  """Human-readable period label keyed off the friendly frequency name."""
166
  if freq == "month":
 
258
  Returns:
259
  dict with:
260
  freq, agg, value_column β€” echo of the chosen settings
261
+ points β€” [{"period": str, "value": number|None, "n": rows,
262
+ "stub": bool}, ...]; a stub has fewer than half
263
+ the median period's rows
264
+ first, last β€” value of the first/last non-empty, non-stub period
265
  change_abs β€” last - first
266
  change_pct β€” (last - first) / first, or None if first == 0
267
  direction β€” "up" | "down" | "flat" (endpoints: last vs first)
 
304
  values = pd.Series(df[value_column].to_numpy(), name="_value")
305
  s = pd.concat([dates, values], axis=1).dropna(subset=["_date"]).set_index("_date")
306
  resampled = _resample(s["_value"], freq, agg)
307
+ counts = _resample(s["_value"], freq, "count")
308
+ stubs, median_n = _stub_periods(counts)
309
 
310
  points = [
311
+ {
312
+ "period": _period_label(ts, freq),
313
+ "value": _clean(val),
314
+ "n": int(counts[ts]),
315
+ "stub": ts in stubs,
316
+ }
317
  for ts, val in resampled.items()
318
  ]
319
 
320
+ # Summary stats are computed over non-empty periods only, and leave stubs out so a
321
+ # thinly covered month cannot set the direction (unless nothing else is left).
322
  non_null = resampled.dropna()
323
+ full = non_null[~non_null.index.isin(list(stubs))]
324
+ if not full.empty:
325
+ non_null = full
326
  first: float | None
327
  last: float | None
328
  change_abs: float | None
 
362
  # On a noisy series they can disagree, and neither is wrong β€” so say so rather than
363
  # letting the assembler assert one. R^2 also lets it qualify a weak trend.
364
  caveats: list[str] = []
365
+ if stubs and median_n is not None:
366
+ thin = ", ".join(
367
+ f"{_period_label(ts, freq)} (n={int(counts[ts])})" for ts in sorted(stubs)
368
+ )
369
+ caveats.append(
370
+ f"Partly covered period(s): {thin}, against a median of {median_n:g} rows per "
371
+ "period. Shown in the points, but left out of first/last, direction and slope."
372
+ )
373
  if slope is not None and change_abs is not None and change_abs != 0 and slope != 0:
374
  if (change_abs > 0) != (slope > 0):
375
  caveats.append(