Spaces:
Running
fix(data): an integer share_of part never rounds above its whole
Browse filesshare_of promises PART <= WHOLE by construction; integer rounding broke it
in two places (Symbotic SYM_09222008_74E_sch.DC_THROUGHPUT, profiled
read-only 2026-09-28: 256 of 3,333 rows with CASES_PROCESSED > ENERGY_KWH).
Engine: whole * frac stored as fmt "int" rounds a 2-decimal whole at frac
near 1.0 above itself (3852.68 -> 3853). _share_of_value floors an int part
only when rounding lands above a non-negative whole.
Gate: the part_exceeds_whole clamp LEAST(part, whole) into NUMBER(38,0) is
stored rounded half away from zero, back above the whole, so the re-profile
recorded repair_failed for a defect it could fix. The INFORMATION_SCHEMA
query now returns the declared type (bare DATA_TYPE is "NUMBER" for both
NUMBER(38,0) and NUMBER(16,2)), and _repair_statements(column_types=) clamps
an integer part to FLOOR(whole) when the whole has decimals. Decimal parts
keep the exact clamp.
Read-only check on the Symbotic table: of 256 rows over, the old clamp
leaves 256 and the FLOOR clamp leaves 0. tests/test_share_of_integer_rounding.py
(21); both end-to-end tests fail on the pre-fix code.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
|
@@ -417,8 +417,8 @@ class BlueprintEngine:
|
|
| 417 |
frac = min(max(low + (high - low) * position, 0.0), 1.0)
|
| 418 |
values[m.name] = self._store(
|
| 419 |
m,
|
| 420 |
-
|
| 421 |
-
values.get(m.share_of, 0.0)
|
| 422 |
),
|
| 423 |
)
|
| 424 |
continue
|
|
@@ -712,6 +712,21 @@ def _format_value(value: float, fmt: str, clamp: bool = True, pct_max: float = 1
|
|
| 712 |
return round(value, 2)
|
| 713 |
|
| 714 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 715 |
def _pct_bound(measure: BlueprintMeasure) -> float:
|
| 716 |
"""Infer whether a pct measure is on a 0-1 or 0-100 scale from its params.
|
| 717 |
|
|
|
|
| 417 |
frac = min(max(low + (high - low) * position, 0.0), 1.0)
|
| 418 |
values[m.name] = self._store(
|
| 419 |
m,
|
| 420 |
+
_share_of_value(
|
| 421 |
+
values.get(m.share_of, 0.0), frac, m.fmt, pct_max=_pct_bound(m)
|
| 422 |
),
|
| 423 |
)
|
| 424 |
continue
|
|
|
|
| 712 |
return round(value, 2)
|
| 713 |
|
| 714 |
|
| 715 |
+
def _share_of_value(whole: float, frac: float, fmt: str, pct_max: float = 100.0) -> Any:
|
| 716 |
+
"""A share_of part for storage: `frac` of its whole, never above it.
|
| 717 |
+
|
| 718 |
+
frac <= 1 keeps the raw value under the whole, but rounding an INTEGER
|
| 719 |
+
part can still overshoot a whole with decimals: 3852.68 x 0.9999 rounds
|
| 720 |
+
to 3853 (Symbotic SYM_09222008_74E_sch: 256 of 3,333 rows with more
|
| 721 |
+
cases than kWh). Floor in exactly that case; everywhere else the rounded
|
| 722 |
+
value is the faithful one.
|
| 723 |
+
"""
|
| 724 |
+
part = _format_value(whole * frac, fmt, pct_max=pct_max)
|
| 725 |
+
if fmt == "int" and 0 <= whole < part:
|
| 726 |
+
part = math.floor(whole * frac)
|
| 727 |
+
return part
|
| 728 |
+
|
| 729 |
+
|
| 730 |
def _pct_bound(measure: BlueprintMeasure) -> float:
|
| 731 |
"""Infer whether a pct measure is on a 0-1 or 0-100 scale from its params.
|
| 732 |
|
|
@@ -129,6 +129,25 @@ def is_date_column(data_type: str) -> bool:
|
|
| 129 |
return data_type.upper().split("(")[0].strip() in _DATE_DATA_TYPES
|
| 130 |
|
| 131 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
# Tolerance when comparing MIN/MAX signatures to shortlist mirrored columns.
|
| 133 |
_MIRROR_EPS = 1e-9
|
| 134 |
|
|
@@ -563,10 +582,13 @@ def _repair_statements(
|
|
| 563 |
columns: list[str],
|
| 564 |
formulas: Mapping[str, str],
|
| 565 |
*,
|
|
|
|
| 566 |
allow_clamp: bool = True,
|
| 567 |
) -> list[tuple[str, str]]:
|
| 568 |
"""(description, SQL) repairs for one defect, most faithful first.
|
| 569 |
|
|
|
|
|
|
|
| 570 |
Every statement here targets a table this run created seconds earlier in a
|
| 571 |
throwaway demo schema. Nothing outside the generated schema is touched.
|
| 572 |
"""
|
|
@@ -587,9 +609,19 @@ def _repair_statements(
|
|
| 587 |
|
| 588 |
if defect["defect"] == "part_exceeds_whole":
|
| 589 |
whole = _quote_ident(defect["whole"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 590 |
repairs = [(
|
| 591 |
-
f"clamped {table_name}.{column} to {
|
| 592 |
-
f"UPDATE {ref} SET {q} = LEAST({q}, {
|
| 593 |
)]
|
| 594 |
ratio_formula = formulas.get(f"{table_name}.{defect.get('ratio_column', '')}")
|
| 595 |
if ratio_formula:
|
|
@@ -1072,9 +1104,15 @@ def run_measure_quality_gate(
|
|
| 1072 |
|
| 1073 |
cursor = connection.cursor()
|
| 1074 |
try:
|
|
|
|
|
|
|
|
|
|
| 1075 |
cursor.execute(
|
| 1076 |
"""
|
| 1077 |
-
SELECT TABLE_NAME, COLUMN_NAME,
|
|
|
|
|
|
|
|
|
|
| 1078 |
FROM INFORMATION_SCHEMA.COLUMNS
|
| 1079 |
WHERE TABLE_SCHEMA = %s
|
| 1080 |
ORDER BY TABLE_NAME, ORDINAL_POSITION
|
|
@@ -1089,7 +1127,9 @@ def run_measure_quality_gate(
|
|
| 1089 |
)
|
| 1090 |
|
| 1091 |
measures_by_table: dict[str, list[str]] = {}
|
|
|
|
| 1092 |
for table_name, column_name, data_type in rows:
|
|
|
|
| 1093 |
if is_measure_column(column_name, data_type or ""):
|
| 1094 |
measures_by_table.setdefault(table_name, []).append(column_name)
|
| 1095 |
|
|
@@ -1182,6 +1222,7 @@ def run_measure_quality_gate(
|
|
| 1182 |
statements = _repair_statements(
|
| 1183 |
schema_name, table_name, defect,
|
| 1184 |
measures_by_table.get(table_name, []), formulas,
|
|
|
|
| 1185 |
allow_clamp=allow_clamp,
|
| 1186 |
)
|
| 1187 |
if not statements and not allow_clamp:
|
|
|
|
| 129 |
return data_type.upper().split("(")[0].strip() in _DATE_DATA_TYPES
|
| 130 |
|
| 131 |
|
| 132 |
+
_INTEGER_DATA_TYPES = {"INT", "INTEGER", "BIGINT", "SMALLINT", "TINYINT", "BYTEINT"}
|
| 133 |
+
_FIXED_POINT_DATA_TYPES = {"NUMBER", "DECIMAL", "NUMERIC"}
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def is_integer_type(data_type: str) -> bool:
|
| 137 |
+
"""True when the column stores whole numbers only: NUMBER(p,0) or an INT alias.
|
| 138 |
+
|
| 139 |
+
Snowflake's NUMBER with no scale (or NUMBER(p)) is scale 0.
|
| 140 |
+
"""
|
| 141 |
+
base, _, args = data_type.upper().partition("(")
|
| 142 |
+
base = base.strip()
|
| 143 |
+
if base in _INTEGER_DATA_TYPES:
|
| 144 |
+
return True
|
| 145 |
+
if base not in _FIXED_POINT_DATA_TYPES:
|
| 146 |
+
return False
|
| 147 |
+
precision_scale = args.rstrip(") ").split(",")
|
| 148 |
+
return len(precision_scale) < 2 or int(precision_scale[1]) == 0
|
| 149 |
+
|
| 150 |
+
|
| 151 |
# Tolerance when comparing MIN/MAX signatures to shortlist mirrored columns.
|
| 152 |
_MIRROR_EPS = 1e-9
|
| 153 |
|
|
|
|
| 582 |
columns: list[str],
|
| 583 |
formulas: Mapping[str, str],
|
| 584 |
*,
|
| 585 |
+
column_types: Mapping[str, str],
|
| 586 |
allow_clamp: bool = True,
|
| 587 |
) -> list[tuple[str, str]]:
|
| 588 |
"""(description, SQL) repairs for one defect, most faithful first.
|
| 589 |
|
| 590 |
+
column_types: {column: declared type} for this table, e.g. "NUMBER(38,0)".
|
| 591 |
+
|
| 592 |
Every statement here targets a table this run created seconds earlier in a
|
| 593 |
throwaway demo schema. Nothing outside the generated schema is touched.
|
| 594 |
"""
|
|
|
|
| 609 |
|
| 610 |
if defect["defect"] == "part_exceeds_whole":
|
| 611 |
whole = _quote_ident(defect["whole"])
|
| 612 |
+
clamp_to, clamp_text = whole, defect["whole"]
|
| 613 |
+
# An integer column stores LEAST(3853, 3852.68) rounded back up to 3853,
|
| 614 |
+
# so the clamp never holds and the re-profile records repair_failed
|
| 615 |
+
# (Symbotic SYM_09222008_74E_sch). Clamp an integer part to the whole's
|
| 616 |
+
# FLOOR. Only there: a decimal part, such as a 0-1 share, keeps the
|
| 617 |
+
# whole's exact value.
|
| 618 |
+
if is_integer_type(column_types[column]) and not is_integer_type(
|
| 619 |
+
column_types[defect["whole"]]
|
| 620 |
+
):
|
| 621 |
+
clamp_to, clamp_text = f"FLOOR({whole})", f"FLOOR({defect['whole']})"
|
| 622 |
repairs = [(
|
| 623 |
+
f"clamped {table_name}.{column} to {clamp_text} so the part cannot exceed the whole",
|
| 624 |
+
f"UPDATE {ref} SET {q} = LEAST({q}, {clamp_to}) WHERE {q} > {whole}",
|
| 625 |
)]
|
| 626 |
ratio_formula = formulas.get(f"{table_name}.{defect.get('ratio_column', '')}")
|
| 627 |
if ratio_formula:
|
|
|
|
| 1104 |
|
| 1105 |
cursor = connection.cursor()
|
| 1106 |
try:
|
| 1107 |
+
# DATA_TYPE alone is "NUMBER" for NUMBER(38,0) and NUMBER(16,2) alike.
|
| 1108 |
+
# Spell out the declared type: the part/whole repair must know which
|
| 1109 |
+
# columns hold integers.
|
| 1110 |
cursor.execute(
|
| 1111 |
"""
|
| 1112 |
+
SELECT TABLE_NAME, COLUMN_NAME,
|
| 1113 |
+
IFF(DATA_TYPE = 'NUMBER',
|
| 1114 |
+
'NUMBER(' || NUMERIC_PRECISION || ',' || NUMERIC_SCALE || ')',
|
| 1115 |
+
DATA_TYPE)
|
| 1116 |
FROM INFORMATION_SCHEMA.COLUMNS
|
| 1117 |
WHERE TABLE_SCHEMA = %s
|
| 1118 |
ORDER BY TABLE_NAME, ORDINAL_POSITION
|
|
|
|
| 1127 |
)
|
| 1128 |
|
| 1129 |
measures_by_table: dict[str, list[str]] = {}
|
| 1130 |
+
types_by_table: dict[str, dict[str, str]] = {}
|
| 1131 |
for table_name, column_name, data_type in rows:
|
| 1132 |
+
types_by_table.setdefault(table_name, {})[column_name] = data_type or ""
|
| 1133 |
if is_measure_column(column_name, data_type or ""):
|
| 1134 |
measures_by_table.setdefault(table_name, []).append(column_name)
|
| 1135 |
|
|
|
|
| 1222 |
statements = _repair_statements(
|
| 1223 |
schema_name, table_name, defect,
|
| 1224 |
measures_by_table.get(table_name, []), formulas,
|
| 1225 |
+
column_types=types_by_table[table_name],
|
| 1226 |
allow_clamp=allow_clamp,
|
| 1227 |
)
|
| 1228 |
if not statements and not allow_clamp:
|
|
@@ -1408,10 +1408,15 @@ composite: a nested ratio has no bound of its own, so only a declared `share_of`
|
|
| 1408 |
RX_PLAN_ATTAINMENT, SPEND_VS_PLAN_PCT, OVERCOUNT_INFLATION_RATIO, PRICE_VS_BENCHMARK_RATE.
|
| 1409 |
Fixing them means changing `measure_names` (`_RATE`/`_RATIO` = 0-1, ATTAINMENT/COVERAGE
|
| 1410 |
as proportions), which also moves range bounds.
|
| 1411 |
-
- [
|
| 1412 |
-
|
| 1413 |
-
|
| 1414 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1415 |
- [ ] Symbotic's schema is already rewritten (508 rows clamped; 1-day time travel,
|
| 1416 |
unrecoverable). A correct board needs a rebuild from the brief.
|
| 1417 |
- [ ] Not handled (pre-existing): a proportion written `CURRENT / PRIOR - 1`, and a
|
|
@@ -1422,5 +1427,5 @@ composite: a nested ratio has no bound of its own, so only a declared `share_of`
|
|
| 1422 |
|
| 1423 |
---
|
| 1424 |
|
| 1425 |
-
*Last updated:
|
| 1426 |
-
(fix/
|
|
|
|
| 1408 |
RX_PLAN_ATTAINMENT, SPEND_VS_PLAN_PCT, OVERCOUNT_INFLATION_RATIO, PRICE_VS_BENCHMARK_RATE.
|
| 1409 |
Fixing them means changing `measure_names` (`_RATE`/`_RATIO` = 0-1, ATTAINMENT/COVERAGE
|
| 1410 |
as proportions), which also moves range bounds.
|
| 1411 |
+
- [x] **Integer rounding breaks `share_of`'s guarantee** ✅ Oct 7, branch
|
| 1412 |
+
`fix/share-of-integer-rounding` off test-mcp `cbacde6` (uncommitted, not pushed).
|
| 1413 |
+
Engine `_share_of_value` floors an int part only when rounding lands above its whole.
|
| 1414 |
+
The gate's INFORMATION_SCHEMA query now returns the declared type (`NUMBER(38,0)`;
|
| 1415 |
+
bare `DATA_TYPE` is `NUMBER` for both), and `_repair_statements(column_types=)`
|
| 1416 |
+
clamps an integer part to `FLOOR(whole)` when the whole has decimals. Decimal parts
|
| 1417 |
+
are unchanged. Read-only check on Symbotic DC_THROUGHPUT: 256 rows over; the old clamp
|
| 1418 |
+
leaves 256, the FLOOR clamp leaves 0. `tests/test_share_of_integer_rounding.py` (21);
|
| 1419 |
+
both end-to-end tests fail on pre-fix code (15/2,000 rows over; `repair_failed`).
|
| 1420 |
- [ ] Symbotic's schema is already rewritten (508 rows clamped; 1-day time travel,
|
| 1421 |
unrecoverable). A correct board needs a rebuild from the brief.
|
| 1422 |
- [ ] Not handled (pre-existing): a proportion written `CURRENT / PRIOR - 1`, and a
|
|
|
|
| 1427 |
|
| 1428 |
---
|
| 1429 |
|
| 1430 |
+
*Last updated: October 7, 2026: an integer share_of part can no longer round above its
|
| 1431 |
+
whole (fix/share-of-integer-rounding, uncommitted, not deployed)*
|
|
@@ -0,0 +1,185 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""An integer part never exceeds its whole.
|
| 2 |
+
|
| 3 |
+
`share_of` promises PART <= WHOLE by construction, and integer rounding broke
|
| 4 |
+
that promise in two places (profiled read-only 2026-09-28 on Symbotic,
|
| 5 |
+
DEMOBUILD_2026_09.SYM_09222008_74E_sch.DC_THROUGHPUT):
|
| 6 |
+
|
| 7 |
+
1. Engine: the part was stored as `_format_value(whole * frac, "int")`. A
|
| 8 |
+
2-decimal whole at frac near 1.0 rounds up past itself: 3852.68 -> 3853.
|
| 9 |
+
256 of 3,333 rows had CASES_PROCESSED > ENERGY_KWH this way.
|
| 10 |
+
2. Gate: the part_exceeds_whole repair ran `SET part = LEAST(part, whole)`.
|
| 11 |
+
Into a NUMBER(38,0) part Snowflake stores 3852.68 rounded half away from
|
| 12 |
+
zero, back to 3853. The re-profile still violated and the run recorded
|
| 13 |
+
`repair_failed`, a send-blocking finding for a defect it could fix.
|
| 14 |
+
|
| 15 |
+
The fix floors only where rounding can overshoot: an integer part of a whole
|
| 16 |
+
with decimals. A decimal part (a 0-1 share of a rate) is never floored.
|
| 17 |
+
|
| 18 |
+
Run with: pytest tests/test_share_of_integer_rounding.py -v
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
import math
|
| 22 |
+
|
| 23 |
+
import pytest
|
| 24 |
+
|
| 25 |
+
from demoprep_app.dataset.engine import BlueprintEngine, _share_of_value
|
| 26 |
+
from demoprep_app.integrations.snowflake.quality_gate import (
|
| 27 |
+
_repair_statements,
|
| 28 |
+
is_integer_type,
|
| 29 |
+
run_measure_quality_gate,
|
| 30 |
+
)
|
| 31 |
+
from tests.test_blueprint_measure_hardening import _blueprint
|
| 32 |
+
from tests.test_quality_gate import FakeConnection, FakeCursor, profile_row
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
# --- engine -------------------------------------------------------------------
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def test_an_integer_part_of_a_decimal_whole_is_floored_only_when_rounding_overshoots():
|
| 39 |
+
# The Symbotic row: rounding would store 3853 cases against 3852.68 kWh.
|
| 40 |
+
assert _share_of_value(3852.68, 1.0, "int") == 3852
|
| 41 |
+
assert _share_of_value(100.5, 1.0, "int") == 100
|
| 42 |
+
# Rounding UP is kept whenever the result still fits under the whole.
|
| 43 |
+
assert _share_of_value(100.0, 0.996, "int") == 100
|
| 44 |
+
assert _share_of_value(3852.68, 0.9, "int") == round(3852.68 * 0.9)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def test_a_decimal_part_keeps_its_rounding():
|
| 48 |
+
assert _share_of_value(3852.68, 0.5, "float") == 1926.34
|
| 49 |
+
assert _share_of_value(0.8765, 1.0, "pct", pct_max=1.0) == 0.8765
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
_UTILIZATION = [
|
| 53 |
+
{"name": "AVAILABLE_HOURS", "fmt": "float", "params": {"mean": 400, "sigma": 0.3}},
|
| 54 |
+
{"name": "UTILIZED_HOURS", "fmt": "int", "share_of": "AVAILABLE_HOURS",
|
| 55 |
+
"params": {"low": 0.97, "high": 1.0}},
|
| 56 |
+
{"name": "UTILIZATION_RATE", "kind": "derived", "fmt": "pct",
|
| 57 |
+
"formula": "UTILIZED_HOURS / AVAILABLE_HOURS"},
|
| 58 |
+
]
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def test_generated_integer_parts_never_exceed_a_decimal_whole():
|
| 62 |
+
rows = BlueprintEngine().generate(_blueprint(_UTILIZATION), 2000).table_map()["LOAD_TRANSACTION"].rows
|
| 63 |
+
over = [r for r in rows if r["UTILIZED_HOURS"] > r["AVAILABLE_HOURS"]]
|
| 64 |
+
assert not over, f"{len(over)} of {len(rows)} rows have UTILIZED_HOURS > AVAILABLE_HOURS"
|
| 65 |
+
assert all(isinstance(r["UTILIZED_HOURS"], int) for r in rows)
|
| 66 |
+
assert max(r["UTILIZATION_RATE"] for r in rows) <= 1.0
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
# --- gate: column types -------------------------------------------------------
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
@pytest.mark.parametrize("data_type, expected", [
|
| 73 |
+
("NUMBER(38,0)", True),
|
| 74 |
+
("number(38, 0)", True),
|
| 75 |
+
("NUMBER", True), # Snowflake default scale is 0
|
| 76 |
+
("NUMBER(10)", True),
|
| 77 |
+
("INTEGER", True),
|
| 78 |
+
("BIGINT", True),
|
| 79 |
+
("NUMBER(16,2)", False),
|
| 80 |
+
("DECIMAL(12,4)", False),
|
| 81 |
+
("FLOAT", False),
|
| 82 |
+
("DOUBLE", False),
|
| 83 |
+
("VARCHAR(100)", False),
|
| 84 |
+
("DATE", False),
|
| 85 |
+
])
|
| 86 |
+
def test_is_integer_type(data_type, expected):
|
| 87 |
+
assert is_integer_type(data_type) is expected
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _clamp_sql(part_type, whole_type):
|
| 91 |
+
defect = {"table": "FACT_UTIL", "column": "UTILIZED_HOURS", "defect": "part_exceeds_whole",
|
| 92 |
+
"whole": "AVAILABLE_HOURS", "detail": "", "ratio_column": "UTILIZATION_RATE"}
|
| 93 |
+
statements = _repair_statements(
|
| 94 |
+
"UTIL_sch", "FACT_UTIL", defect, ["UTILIZED_HOURS", "AVAILABLE_HOURS"], {},
|
| 95 |
+
column_types={"UTILIZED_HOURS": part_type, "AVAILABLE_HOURS": whole_type},
|
| 96 |
+
)
|
| 97 |
+
return statements[0][1]
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def test_an_integer_part_is_clamped_to_the_floor_of_a_decimal_whole():
|
| 101 |
+
sql = _clamp_sql("NUMBER(38,0)", "NUMBER(16,2)")
|
| 102 |
+
assert 'SET "UTILIZED_HOURS" = LEAST("UTILIZED_HOURS", FLOOR("AVAILABLE_HOURS"))' in sql
|
| 103 |
+
# Only rows that actually violate are touched.
|
| 104 |
+
assert sql.endswith('WHERE "UTILIZED_HOURS" > "AVAILABLE_HOURS"')
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
@pytest.mark.parametrize("part_type, whole_type", [
|
| 108 |
+
("NUMBER(16,2)", "NUMBER(16,2)"), # a decimal share: FLOOR would destroy it
|
| 109 |
+
("NUMBER(16,2)", "NUMBER(38,0)"),
|
| 110 |
+
("FLOAT", "NUMBER(16,2)"),
|
| 111 |
+
("NUMBER(38,0)", "NUMBER(38,0)"), # integer whole: nothing to round
|
| 112 |
+
])
|
| 113 |
+
def test_the_clamp_is_exact_everywhere_else(part_type, whole_type):
|
| 114 |
+
sql = _clamp_sql(part_type, whole_type)
|
| 115 |
+
assert "FLOOR" not in sql
|
| 116 |
+
assert 'LEAST("UTILIZED_HOURS", "AVAILABLE_HOURS")' in sql
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
# --- gate: end to end ---------------------------------------------------------
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
class _StoringCursor(FakeCursor):
|
| 123 |
+
"""Holds (UTILIZED_HOURS, AVAILABLE_HOURS) values and stores the clamp the way
|
| 124 |
+
Snowflake does: rounded half away from zero into the NUMBER(38,0) part.
|
| 125 |
+
|
| 126 |
+
Snowflake behaviour checked read-only 2026-10-07:
|
| 127 |
+
LEAST(3853, 3852.68)::NUMBER(38,0) = 3853, LEAST(3853, FLOOR(3852.68)) = 3852.
|
| 128 |
+
"""
|
| 129 |
+
|
| 130 |
+
def __init__(self, *args, values, **kwargs):
|
| 131 |
+
super().__init__(*args, **kwargs)
|
| 132 |
+
self.values = [list(v) for v in values]
|
| 133 |
+
|
| 134 |
+
def execute(self, sql, params=None):
|
| 135 |
+
if sql.startswith("SELECT SUM(IFF("):
|
| 136 |
+
self.executed.append(sql)
|
| 137 |
+
self._last_result = (sum(1 for part, whole in self.values if part > whole),)
|
| 138 |
+
return
|
| 139 |
+
if sql.startswith('UPDATE') and 'SET "UTILIZED_HOURS" = LEAST(' in sql:
|
| 140 |
+
self.executed.append(sql)
|
| 141 |
+
floored = 'FLOOR("AVAILABLE_HOURS")' in sql
|
| 142 |
+
for v in self.values:
|
| 143 |
+
if v[0] > v[1]:
|
| 144 |
+
clamped = min(v[0], math.floor(v[1]) if floored else v[1])
|
| 145 |
+
v[0] = math.floor(clamped + 0.5)
|
| 146 |
+
return
|
| 147 |
+
super().execute(sql, params)
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def test_the_gate_proves_its_clamp_of_an_integer_part():
|
| 151 |
+
stats = [
|
| 152 |
+
("UTILIZED_HOURS", {"non_zero": 3, "min": 101.0, "max": 3853.0, "avg": 1358.0}),
|
| 153 |
+
("AVAILABLE_HOURS", {"non_zero": 3, "min": 100.5, "max": 3852.68, "avg": 1361.14}),
|
| 154 |
+
("UTILIZATION_RATE", {"non_zero": 3, "min": 0.92, "max": 1.0, "avg": 0.97}),
|
| 155 |
+
]
|
| 156 |
+
row = profile_row(3, stats)
|
| 157 |
+
cursor = _StoringCursor(
|
| 158 |
+
[
|
| 159 |
+
("FACT_UTIL", "UTILIZED_HOURS", "NUMBER(38,0)"),
|
| 160 |
+
("FACT_UTIL", "AVAILABLE_HOURS", "NUMBER(16,2)"),
|
| 161 |
+
("FACT_UTIL", "UTILIZATION_RATE", "NUMBER(16,2)"),
|
| 162 |
+
],
|
| 163 |
+
profiles={"FACT_UTIL": [row, row]},
|
| 164 |
+
values=[(3853, 3852.68), (101, 100.5), (120, 130.25)],
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
profile = run_measure_quality_gate(
|
| 168 |
+
FakeConnection(cursor), "UTIL_sch",
|
| 169 |
+
formulas={"FACT_UTIL.UTILIZATION_RATE": "UTILIZED_HOURS / AVAILABLE_HOURS"},
|
| 170 |
+
shares={"FACT_UTIL.UTILIZED_HOURS": "AVAILABLE_HOURS"},
|
| 171 |
+
)
|
| 172 |
+
|
| 173 |
+
assert [d["column"] for d in profile["defects"] if d["defect"] == "part_exceeds_whole"] == [
|
| 174 |
+
"UTILIZED_HOURS"
|
| 175 |
+
]
|
| 176 |
+
assert profile["repair_failures"] == []
|
| 177 |
+
assert not [f for f in profile["findings"] if f["code"] == "repair_failed"]
|
| 178 |
+
assert cursor.values == [[3852, 3852.68], [100, 100.5], [120, 130.25]]
|
| 179 |
+
assert profile["repairs"][0].startswith(
|
| 180 |
+
"clamped FACT_UTIL.UTILIZED_HOURS to FLOOR(AVAILABLE_HOURS)"
|
| 181 |
+
)
|
| 182 |
+
# The gate must read declared types: bare DATA_TYPE says "NUMBER" for both,
|
| 183 |
+
# which would read the 2-decimal whole as an integer and skip the FLOOR.
|
| 184 |
+
info_sql = next(s for s in cursor.executed if "INFORMATION_SCHEMA.COLUMNS" in s)
|
| 185 |
+
assert "NUMERIC_SCALE" in info_sql
|