File size: 1,550 Bytes
211ae03
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
"""Check if q01 and q99 are nearly identical for each dimension in a stats JSON file.

A dimension is flagged if |q99[i] - q01[i]| < threshold (default 1e-5),
meaning the feature has almost no variance across the dataset.

Usage:
    python check_stats_range.py <stats_json_path> [--threshold 1e-5]
"""

import json
import argparse
from pathlib import Path


def check_stats(path: str, threshold: float = 1e-5) -> None:
    with open(path) as f:
        stats = json.load(f)

    any_flagged = False
    for key, fields in stats.items():
        q01 = fields.get("q01")
        q99 = fields.get("q99")
        if q01 is None or q99 is None:
            continue

        # Scalar case: wrap in list
        if not isinstance(q01, list):
            q01 = [q01]
            q99 = [q99]

        flagged = [(i, q01[i], q99[i]) for i in range(len(q01)) if abs(q99[i] - q01[i]) < threshold]
        if flagged:
            any_flagged = True
            print(f"\n[{key}] {len(flagged)}/{len(q01)} dimensions have |q99-q01| < {threshold:.0e}:")
            for i, lo, hi in flagged:
                print(f"  dim {i:3d}: q01={lo:.6e}  q99={hi:.6e}  diff={abs(hi-lo):.6e}")

    if not any_flagged:
        print(f"All dimensions have |q99-q01| >= {threshold:.0e}. No issues found.")


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("path", help="Path to stats JSON file")
    parser.add_argument("--threshold", type=float, default=1e-5)
    args = parser.parse_args()
    check_stats(args.path, args.threshold)