Download baselines/pi05/check_stats_range.py from Tinghong-Ye22/code-Psi0-yth-v21: direct link, hf CLI and curl.
- Browser
- Download file 1.55 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/baselines/pi05/check_stats_range.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v21/baselines/pi05/check_stats_range.py
-
curl -L -o check_stats_range.py https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/baselines/pi05/check_stats_range.py
1.55 kB
| """Check if q01 and q99 are nearly identical for each dimension in a stats JSON file. | |
| A dimension is flagged if |q99[i] - q01[i]| < threshold (default 1e-5), | |
| meaning the feature has almost no variance across the dataset. | |
| Usage: | |
| python check_stats_range.py <stats_json_path> [--threshold 1e-5] | |
| """ | |
| import json | |
| import argparse | |
| from pathlib import Path | |
| def check_stats(path: str, threshold: float = 1e-5) -> None: | |
| with open(path) as f: | |
| stats = json.load(f) | |
| any_flagged = False | |
| for key, fields in stats.items(): | |
| q01 = fields.get("q01") | |
| q99 = fields.get("q99") | |
| if q01 is None or q99 is None: | |
| continue | |
| # Scalar case: wrap in list | |
| if not isinstance(q01, list): | |
| q01 = [q01] | |
| q99 = [q99] | |
| flagged = [(i, q01[i], q99[i]) for i in range(len(q01)) if abs(q99[i] - q01[i]) < threshold] | |
| if flagged: | |
| any_flagged = True | |
| print(f"\n[{key}] {len(flagged)}/{len(q01)} dimensions have |q99-q01| < {threshold:.0e}:") | |
| for i, lo, hi in flagged: | |
| print(f" dim {i:3d}: q01={lo:.6e} q99={hi:.6e} diff={abs(hi-lo):.6e}") | |
| if not any_flagged: | |
| print(f"All dimensions have |q99-q01| >= {threshold:.0e}. No issues found.") | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("path", help="Path to stats JSON file") | |
| parser.add_argument("--threshold", type=float, default=1e-5) | |
| args = parser.parse_args() | |
| check_stats(args.path, args.threshold) | |