Buckets:
| """Thin command-line entrypoint for training and score diagnostics.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from .config import load_config | |
| from .data import InterleavedCache, verify_small_subset_rows | |
| def _parser() -> argparse.ArgumentParser: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| subparsers = parser.add_subparsers(dest="command", required=True) | |
| for name in ("train", "probe", "inspect"): | |
| command = subparsers.add_parser(name) | |
| command.add_argument("--config", required=True) | |
| command.add_argument( | |
| "--set", | |
| action="append", | |
| default=[], | |
| metavar="KEY=VALUE", | |
| help="strict dotted YAML override; may be repeated", | |
| ) | |
| if name == "probe": | |
| command.add_argument("--output-dir") | |
| return parser | |
| def main() -> None: | |
| args = _parser().parse_args() | |
| config = load_config(args.config, args.set) | |
| if args.command == "train": | |
| from .trainer import train | |
| train(config) | |
| elif args.command == "probe": | |
| from .probe import probe_small_subset | |
| probe_small_subset(config, args.output_dir) | |
| else: | |
| cache = InterleavedCache(config.data.cache_root) | |
| rows = [cache.by_id[sample_id] for sample_id in config.probe.sample_ids] | |
| provenance = verify_small_subset_rows(rows, config.data.small_subset_manifest) | |
| samples = cache.load_ids( | |
| config.probe.sample_ids, config.objective.information_gain | |
| ) | |
| report = [] | |
| for origin, sample in zip(provenance, samples): | |
| report.append( | |
| origin | |
| | { | |
| "sequence_length": int(sample.input_ids.numel()), | |
| "visual_positions": int(sample.visual_codes.shape[0]), | |
| "transitions": [ | |
| { | |
| "index": event.index, | |
| "direction": event.direction, | |
| "source_tokens": sum( | |
| span.length for span in event.source_sequence_spans | |
| ), | |
| "target_steps": event.target_steps, | |
| } | |
| for event in sample.transitions | |
| ], | |
| } | |
| ) | |
| print(json.dumps(report, indent=2, ensure_ascii=False)) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 2.47 kB
- Xet hash:
- c303e4962b10820bd4a5349d704a8b028689673200edd4464687d50d52aed123
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.