Buckets:
| """Relocate a copied BAGEL cache's path metadata; preserve original metadata.""" | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| def relocate(value, old_root, new_root): | |
| if isinstance(value, str): | |
| return new_root + value[len(old_root):] if value.startswith(old_root + "/") else value | |
| if isinstance(value, list): | |
| return [relocate(item, old_root, new_root) for item in value] | |
| if isinstance(value, dict): | |
| return {relocate(key, old_root, new_root): relocate(item, old_root, new_root) | |
| for key, item in value.items()} | |
| return value | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--project-root", type=Path, required=True) | |
| parser.add_argument("--source-root", required=True) | |
| args = parser.parse_args() | |
| root = args.project_root.resolve() | |
| cache = root / "data/ThinkMorph/Bagel-SFT-IDVal5" | |
| for name in ("manifest.json", "registry.json", "parquet_info.json"): | |
| path = cache / name | |
| original = path.read_text() | |
| payload = relocate(json.loads(original), args.source_root, str(root)) | |
| if payload == json.loads(original): | |
| continue | |
| backup = path.with_suffix(".cluster-original.json") | |
| if backup.exists(): | |
| raise FileExistsError(backup) | |
| backup.write_text(original) | |
| staging = path.with_suffix(".relocating.json") | |
| staging.write_text(json.dumps(payload, indent=2) + "\n") | |
| staging.replace(path) | |
| manifest = json.loads((cache / "manifest.json").read_text()) | |
| for record in manifest["output_files"]: | |
| if not Path(record["path"]).is_file(): | |
| raise FileNotFoundError(record["path"]) | |
| print(json.dumps({"root": str(root), "files": len(manifest["output_files"]), | |
| "splits": manifest["splits"]})) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 1.89 kB
- Xet hash:
- 3fa159382ebf990f5024ddb0c7503af740f880f915c01686b35c781856f70615
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.