Buckets:
| """One-off mechanical rewrite of live references using the reviewed path map. | |
| Never visits output payloads, immutable snapshots, model/data trees, or the | |
| read-only UniAR upstream. Original live files are backed up in the audit. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import ast | |
| import collections | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import re | |
| import shutil | |
| def prefix_mapping(records): | |
| candidates = collections.defaultdict(set) | |
| for row in records: | |
| old, new = Path(row['source']).parts, Path(row['destination']).parts | |
| for end in range(1, len(old) + 1): | |
| tail = old[end:] | |
| mapped = new[:-len(tail)] if tail else new | |
| candidates['/'.join(old[:end])].add('/'.join(mapped) if not tail or new[-len(tail):] == tail else None) | |
| return {'experiments/output/' + old: 'experiments/output/' + next(iter(news)) if next(iter(news)) else 'experiments/output' | |
| for old, news in candidates.items() if len(news) == 1 and None not in news} | |
| def rewrite(text, mapping, *, python=False): | |
| pattern = re.compile('|'.join(re.escape(key) for key in sorted(mapping, key=len, reverse=True))) | |
| def change(value): | |
| def replacement(match): | |
| end = match.end() | |
| if end < len(value) and value[end] not in '/\\ \t\r\n\"\x27`),]}:': | |
| return match.group() | |
| return mapping[match.group()] | |
| return pattern.sub(replacement, value) | |
| text = change(text) | |
| if not python: | |
| return text | |
| tree = ast.parse(text) | |
| fstring_children = {id(child) for node in ast.walk(tree) if isinstance(node, ast.JoinedStr) for child in ast.walk(node)} | |
| encoded = text.encode('utf-8') | |
| lines = encoded.splitlines(keepends=True) | |
| offsets = [0] | |
| for line in lines: | |
| offsets.append(offsets[-1] + len(line)) | |
| edits = [] | |
| for node in ast.walk(tree): | |
| if isinstance(node, ast.Constant) and isinstance(node.value, str) and id(node) not in fstring_children: | |
| new = change(node.value) | |
| if new != node.value: | |
| start = offsets[node.lineno - 1] + node.col_offset | |
| end = offsets[node.end_lineno - 1] + node.end_col_offset | |
| edits.append((start, end, repr(new).encode('utf-8'))) | |
| for start, end, value in sorted(edits, reverse=True): | |
| encoded = encoded[:start] + value + encoded[end:] | |
| result = encoded.decode('utf-8') | |
| ast.parse(result) | |
| return result | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('--plan', type=Path, required=True) | |
| parser.add_argument('--apply', action='store_true') | |
| args = parser.parse_args() | |
| plan = json.loads(args.plan.read_text()) | |
| root = Path(plan['output_root']).parents[1] | |
| mapping = prefix_mapping(plan['records']) | |
| top = ['OURS', 'OURS-Bagel', 'OURS-LatentUM', 'ThinkMorph-OPD', 'docs', | |
| 'experiments/src', 'experiments/tests', 'experiments/sites', | |
| 'projects/VLMEvalKit_Thinkmorph_llm'] | |
| files = [root / name for name in ('AGENTS.md', 'README.md', 'experiments/README.md')] | |
| skip = {'.git', '__pycache__', 'node_modules', 'public', 'dist', '.openai', '.cache'} | |
| extensions = {'.py', '.md', '.sh', '.sbatch', '.json', '.yaml', '.yml', '.toml', '.js', '.ts'} | |
| for relative in top: | |
| for directory, dirs, names in os.walk(root / relative, followlinks=False): | |
| dirs[:] = [d for d in dirs if d not in skip] | |
| files.extend(Path(directory) / name for name in names if Path(name).suffix in extensions) | |
| changes = [] | |
| for file in sorted(set(files)): | |
| if file.name in {'output_layout_migration.py', 'output_relocation.py', | |
| 'rewrite_output_references.py', 'test_output_relocation.py', | |
| 'migrate_v1_outputs.py'}: | |
| continue | |
| if not file.is_file() or file.is_symlink() or file.stat().st_size > 8_000_000: | |
| continue | |
| old = file.read_text(encoding='utf-8') | |
| new = rewrite(old, mapping, python=file.suffix == '.py') | |
| if new == old: | |
| continue | |
| relative = file.relative_to(root) | |
| changes.append(dict(path=str(relative), old_sha256=hashlib.sha256(old.encode()).hexdigest(), | |
| new_sha256=hashlib.sha256(new.encode()).hexdigest())) | |
| if args.apply: | |
| backup = args.plan.parent / 'live-reference-backup' / relative | |
| backup.parent.mkdir(parents=True, exist_ok=True) | |
| if backup.exists(): | |
| raise FileExistsError(backup) | |
| shutil.copy2(file, backup) | |
| # Atomic replacement also avoids modifying any hardlinked snapshot. | |
| temporary = file.with_name('.' + file.name + '.relocation.tmp') | |
| with temporary.open('x', encoding='utf-8') as out: | |
| out.write(new) | |
| os.chmod(temporary, file.stat().st_mode) | |
| os.replace(temporary, file) | |
| if args.apply: | |
| (args.plan.parent / 'live-reference-changes.json').write_text(json.dumps(changes, indent=2) + '\n') | |
| print(json.dumps(dict(files=len(changes), mapping_prefixes=len(mapping), applied=args.apply))) | |
| for row in changes: | |
| print(row['path']) | |
| if __name__ == '__main__': | |
| main() | |
Xet Storage Details
- Size:
- 5.29 kB
- Xet hash:
- 4c8bcff645b581cb853e2ae4f4601b40e81287b5327c6b3997cec6b4a65f6f4a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.