Download persist2.py from flokymind/tashkil-scripts: direct link, hf CLI and curl.
- Browser
- Download file 3.05 kB
-
https://huggingface.co/flokymind/tashkil-scripts/resolve/main/persist2.py
- Command line
-
hf download hf://flokymind/tashkil-scripts/persist2.py
-
curl -L -o persist2.py https://huggingface.co/flokymind/tashkil-scripts/resolve/main/persist2.py
3.05 kB
| import json, os, shutil, subprocess, time, tarfile | |
| TASH = '/kaggle/working/tashkil' | |
| NSH = 4 | |
| SLEEP = 900 | |
| def shard_books(): | |
| root = os.path.join(TASH, 'books_raw') | |
| books = sorted((f[:-4] for f in os.listdir(root) if f.endswith('.txt')), key=int) | |
| return [books[i::NSH] for i in range(NSH)] | |
| def main(): | |
| shbooks = shard_books() | |
| print(f'persist: {len(shbooks[0])} books per shard, {sum(len(s) for s in shbooks)} total', flush=True) | |
| last = [None] * NSH | |
| while True: | |
| for sid, books in enumerate(shbooks): | |
| outroot = os.path.join(TASH, 'out') | |
| entries = [os.path.join(outroot, b + '.txt') for b in books | |
| if os.path.exists(os.path.join(outroot, b + '.txt'))] | |
| if not entries: | |
| continue | |
| sz = 0 | |
| mt = 0 | |
| n = 0 | |
| for f in entries: | |
| st = os.stat(f) | |
| sz += st.st_size | |
| mt = max(mt, st.st_mtime) | |
| n += 1 | |
| fp = f'{n}:{sz}:{int(mt)}' | |
| if fp == last[sid]: | |
| continue | |
| snap = f'/tmp/snap{sid}' | |
| os.makedirs(snap, exist_ok=True) | |
| pf = os.path.join(TASH, f'progress_shard{sid}.json') | |
| if os.path.exists(pf): | |
| shutil.copy(pf, os.path.join(snap, f'progress_shard{sid}.json')) | |
| meta = { | |
| 'title': f'tashkil mishkala shard {sid}', | |
| 'id': f'crushlab/tashkil-shard{sid}', | |
| 'version': 1, | |
| 'description': f'snapshot {time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime())} mishkala step28682', | |
| 'licenses': [{'name': 'mit'}], | |
| } | |
| json.dump(meta, open(os.path.join(snap, 'dataset-metadata.json'), 'w')) | |
| tarpath = os.path.join(snap, 'books.tar') | |
| with tarfile.open(tarpath, 'w') as tf: | |
| for f in entries: | |
| tf.add(f, arcname=os.path.basename(f)) | |
| flagf = os.path.join(TASH, f'created{sid}.flag') | |
| created = os.path.exists(flagf) | |
| if created: | |
| cmd = ['kaggle', 'datasets', 'version', '-p', snap, '-m', time.strftime('%m%d-%H%M'), '-q'] | |
| else: | |
| cmd = ['kaggle', 'datasets', 'create', '-p', snap, '-q'] | |
| t0 = time.time() | |
| try: | |
| r = subprocess.run(cmd, capture_output=True, text=True, timeout=5400) | |
| rc, out, err = r.returncode, r.stdout, r.stderr | |
| except subprocess.TimeoutExpired: | |
| rc, out, err = -1, '', 'TIMEOUT' | |
| print(f'shard{sid} upload rc={rc} {time.time()-t0:.0f}s files={n} sz={sz//1048576}MB out={out[-150:]} err={err[-150:]}', flush=True) | |
| if rc == 0: | |
| if not created: | |
| open(flagf, 'w').write('1') | |
| last[sid] = fp | |
| try: | |
| os.remove(tarpath) | |
| except OSError: | |
| pass | |
| time.sleep(SLEEP) | |
| if __name__ == '__main__': | |
| main() | |