import json, os, shutil, subprocess, time, tarfile TASH = '/kaggle/working/tashkil' NSH = 4 SLEEP = 900 def shard_books(): root = os.path.join(TASH, 'books_raw') books = sorted((f[:-4] for f in os.listdir(root) if f.endswith('.txt')), key=int) return [books[i::NSH] for i in range(NSH)] def main(): shbooks = shard_books() print(f'persist: {len(shbooks[0])} books per shard, {sum(len(s) for s in shbooks)} total', flush=True) last = [None] * NSH while True: for sid, books in enumerate(shbooks): outroot = os.path.join(TASH, 'out') entries = [os.path.join(outroot, b + '.txt') for b in books if os.path.exists(os.path.join(outroot, b + '.txt'))] if not entries: continue sz = 0 mt = 0 n = 0 for f in entries: st = os.stat(f) sz += st.st_size mt = max(mt, st.st_mtime) n += 1 fp = f'{n}:{sz}:{int(mt)}' if fp == last[sid]: continue snap = f'/tmp/snap{sid}' os.makedirs(snap, exist_ok=True) pf = os.path.join(TASH, f'progress_shard{sid}.json') if os.path.exists(pf): shutil.copy(pf, os.path.join(snap, f'progress_shard{sid}.json')) meta = { 'title': f'tashkil mishkala shard {sid}', 'id': f'crushlab/tashkil-shard{sid}', 'version': 1, 'description': f'snapshot {time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime())} mishkala step28682', 'licenses': [{'name': 'mit'}], } json.dump(meta, open(os.path.join(snap, 'dataset-metadata.json'), 'w')) tarpath = os.path.join(snap, 'books.tar') with tarfile.open(tarpath, 'w') as tf: for f in entries: tf.add(f, arcname=os.path.basename(f)) flagf = os.path.join(TASH, f'created{sid}.flag') created = os.path.exists(flagf) if created: cmd = ['kaggle', 'datasets', 'version', '-p', snap, '-m', time.strftime('%m%d-%H%M'), '-q'] else: cmd = ['kaggle', 'datasets', 'create', '-p', snap, '-q'] t0 = time.time() try: r = subprocess.run(cmd, capture_output=True, text=True, timeout=5400) rc, out, err = r.returncode, r.stdout, r.stderr except subprocess.TimeoutExpired: rc, out, err = -1, '', 'TIMEOUT' print(f'shard{sid} upload rc={rc} {time.time()-t0:.0f}s files={n} sz={sz//1048576}MB out={out[-150:]} err={err[-150:]}', flush=True) if rc == 0: if not created: open(flagf, 'w').write('1') last[sid] = fp try: os.remove(tarpath) except OSError: pass time.sleep(SLEEP) if __name__ == '__main__': main()