tashkil-scripts / persist2.py
flokymind's picture
Upload persist2.py with huggingface_hub
7fcb24c verified
Raw History Blame Contribute Delete
3.05 kB
import json, os, shutil, subprocess, time, tarfile
TASH = '/kaggle/working/tashkil'
NSH = 4
SLEEP = 900
def shard_books():
root = os.path.join(TASH, 'books_raw')
books = sorted((f[:-4] for f in os.listdir(root) if f.endswith('.txt')), key=int)
return [books[i::NSH] for i in range(NSH)]
def main():
shbooks = shard_books()
print(f'persist: {len(shbooks[0])} books per shard, {sum(len(s) for s in shbooks)} total', flush=True)
last = [None] * NSH
while True:
for sid, books in enumerate(shbooks):
outroot = os.path.join(TASH, 'out')
entries = [os.path.join(outroot, b + '.txt') for b in books
if os.path.exists(os.path.join(outroot, b + '.txt'))]
if not entries:
continue
sz = 0
mt = 0
n = 0
for f in entries:
st = os.stat(f)
sz += st.st_size
mt = max(mt, st.st_mtime)
n += 1
fp = f'{n}:{sz}:{int(mt)}'
if fp == last[sid]:
continue
snap = f'/tmp/snap{sid}'
os.makedirs(snap, exist_ok=True)
pf = os.path.join(TASH, f'progress_shard{sid}.json')
if os.path.exists(pf):
shutil.copy(pf, os.path.join(snap, f'progress_shard{sid}.json'))
meta = {
'title': f'tashkil mishkala shard {sid}',
'id': f'crushlab/tashkil-shard{sid}',
'version': 1,
'description': f'snapshot {time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime())} mishkala step28682',
'licenses': [{'name': 'mit'}],
}
json.dump(meta, open(os.path.join(snap, 'dataset-metadata.json'), 'w'))
tarpath = os.path.join(snap, 'books.tar')
with tarfile.open(tarpath, 'w') as tf:
for f in entries:
tf.add(f, arcname=os.path.basename(f))
flagf = os.path.join(TASH, f'created{sid}.flag')
created = os.path.exists(flagf)
if created:
cmd = ['kaggle', 'datasets', 'version', '-p', snap, '-m', time.strftime('%m%d-%H%M'), '-q']
else:
cmd = ['kaggle', 'datasets', 'create', '-p', snap, '-q']
t0 = time.time()
try:
r = subprocess.run(cmd, capture_output=True, text=True, timeout=5400)
rc, out, err = r.returncode, r.stdout, r.stderr
except subprocess.TimeoutExpired:
rc, out, err = -1, '', 'TIMEOUT'
print(f'shard{sid} upload rc={rc} {time.time()-t0:.0f}s files={n} sz={sz//1048576}MB out={out[-150:]} err={err[-150:]}', flush=True)
if rc == 0:
if not created:
open(flagf, 'w').write('1')
last[sid] = fp
try:
os.remove(tarpath)
except OSError:
pass
time.sleep(SLEEP)
if __name__ == '__main__':
main()