File size: 3,047 Bytes
7fcb24c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
import json, os, shutil, subprocess, time, tarfile

TASH = '/kaggle/working/tashkil'
NSH = 4
SLEEP = 900

def shard_books():
    root = os.path.join(TASH, 'books_raw')
    books = sorted((f[:-4] for f in os.listdir(root) if f.endswith('.txt')), key=int)
    return [books[i::NSH] for i in range(NSH)]

def main():
    shbooks = shard_books()
    print(f'persist: {len(shbooks[0])} books per shard, {sum(len(s) for s in shbooks)} total', flush=True)
    last = [None] * NSH
    while True:
        for sid, books in enumerate(shbooks):
            outroot = os.path.join(TASH, 'out')
            entries = [os.path.join(outroot, b + '.txt') for b in books
                       if os.path.exists(os.path.join(outroot, b + '.txt'))]
            if not entries:
                continue
            sz = 0
            mt = 0
            n = 0
            for f in entries:
                st = os.stat(f)
                sz += st.st_size
                mt = max(mt, st.st_mtime)
                n += 1
            fp = f'{n}:{sz}:{int(mt)}'
            if fp == last[sid]:
                continue
            snap = f'/tmp/snap{sid}'
            os.makedirs(snap, exist_ok=True)
            pf = os.path.join(TASH, f'progress_shard{sid}.json')
            if os.path.exists(pf):
                shutil.copy(pf, os.path.join(snap, f'progress_shard{sid}.json'))
            meta = {
                'title': f'tashkil mishkala shard {sid}',
                'id': f'crushlab/tashkil-shard{sid}',
                'version': 1,
                'description': f'snapshot {time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime())} mishkala step28682',
                'licenses': [{'name': 'mit'}],
            }
            json.dump(meta, open(os.path.join(snap, 'dataset-metadata.json'), 'w'))
            tarpath = os.path.join(snap, 'books.tar')
            with tarfile.open(tarpath, 'w') as tf:
                for f in entries:
                    tf.add(f, arcname=os.path.basename(f))
            flagf = os.path.join(TASH, f'created{sid}.flag')
            created = os.path.exists(flagf)
            if created:
                cmd = ['kaggle', 'datasets', 'version', '-p', snap, '-m', time.strftime('%m%d-%H%M'), '-q']
            else:
                cmd = ['kaggle', 'datasets', 'create', '-p', snap, '-q']
            t0 = time.time()
            try:
                r = subprocess.run(cmd, capture_output=True, text=True, timeout=5400)
                rc, out, err = r.returncode, r.stdout, r.stderr
            except subprocess.TimeoutExpired:
                rc, out, err = -1, '', 'TIMEOUT'
            print(f'shard{sid} upload rc={rc} {time.time()-t0:.0f}s files={n} sz={sz//1048576}MB out={out[-150:]} err={err[-150:]}', flush=True)
            if rc == 0:
                if not created:
                    open(flagf, 'w').write('1')
                last[sid] = fp
            try:
                os.remove(tarpath)
            except OSError:
                pass
        time.sleep(SLEEP)

if __name__ == '__main__':
    main()