Prahaladha commited on
Commit
3e77d5d
·
verified ·
1 Parent(s): 59a2832

sync learned_computer: 3 files from /dev/shm/neucpu

Browse files
neucpu/hfsync_cpu.sh ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # push learned-computer checkpoints, code, rollouts and data to private HF repos as they appear
3
+ cd /dev/shm/neucpu
4
+ exec /dev/shm/nomas/hyenv/bin/python -u /dev/shm/nomas/hf_sync.py --name learned_computer --every 300 --settle 90 \
5
+ --map /dev/shm/neucpu=model:Prahaladha/learned-computer:neucpu \
6
+ --map /dev/shm/neurv=model:Prahaladha/learned-computer:neurv \
7
+ --map $HOME/neurv_data=dataset:Prahaladha/learned-computer-data:rv32 \
8
+ --exclude 'neucpu/__pycache__/*' --exclude 'neucpu/*.tar' \
9
+ --exclude 'neurv/pkgs/*' --exclude 'neurv/zigcache/*' --exclude 'neurv/uvcache/*' --exclude 'neurv/src/*' --exclude 'neurv/tmp/*' --exclude 'neurv/*.tar' --exclude 'neurv/__pycache__/*' \
10
+ > /dev/shm/neucpu/logs/hfsync.log 2>&1
neucpu/logs/hfsync.log ADDED
The diff for this file is too large to render. See raw diff
 
neucpu/mkrepos.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ from huggingface_hub import HfApi
2
+ api = HfApi(token=open("/dev/shm/nomas/.hf_token").read().strip())
3
+ for rid, rtype in (("Prahaladha/learned-computer", "model"), ("Prahaladha/learned-computer-data", "dataset")):
4
+ u = api.create_repo(rid, repo_type=rtype, private=True, exist_ok=True)
5
+ print("REPO_OK", rid, rtype)