sync learned_computer: 3 files from /dev/shm/neucpu
Browse files- neucpu/hfsync_cpu.sh +10 -0
- neucpu/logs/hfsync.log +0 -0
- neucpu/mkrepos.py +5 -0
neucpu/hfsync_cpu.sh
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# push learned-computer checkpoints, code, rollouts and data to private HF repos as they appear
|
| 3 |
+
cd /dev/shm/neucpu
|
| 4 |
+
exec /dev/shm/nomas/hyenv/bin/python -u /dev/shm/nomas/hf_sync.py --name learned_computer --every 300 --settle 90 \
|
| 5 |
+
--map /dev/shm/neucpu=model:Prahaladha/learned-computer:neucpu \
|
| 6 |
+
--map /dev/shm/neurv=model:Prahaladha/learned-computer:neurv \
|
| 7 |
+
--map $HOME/neurv_data=dataset:Prahaladha/learned-computer-data:rv32 \
|
| 8 |
+
--exclude 'neucpu/__pycache__/*' --exclude 'neucpu/*.tar' \
|
| 9 |
+
--exclude 'neurv/pkgs/*' --exclude 'neurv/zigcache/*' --exclude 'neurv/uvcache/*' --exclude 'neurv/src/*' --exclude 'neurv/tmp/*' --exclude 'neurv/*.tar' --exclude 'neurv/__pycache__/*' \
|
| 10 |
+
> /dev/shm/neucpu/logs/hfsync.log 2>&1
|
neucpu/logs/hfsync.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
neucpu/mkrepos.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from huggingface_hub import HfApi
|
| 2 |
+
api = HfApi(token=open("/dev/shm/nomas/.hf_token").read().strip())
|
| 3 |
+
for rid, rtype in (("Prahaladha/learned-computer", "model"), ("Prahaladha/learned-computer-data", "dataset")):
|
| 4 |
+
u = api.create_repo(rid, repo_type=rtype, private=True, exist_ok=True)
|
| 5 |
+
print("REPO_OK", rid, rtype)
|