eyad-silx commited on
Commit
ebf214c
·
verified ·
1 Parent(s): a532b21

push artifacts/scripts/hf_push.py

Browse files
Files changed (1) hide show
  1. artifacts/scripts/hf_push.py +26 -8
artifacts/scripts/hf_push.py CHANGED
@@ -11,6 +11,10 @@ from pathlib import Path
11
  from huggingface_hub import HfApi
12
 
13
  REPO = "eyad-silx/progress"
 
 
 
 
14
  TOKEN = Path("/root/.hf_token").read_text().strip()
15
  STATE_P = Path("/root/.hf_push_state.json")
16
  LOG_P = Path("/root/hf_push.log")
@@ -107,11 +111,23 @@ def main():
107
  continue
108
  try:
109
  d = sha256(local)
110
- api.upload_file(path_or_fileobj=local, path_in_repo=repo, repo_id=REPO,
111
- repo_type="model", commit_message=f"push {repo}")
112
- state[repo] = {"fp": fp, "sha256": d, "size": st.st_size, "uploaded_at": now()}
 
 
 
 
 
 
 
 
 
 
 
113
  save_state(state)
114
- log("uploaded", local=local, repo=repo, sha256=d, size=st.st_size)
 
115
  results.append((repo, "uploaded"))
116
  except Exception as e:
117
  log("error", local=local, repo=repo, err=repr(e)[:500])
@@ -119,12 +135,14 @@ def main():
119
 
120
  # regenerate docs (manifest / progress / readme) from state
121
  try:
122
- manifest = ["# Manifest — eyad-silx/progress", "",
123
- "| repo path | size (bytes) | sha256 | uploaded (UTC) |",
124
- "|---|---|---|---|"]
 
 
125
  for repo in sorted(state):
126
  v = state[repo]
127
- manifest.append(f"| `{repo}` | {v['size']:,} | `{v['sha256']}` | {v['uploaded_at']} |")
128
  Path("/tmp/MANIFEST.md").write_text("\n".join(manifest) + "\n")
129
  api.upload_file(path_or_fileobj="/tmp/MANIFEST.md", path_in_repo="MANIFEST.md",
130
  repo_id=REPO, repo_type="model", commit_message="update MANIFEST.md")
 
11
  from huggingface_hub import HfApi
12
 
13
  REPO = "eyad-silx/progress"
14
+ # Free-plan private-storage cap (~20 GiB) on eyad-silx fills up with 3 GGUFs.
15
+ # Files that hit the cap overflow to this private org repo (token has repo.write):
16
+ OVERFLOW_REPO = "silx-ai/Progress"
17
+ OVERFLOW_PREFIX = "ingot-backup/"
18
  TOKEN = Path("/root/.hf_token").read_text().strip()
19
  STATE_P = Path("/root/.hf_push_state.json")
20
  LOG_P = Path("/root/hf_push.log")
 
111
  continue
112
  try:
113
  d = sha256(local)
114
+ dest = REPO
115
+ try:
116
+ api.upload_file(path_or_fileobj=local, path_in_repo=repo, repo_id=REPO,
117
+ repo_type="model", commit_message=f"push {repo}")
118
+ except Exception as e1:
119
+ msg = str(e1)
120
+ if "storage limit" not in msg.lower():
121
+ raise
122
+ api.upload_file(path_or_fileobj=local, path_in_repo=OVERFLOW_PREFIX + repo,
123
+ repo_id=OVERFLOW_REPO, repo_type="model",
124
+ commit_message=f"overflow push {repo} (main repo private storage full)")
125
+ dest = OVERFLOW_REPO
126
+ log("overflowed", local=local, repo=repo, to=OVERFLOW_REPO, sha256=d, size=st.st_size)
127
+ state[repo] = {"fp": fp, "sha256": d, "size": st.st_size, "uploaded_at": now(), "dest": dest}
128
  save_state(state)
129
+ if dest == REPO:
130
+ log("uploaded", local=local, repo=repo, sha256=d, size=st.st_size)
131
  results.append((repo, "uploaded"))
132
  except Exception as e:
133
  log("error", local=local, repo=repo, err=repr(e)[:500])
 
135
 
136
  # regenerate docs (manifest / progress / readme) from state
137
  try:
138
+ manifest = ["# Manifest — eyad-silx/progress (+ overflow repo silx-ai/Progress)", "",
139
+ "Files with `stored in` = silx-ai/Progress live at `ingot-backup/<repo path>` in that repo",
140
+ "(eyad-silx free-plan private-storage cap ~20 GiB; overflow keeps everything private).", "",
141
+ "| repo path | size (bytes) | sha256 | uploaded (UTC) | stored in |",
142
+ "|---|---|---|---|---|"]
143
  for repo in sorted(state):
144
  v = state[repo]
145
+ manifest.append(f"| `{repo}` | {v['size']:,} | `{v['sha256']}` | {v['uploaded_at']} | {v.get('dest', REPO)} |")
146
  Path("/tmp/MANIFEST.md").write_text("\n".join(manifest) + "\n")
147
  api.upload_file(path_or_fileobj="/tmp/MANIFEST.md", path_in_repo="MANIFEST.md",
148
  repo_id=REPO, repo_type="model", commit_message="update MANIFEST.md")