Spaces:
Runtime error
Runtime error
Update scripts/sync_hf.py
Browse files- scripts/sync_hf.py +85 -68
scripts/sync_hf.py
CHANGED
|
@@ -1,14 +1,14 @@
|
|
| 1 |
#!/usr/bin/env python3
|
| 2 |
"""
|
| 3 |
-
OpenClaw HF Spaces Persistence
|
| 4 |
=====================================================
|
| 5 |
|
| 6 |
Simplified persistence: upload/download the entire ~/.openclaw directory
|
| 7 |
as-is to/from a Hugging Face Dataset repo.
|
| 8 |
|
| 9 |
-
- Startup: snapshot_download
|
| 10 |
-
- Periodic: upload_folder
|
| 11 |
-
- Shutdown: final upload_folder
|
| 12 |
"""
|
| 13 |
|
| 14 |
import os
|
|
@@ -40,7 +40,7 @@ _logging.getLogger("filelock").setLevel(_logging.WARNING)
|
|
| 40 |
|
| 41 |
from huggingface_hub import HfApi, snapshot_download
|
| 42 |
|
| 43 |
-
#
|
| 44 |
|
| 45 |
class TeeLogger:
|
| 46 |
"""Duplicate output to stream and file."""
|
|
@@ -57,7 +57,7 @@ class TeeLogger:
|
|
| 57 |
def fileno(self):
|
| 58 |
return self.stream.fileno()
|
| 59 |
|
| 60 |
-
#
|
| 61 |
|
| 62 |
HF_TOKEN = os.environ.get("HF_TOKEN")
|
| 63 |
OPENCLAW_HOME = Path.home() / ".openclaw"
|
|
@@ -76,7 +76,7 @@ GATEWAY_TOKEN = os.environ.get("GATEWAY_TOKEN", "huggingclaw")
|
|
| 76 |
AGENT_NAME = os.environ.get("AGENT_NAME", "HuggingClaw")
|
| 77 |
A2A_PEERS = os.environ.get("A2A_PEERS", "") # comma-separated peer URLs
|
| 78 |
|
| 79 |
-
# Default model
|
| 80 |
OPENCLAW_DEFAULT_MODEL = os.environ.get("OPENCLAW_DEFAULT_MODEL", "")
|
| 81 |
|
| 82 |
# HF Spaces built-in env vars (auto-set by HF runtime)
|
|
@@ -91,16 +91,16 @@ AUTO_CREATE_DATASET = os.environ.get("AUTO_CREATE_DATASET", "false").lower() in
|
|
| 91 |
# This ensures each duplicated Space gets its own dataset automatically.
|
| 92 |
HF_REPO_ID = os.environ.get("OPENCLAW_DATASET_REPO", "")
|
| 93 |
if not HF_REPO_ID and SPACE_ID:
|
| 94 |
-
# SPACE_ID = "username/SpaceName"
|
| 95 |
HF_REPO_ID = f"{SPACE_ID}-data"
|
| 96 |
-
print(f"[SYNC] OPENCLAW_DATASET_REPO not set
|
| 97 |
elif not HF_REPO_ID and HF_TOKEN:
|
| 98 |
# Fallback: no SPACE_ID (local Docker), derive from HF_TOKEN username
|
| 99 |
try:
|
| 100 |
_api = HfApi(token=HF_TOKEN)
|
| 101 |
_username = _api.whoami()["name"]
|
| 102 |
HF_REPO_ID = f"{_username}/HuggingClaw-data"
|
| 103 |
-
print(f"[SYNC] OPENCLAW_DATASET_REPO not set
|
| 104 |
del _api, _username
|
| 105 |
except Exception as e:
|
| 106 |
print(f"[SYNC] WARNING: Could not derive username from HF_TOKEN: {e}")
|
|
@@ -112,7 +112,7 @@ log_dir.mkdir(parents=True, exist_ok=True)
|
|
| 112 |
sys.stdout = TeeLogger(log_dir / "sync.log", sys.stdout)
|
| 113 |
sys.stderr = sys.stdout
|
| 114 |
|
| 115 |
-
#
|
| 116 |
#
|
| 117 |
# HF Spaces blocks DNS for api.telegram.org. grammY uses Node 22's built-in
|
| 118 |
# fetch (undici) which bypasses dns.lookup patching and /etc/hosts.
|
|
@@ -123,7 +123,7 @@ sys.stderr = sys.stdout
|
|
| 123 |
# 2. telegram-proxy.cjs (loaded via NODE_OPTIONS --require) intercepts
|
| 124 |
# globalThis.fetch() and rewrites api.telegram.org URLs to the mirror.
|
| 125 |
#
|
| 126 |
-
# This works without a bot token
|
| 127 |
# If a bot token IS available, we do a full getMe verification.
|
| 128 |
|
| 129 |
# User can force a specific base via env var (skip auto-probe)
|
|
@@ -140,7 +140,7 @@ def probe_telegram_api(timeout: int = 8) -> str:
|
|
| 140 |
"""Probe Telegram API endpoints and return the first reachable one.
|
| 141 |
|
| 142 |
First checks if official api.telegram.org is reachable (HTTP level).
|
| 143 |
-
If not, tries mirrors. No bot token required
|
| 144 |
Returns the working base URL (without trailing slash), or "" if all fail.
|
| 145 |
"""
|
| 146 |
ctx = ssl.create_default_context()
|
|
@@ -149,22 +149,22 @@ def probe_telegram_api(timeout: int = 8) -> str:
|
|
| 149 |
try:
|
| 150 |
req = urllib.request.Request(url, method="GET")
|
| 151 |
resp = urllib.request.urlopen(req, timeout=timeout, context=ctx)
|
| 152 |
-
print(f"[TELEGRAM]
|
| 153 |
return base.rstrip("/")
|
| 154 |
except urllib.error.HTTPError as e:
|
| 155 |
# HTTP error (4xx/5xx) still means the host IS reachable
|
| 156 |
-
print(f"[TELEGRAM]
|
| 157 |
return base.rstrip("/")
|
| 158 |
except Exception as e:
|
| 159 |
reason = str(e)[:80]
|
| 160 |
-
print(f"[TELEGRAM]
|
| 161 |
continue
|
| 162 |
|
| 163 |
print("[TELEGRAM] WARNING: All API endpoints unreachable!")
|
| 164 |
return ""
|
| 165 |
|
| 166 |
|
| 167 |
-
#
|
| 168 |
|
| 169 |
class OpenClawFullSync:
|
| 170 |
"""Upload/download the entire ~/.openclaw directory to HF Dataset."""
|
|
@@ -186,7 +186,7 @@ class OpenClawFullSync:
|
|
| 186 |
self.api = HfApi(token=HF_TOKEN)
|
| 187 |
self.dataset_exists = self._ensure_repo_exists()
|
| 188 |
|
| 189 |
-
#
|
| 190 |
|
| 191 |
def _ensure_repo_exists(self):
|
| 192 |
"""Check if dataset repo exists; auto-create only when AUTO_CREATE_DATASET=true AND HF_TOKEN is set."""
|
|
@@ -200,23 +200,23 @@ class OpenClawFullSync:
|
|
| 200 |
print(f"[SYNC] Set AUTO_CREATE_DATASET=true to auto-create.")
|
| 201 |
print(f"[SYNC] Persistence disabled (app will still run normally).")
|
| 202 |
return False
|
| 203 |
-
print(f"[SYNC] Dataset repo NOT found: {HF_REPO_ID}
|
| 204 |
try:
|
| 205 |
self.api.create_repo(
|
| 206 |
repo_id=HF_REPO_ID,
|
| 207 |
repo_type="dataset",
|
| 208 |
private=True,
|
| 209 |
)
|
| 210 |
-
print(f"[SYNC]
|
| 211 |
return True
|
| 212 |
except Exception as e:
|
| 213 |
-
print(f"[SYNC]
|
| 214 |
return False
|
| 215 |
|
| 216 |
-
#
|
| 217 |
|
| 218 |
def load_from_repo(self):
|
| 219 |
-
"""Download from dataset
|
| 220 |
if not self.enabled:
|
| 221 |
print("[SYNC] Persistence disabled - skipping restore")
|
| 222 |
self._ensure_default_config()
|
|
@@ -229,7 +229,7 @@ class OpenClawFullSync:
|
|
| 229 |
self._patch_config()
|
| 230 |
return
|
| 231 |
|
| 232 |
-
print(f"[SYNC]
|
| 233 |
OPENCLAW_HOME.mkdir(parents=True, exist_ok=True)
|
| 234 |
|
| 235 |
try:
|
|
@@ -259,22 +259,22 @@ class OpenClawFullSync:
|
|
| 259 |
dest = OPENCLAW_HOME / rel
|
| 260 |
dest.parent.mkdir(parents=True, exist_ok=True)
|
| 261 |
shutil.copy2(str(item), str(dest))
|
| 262 |
-
print("[SYNC]
|
| 263 |
else:
|
| 264 |
print("[SYNC] Downloaded snapshot but dir not found. Starting fresh.")
|
| 265 |
|
| 266 |
except Exception as e:
|
| 267 |
-
print(f"[SYNC]
|
| 268 |
traceback.print_exc()
|
| 269 |
|
| 270 |
# Patch config after restore
|
| 271 |
self._patch_config()
|
| 272 |
self._debug_list_files()
|
| 273 |
|
| 274 |
-
#
|
| 275 |
|
| 276 |
def save_to_repo(self):
|
| 277 |
-
"""Upload entire ~/.openclaw directory
|
| 278 |
if not self.enabled:
|
| 279 |
return
|
| 280 |
if not OPENCLAW_HOME.exists():
|
|
@@ -286,7 +286,7 @@ class OpenClawFullSync:
|
|
| 286 |
print(f"[SYNC] Dataset {HF_REPO_ID} unavailable - skipping save")
|
| 287 |
return
|
| 288 |
|
| 289 |
-
print(f"[SYNC]
|
| 290 |
|
| 291 |
try:
|
| 292 |
# Count files to upload (no per-file logging to reduce noise)
|
|
@@ -310,16 +310,16 @@ class OpenClawFullSync:
|
|
| 310 |
repo_id=HF_REPO_ID,
|
| 311 |
repo_type="dataset",
|
| 312 |
token=HF_TOKEN,
|
| 313 |
-
commit_message=f"Sync .openclaw
|
| 314 |
ignore_patterns=[
|
| 315 |
-
"*.log", # Log files (sync.log, startup.log)
|
| 316 |
-
"*.lock", # Lock files
|
| 317 |
"*.tmp", # Temp files
|
| 318 |
"*.pid", # PID files
|
| 319 |
"__pycache__", # Python cache
|
| 320 |
],
|
| 321 |
)
|
| 322 |
-
print(f"[SYNC]
|
| 323 |
|
| 324 |
# Verify (summary only)
|
| 325 |
try:
|
|
@@ -330,10 +330,10 @@ class OpenClawFullSync:
|
|
| 330 |
pass
|
| 331 |
|
| 332 |
except Exception as e:
|
| 333 |
-
print(f"[SYNC]
|
| 334 |
traceback.print_exc()
|
| 335 |
|
| 336 |
-
#
|
| 337 |
|
| 338 |
def _ensure_default_config(self):
|
| 339 |
config_path = OPENCLAW_HOME / "openclaw.json"
|
|
@@ -349,7 +349,7 @@ class OpenClawFullSync:
|
|
| 349 |
# Set gateway token
|
| 350 |
if "gateway" in cfg:
|
| 351 |
cfg["gateway"]["auth"] = {"token": GATEWAY_TOKEN}
|
| 352 |
-
# Don't manage providers
|
| 353 |
# Just remove template entries with unresolved ${...} placeholders
|
| 354 |
# so OpenClaw starts clean and discovers providers from env vars.
|
| 355 |
providers = cfg.get("models", {}).get("providers", {})
|
|
@@ -392,7 +392,7 @@ class OpenClawFullSync:
|
|
| 392 |
data = json.load(f)
|
| 393 |
print("[SYNC] Config parsed OK.")
|
| 394 |
except (json.JSONDecodeError, Exception) as e:
|
| 395 |
-
# Config is corrupt
|
| 396 |
print(f"[SYNC] Config JSON is corrupt: {e}")
|
| 397 |
backup = config_path.with_suffix(f".corrupt_{int(time.time())}")
|
| 398 |
try:
|
|
@@ -447,7 +447,7 @@ class OpenClawFullSync:
|
|
| 447 |
data.setdefault("agents", {}).setdefault("defaults", {}).setdefault("model", {})
|
| 448 |
data.setdefault("session", {})["scope"] = "global"
|
| 449 |
|
| 450 |
-
#
|
| 451 |
# The persisted config may contain template placeholders such as
|
| 452 |
# __OPENROUTER_API_KEY__ or __NVIDIA_API_KEY__. Resolve those from
|
| 453 |
# the current HF Space environment instead of treating them as
|
|
@@ -469,12 +469,12 @@ class OpenClawFullSync:
|
|
| 469 |
if pid not in restored_providers:
|
| 470 |
if env_key:
|
| 471 |
restored_providers[pid] = {**provider_defaults[pid], "apiKey": env_key}
|
| 472 |
-
print(f"[SYNC]
|
| 473 |
continue
|
| 474 |
current_key = str(restored_providers[pid].get("apiKey", "")).strip()
|
| 475 |
if env_key and (not current_key or current_key.startswith("${") or current_key.startswith("__")):
|
| 476 |
restored_providers[pid]["apiKey"] = env_key
|
| 477 |
-
print(f"[SYNC]
|
| 478 |
elif not env_key and (not current_key or current_key.startswith("${") or current_key.startswith("__")):
|
| 479 |
del restored_providers[pid]
|
| 480 |
print(f"[SYNC] Removed provider '{pid}' (no API key available in {env_name})")
|
|
@@ -482,7 +482,7 @@ class OpenClawFullSync:
|
|
| 482 |
if OPENCLAW_DEFAULT_MODEL:
|
| 483 |
data["agents"]["defaults"]["model"]["primary"] = OPENCLAW_DEFAULT_MODEL
|
| 484 |
|
| 485 |
-
#
|
| 486 |
data["acp"] = {
|
| 487 |
"enabled": True,
|
| 488 |
"backend": "acpx",
|
|
@@ -501,7 +501,7 @@ class OpenClawFullSync:
|
|
| 501 |
plugin_allow.append("a2a-gateway")
|
| 502 |
data["plugins"]["allow"] = plugin_allow
|
| 503 |
|
| 504 |
-
#
|
| 505 |
data["plugins"]["entries"]["acpx"] = {
|
| 506 |
"enabled": True,
|
| 507 |
"config": {
|
|
@@ -509,7 +509,7 @@ class OpenClawFullSync:
|
|
| 509 |
}
|
| 510 |
}
|
| 511 |
|
| 512 |
-
#
|
| 513 |
CODING_TARGET_SPACE = os.environ.get("CODING_AGENT_TARGET_SPACE", "")
|
| 514 |
CODING_TARGET_DATASET = os.environ.get("CODING_AGENT_TARGET_DATASET", "")
|
| 515 |
if CODING_TARGET_SPACE:
|
|
@@ -528,7 +528,7 @@ class OpenClawFullSync:
|
|
| 528 |
elif isinstance(data["plugins"]["entries"]["telegram"], dict):
|
| 529 |
data["plugins"]["entries"]["telegram"]["enabled"] = True
|
| 530 |
|
| 531 |
-
#
|
| 532 |
if A2A_PEERS:
|
| 533 |
peers = []
|
| 534 |
for peer_url in A2A_PEERS.split(","):
|
|
@@ -540,7 +540,7 @@ class OpenClawFullSync:
|
|
| 540 |
"name": name,
|
| 541 |
"agentCardUrl": f"{peer_url}/.well-known/agent-card.json"
|
| 542 |
})
|
| 543 |
-
print(f"[SYNC] A2A peer: {name}
|
| 544 |
|
| 545 |
data["plugins"]["entries"]["a2a-gateway"] = {
|
| 546 |
"enabled": True,
|
|
@@ -558,23 +558,23 @@ class OpenClawFullSync:
|
|
| 558 |
}
|
| 559 |
print(f"[SYNC] A2A gateway configured: name={AGENT_NAME}, port=18800, peers={len(peers)}")
|
| 560 |
|
| 561 |
-
#
|
| 562 |
-
# Personal bot on HF Spaces
|
| 563 |
tg_ch = data.setdefault("channels", {}).setdefault("telegram", {})
|
| 564 |
tg_ch["dmPolicy"] = "open"
|
| 565 |
tg_ch["allowFrom"] = ["*"]
|
| 566 |
tg_ch["configWrites"] = True
|
| 567 |
print("[SYNC] Set channels.telegram: dmPolicy=open, allowFrom=[*], configWrites=true")
|
| 568 |
|
| 569 |
-
#
|
| 570 |
-
# Probe is done in run_openclaw()
|
| 571 |
# for the telegram-proxy.cjs preload script to intercept fetch().
|
| 572 |
|
| 573 |
with open(config_path, "w") as f:
|
| 574 |
json.dump(data, f, indent=2)
|
| 575 |
print("[SYNC] Config patched and saved.")
|
| 576 |
|
| 577 |
-
#
|
| 578 |
workspace_dir = Path(OPENCLAW_HOME) / "workspace"
|
| 579 |
workspace_dir.mkdir(parents=True, exist_ok=True)
|
| 580 |
templates_dir = Path("/home/node/workspace-templates")
|
|
@@ -618,17 +618,17 @@ class OpenClawFullSync:
|
|
| 618 |
except Exception as e:
|
| 619 |
print(f"[SYNC] listing failed: {e}")
|
| 620 |
|
| 621 |
-
#
|
| 622 |
|
| 623 |
def background_sync_loop(self, stop_event):
|
| 624 |
print(f"[SYNC] Background sync started (interval={SYNC_INTERVAL}s)")
|
| 625 |
while not stop_event.is_set():
|
| 626 |
if stop_event.wait(timeout=SYNC_INTERVAL):
|
| 627 |
break
|
| 628 |
-
print(f"[SYNC]
|
| 629 |
self.save_to_repo()
|
| 630 |
|
| 631 |
-
#
|
| 632 |
|
| 633 |
def run_doctor_fix(self):
|
| 634 |
"""Run OpenClaw doctor --fix in headless mode before starting the gateway."""
|
|
@@ -647,9 +647,26 @@ class OpenClawFullSync:
|
|
| 647 |
print("[SYNC] Doctor output will be streamed below.")
|
| 648 |
env = os.environ.copy()
|
| 649 |
env["NODE_ENV"] = "production"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 650 |
timeout = int(os.environ.get("DOCTOR_TIMEOUT", "300"))
|
| 651 |
try:
|
| 652 |
-
process = subprocess.Popen(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 653 |
start_time = time.time()
|
| 654 |
while True:
|
| 655 |
return_code = process.poll()
|
|
@@ -657,12 +674,12 @@ class OpenClawFullSync:
|
|
| 657 |
elapsed = time.time() - start_time
|
| 658 |
print(f"[SYNC] Doctor finished in {elapsed:.1f}s with exit code {return_code}")
|
| 659 |
if return_code == 0:
|
| 660 |
-
print("[SYNC]
|
| 661 |
return True
|
| 662 |
-
print(f"[SYNC]
|
| 663 |
return False
|
| 664 |
if time.time() - start_time > timeout:
|
| 665 |
-
print(f"[SYNC]
|
| 666 |
try:
|
| 667 |
process.terminate()
|
| 668 |
process.wait(timeout=10)
|
|
@@ -673,11 +690,11 @@ class OpenClawFullSync:
|
|
| 673 |
return False
|
| 674 |
time.sleep(0.5)
|
| 675 |
except Exception as e:
|
| 676 |
-
print(f"[SYNC]
|
| 677 |
traceback.print_exc()
|
| 678 |
return False
|
| 679 |
|
| 680 |
-
#
|
| 681 |
|
| 682 |
def run_openclaw(self):
|
| 683 |
log_file = OPENCLAW_HOME / "workspace" / "startup.log"
|
|
@@ -715,7 +732,7 @@ class OpenClawFullSync:
|
|
| 715 |
# Pass entire environment to OpenClaw (all API keys are already in os.environ)
|
| 716 |
env = os.environ.copy()
|
| 717 |
|
| 718 |
-
#
|
| 719 |
# Determine working Telegram API endpoint and set env var for
|
| 720 |
# telegram-proxy.cjs to intercept fetch() calls.
|
| 721 |
if TELEGRAM_API_BASE:
|
|
@@ -730,12 +747,12 @@ class OpenClawFullSync:
|
|
| 730 |
print(f"[TELEGRAM] Set TELEGRAM_API_ROOT={tg_root}")
|
| 731 |
print(f"[TELEGRAM] telegram-proxy.cjs will redirect fetch() calls")
|
| 732 |
elif tg_root:
|
| 733 |
-
print("[TELEGRAM] Official API reachable
|
| 734 |
else:
|
| 735 |
-
print("[TELEGRAM] No reachable endpoint found
|
| 736 |
try:
|
| 737 |
# Use Popen without shell to avoid pipe issues
|
| 738 |
-
# auth disabled in config
|
| 739 |
process = subprocess.Popen(
|
| 740 |
entry_cmd,
|
| 741 |
cwd=str(APP_DIR),
|
|
@@ -759,7 +776,7 @@ class OpenClawFullSync:
|
|
| 759 |
continue
|
| 760 |
# Skip progress bars and download noise
|
| 761 |
if any(skip in stripped for skip in [
|
| 762 |
-
'Downloading', 'Fetching', '%|', '
|
| 763 |
'Already cached', 'Using cache', 'tokenizer',
|
| 764 |
'.safetensors', 'model-', 'shard',
|
| 765 |
]):
|
|
@@ -782,7 +799,7 @@ class OpenClawFullSync:
|
|
| 782 |
traceback.print_exc()
|
| 783 |
return None
|
| 784 |
|
| 785 |
-
#
|
| 786 |
|
| 787 |
def main():
|
| 788 |
try:
|
|
@@ -825,7 +842,7 @@ def main():
|
|
| 825 |
t0 = time.time()
|
| 826 |
process = sync.run_openclaw()
|
| 827 |
print(f"[TIMER] run_openclaw launch: {time.time() - t0:.1f}s")
|
| 828 |
-
print(f"[TIMER] Total startup (init
|
| 829 |
|
| 830 |
# 4. Start conversation-loop (RUN_ORCHESTRATOR=1, runs on God Space)
|
| 831 |
# OFFICE_MODE=1 is for frontend serving only (Home Space)
|
|
@@ -866,7 +883,7 @@ def main():
|
|
| 866 |
conv_thread = threading.Thread(target=run_conversation_loop_forever, daemon=True)
|
| 867 |
conv_thread.start()
|
| 868 |
else:
|
| 869 |
-
print("[SYNC] RUN_ORCHESTRATOR not set
|
| 870 |
|
| 871 |
# Signal handler
|
| 872 |
def handle_signal(sig, frame):
|
|
@@ -916,4 +933,4 @@ def main():
|
|
| 916 |
|
| 917 |
|
| 918 |
if __name__ == "__main__":
|
| 919 |
-
main()
|
|
|
|
| 1 |
#!/usr/bin/env python3
|
| 2 |
"""
|
| 3 |
+
OpenClaw HF Spaces Persistence — Full Directory Sync
|
| 4 |
=====================================================
|
| 5 |
|
| 6 |
Simplified persistence: upload/download the entire ~/.openclaw directory
|
| 7 |
as-is to/from a Hugging Face Dataset repo.
|
| 8 |
|
| 9 |
+
- Startup: snapshot_download → ~/.openclaw
|
| 10 |
+
- Periodic: upload_folder → dataset openclaw_data/
|
| 11 |
+
- Shutdown: final upload_folder → dataset openclaw_data/
|
| 12 |
"""
|
| 13 |
|
| 14 |
import os
|
|
|
|
| 40 |
|
| 41 |
from huggingface_hub import HfApi, snapshot_download
|
| 42 |
|
| 43 |
+
# ── Logging helper ──────────────────────────────────────────────────────────
|
| 44 |
|
| 45 |
class TeeLogger:
|
| 46 |
"""Duplicate output to stream and file."""
|
|
|
|
| 57 |
def fileno(self):
|
| 58 |
return self.stream.fileno()
|
| 59 |
|
| 60 |
+
# ── Configuration ───────────────────────────────────────────────────────────
|
| 61 |
|
| 62 |
HF_TOKEN = os.environ.get("HF_TOKEN")
|
| 63 |
OPENCLAW_HOME = Path.home() / ".openclaw"
|
|
|
|
| 76 |
AGENT_NAME = os.environ.get("AGENT_NAME", "HuggingClaw")
|
| 77 |
A2A_PEERS = os.environ.get("A2A_PEERS", "") # comma-separated peer URLs
|
| 78 |
|
| 79 |
+
# Default model — only set if user explicitly provides it; otherwise let OpenClaw decide.
|
| 80 |
OPENCLAW_DEFAULT_MODEL = os.environ.get("OPENCLAW_DEFAULT_MODEL", "")
|
| 81 |
|
| 82 |
# HF Spaces built-in env vars (auto-set by HF runtime)
|
|
|
|
| 91 |
# This ensures each duplicated Space gets its own dataset automatically.
|
| 92 |
HF_REPO_ID = os.environ.get("OPENCLAW_DATASET_REPO", "")
|
| 93 |
if not HF_REPO_ID and SPACE_ID:
|
| 94 |
+
# SPACE_ID = "username/SpaceName" → derive "username/SpaceName-data"
|
| 95 |
HF_REPO_ID = f"{SPACE_ID}-data"
|
| 96 |
+
print(f"[SYNC] OPENCLAW_DATASET_REPO not set — auto-derived from SPACE_ID: {HF_REPO_ID}")
|
| 97 |
elif not HF_REPO_ID and HF_TOKEN:
|
| 98 |
# Fallback: no SPACE_ID (local Docker), derive from HF_TOKEN username
|
| 99 |
try:
|
| 100 |
_api = HfApi(token=HF_TOKEN)
|
| 101 |
_username = _api.whoami()["name"]
|
| 102 |
HF_REPO_ID = f"{_username}/HuggingClaw-data"
|
| 103 |
+
print(f"[SYNC] OPENCLAW_DATASET_REPO not set — auto-derived from HF_TOKEN: {HF_REPO_ID}")
|
| 104 |
del _api, _username
|
| 105 |
except Exception as e:
|
| 106 |
print(f"[SYNC] WARNING: Could not derive username from HF_TOKEN: {e}")
|
|
|
|
| 112 |
sys.stdout = TeeLogger(log_dir / "sync.log", sys.stdout)
|
| 113 |
sys.stderr = sys.stdout
|
| 114 |
|
| 115 |
+
# ── Telegram API Base Auto-Probe ────────────────────────────────────────────
|
| 116 |
#
|
| 117 |
# HF Spaces blocks DNS for api.telegram.org. grammY uses Node 22's built-in
|
| 118 |
# fetch (undici) which bypasses dns.lookup patching and /etc/hosts.
|
|
|
|
| 123 |
# 2. telegram-proxy.cjs (loaded via NODE_OPTIONS --require) intercepts
|
| 124 |
# globalThis.fetch() and rewrites api.telegram.org URLs to the mirror.
|
| 125 |
#
|
| 126 |
+
# This works without a bot token — we just test HTTP reachability.
|
| 127 |
# If a bot token IS available, we do a full getMe verification.
|
| 128 |
|
| 129 |
# User can force a specific base via env var (skip auto-probe)
|
|
|
|
| 140 |
"""Probe Telegram API endpoints and return the first reachable one.
|
| 141 |
|
| 142 |
First checks if official api.telegram.org is reachable (HTTP level).
|
| 143 |
+
If not, tries mirrors. No bot token required — just tests connectivity.
|
| 144 |
Returns the working base URL (without trailing slash), or "" if all fail.
|
| 145 |
"""
|
| 146 |
ctx = ssl.create_default_context()
|
|
|
|
| 149 |
try:
|
| 150 |
req = urllib.request.Request(url, method="GET")
|
| 151 |
resp = urllib.request.urlopen(req, timeout=timeout, context=ctx)
|
| 152 |
+
print(f"[TELEGRAM] ✓ Reachable: {base} (HTTP {resp.status})")
|
| 153 |
return base.rstrip("/")
|
| 154 |
except urllib.error.HTTPError as e:
|
| 155 |
# HTTP error (4xx/5xx) still means the host IS reachable
|
| 156 |
+
print(f"[TELEGRAM] ✓ Reachable: {base} (HTTP {e.code})")
|
| 157 |
return base.rstrip("/")
|
| 158 |
except Exception as e:
|
| 159 |
reason = str(e)[:80]
|
| 160 |
+
print(f"[TELEGRAM] ✗ Unreachable: {base} ({reason})")
|
| 161 |
continue
|
| 162 |
|
| 163 |
print("[TELEGRAM] WARNING: All API endpoints unreachable!")
|
| 164 |
return ""
|
| 165 |
|
| 166 |
|
| 167 |
+
# ── Sync Manager ────────────────────────────────────────────────────────────
|
| 168 |
|
| 169 |
class OpenClawFullSync:
|
| 170 |
"""Upload/download the entire ~/.openclaw directory to HF Dataset."""
|
|
|
|
| 186 |
self.api = HfApi(token=HF_TOKEN)
|
| 187 |
self.dataset_exists = self._ensure_repo_exists()
|
| 188 |
|
| 189 |
+
# ── Repo management ────────────────────────────────────────────────
|
| 190 |
|
| 191 |
def _ensure_repo_exists(self):
|
| 192 |
"""Check if dataset repo exists; auto-create only when AUTO_CREATE_DATASET=true AND HF_TOKEN is set."""
|
|
|
|
| 200 |
print(f"[SYNC] Set AUTO_CREATE_DATASET=true to auto-create.")
|
| 201 |
print(f"[SYNC] Persistence disabled (app will still run normally).")
|
| 202 |
return False
|
| 203 |
+
print(f"[SYNC] Dataset repo NOT found: {HF_REPO_ID} — creating...")
|
| 204 |
try:
|
| 205 |
self.api.create_repo(
|
| 206 |
repo_id=HF_REPO_ID,
|
| 207 |
repo_type="dataset",
|
| 208 |
private=True,
|
| 209 |
)
|
| 210 |
+
print(f"[SYNC] ✓ Dataset repo created: {HF_REPO_ID}")
|
| 211 |
return True
|
| 212 |
except Exception as e:
|
| 213 |
+
print(f"[SYNC] ✗ Failed to create dataset repo: {e}")
|
| 214 |
return False
|
| 215 |
|
| 216 |
+
# ── Restore (startup) ──────────────â��€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€
|
| 217 |
|
| 218 |
def load_from_repo(self):
|
| 219 |
+
"""Download from dataset → ~/.openclaw"""
|
| 220 |
if not self.enabled:
|
| 221 |
print("[SYNC] Persistence disabled - skipping restore")
|
| 222 |
self._ensure_default_config()
|
|
|
|
| 229 |
self._patch_config()
|
| 230 |
return
|
| 231 |
|
| 232 |
+
print(f"[SYNC] â–¶ Restoring ~/.openclaw from dataset {HF_REPO_ID} ...")
|
| 233 |
OPENCLAW_HOME.mkdir(parents=True, exist_ok=True)
|
| 234 |
|
| 235 |
try:
|
|
|
|
| 259 |
dest = OPENCLAW_HOME / rel
|
| 260 |
dest.parent.mkdir(parents=True, exist_ok=True)
|
| 261 |
shutil.copy2(str(item), str(dest))
|
| 262 |
+
print("[SYNC] ✓ Restore completed.")
|
| 263 |
else:
|
| 264 |
print("[SYNC] Downloaded snapshot but dir not found. Starting fresh.")
|
| 265 |
|
| 266 |
except Exception as e:
|
| 267 |
+
print(f"[SYNC] ✗ Restore failed: {e}")
|
| 268 |
traceback.print_exc()
|
| 269 |
|
| 270 |
# Patch config after restore
|
| 271 |
self._patch_config()
|
| 272 |
self._debug_list_files()
|
| 273 |
|
| 274 |
+
# ── Save (periodic + shutdown) ─────────────────────────────────────
|
| 275 |
|
| 276 |
def save_to_repo(self):
|
| 277 |
+
"""Upload entire ~/.openclaw directory → dataset (all files, no filtering)"""
|
| 278 |
if not self.enabled:
|
| 279 |
return
|
| 280 |
if not OPENCLAW_HOME.exists():
|
|
|
|
| 286 |
print(f"[SYNC] Dataset {HF_REPO_ID} unavailable - skipping save")
|
| 287 |
return
|
| 288 |
|
| 289 |
+
print(f"[SYNC] ▶ Uploading ~/.openclaw → dataset {HF_REPO_ID}/{DATASET_PATH}/ ...")
|
| 290 |
|
| 291 |
try:
|
| 292 |
# Count files to upload (no per-file logging to reduce noise)
|
|
|
|
| 310 |
repo_id=HF_REPO_ID,
|
| 311 |
repo_type="dataset",
|
| 312 |
token=HF_TOKEN,
|
| 313 |
+
commit_message=f"Sync .openclaw — {datetime.now().isoformat()}",
|
| 314 |
ignore_patterns=[
|
| 315 |
+
"*.log", # Log files (sync.log, startup.log) — regenerated on boot
|
| 316 |
+
"*.lock", # Lock files — stale after restart
|
| 317 |
"*.tmp", # Temp files
|
| 318 |
"*.pid", # PID files
|
| 319 |
"__pycache__", # Python cache
|
| 320 |
],
|
| 321 |
)
|
| 322 |
+
print(f"[SYNC] ✓ Upload completed at {datetime.now().isoformat()}")
|
| 323 |
|
| 324 |
# Verify (summary only)
|
| 325 |
try:
|
|
|
|
| 330 |
pass
|
| 331 |
|
| 332 |
except Exception as e:
|
| 333 |
+
print(f"[SYNC] ✗ Upload failed: {e}")
|
| 334 |
traceback.print_exc()
|
| 335 |
|
| 336 |
+
# ── Config helpers ─────────────────────────────────────────────────
|
| 337 |
|
| 338 |
def _ensure_default_config(self):
|
| 339 |
config_path = OPENCLAW_HOME / "openclaw.json"
|
|
|
|
| 349 |
# Set gateway token
|
| 350 |
if "gateway" in cfg:
|
| 351 |
cfg["gateway"]["auth"] = {"token": GATEWAY_TOKEN}
|
| 352 |
+
# Don't manage providers — leave them for OpenClaw to handle.
|
| 353 |
# Just remove template entries with unresolved ${...} placeholders
|
| 354 |
# so OpenClaw starts clean and discovers providers from env vars.
|
| 355 |
providers = cfg.get("models", {}).get("providers", {})
|
|
|
|
| 392 |
data = json.load(f)
|
| 393 |
print("[SYNC] Config parsed OK.")
|
| 394 |
except (json.JSONDecodeError, Exception) as e:
|
| 395 |
+
# Config is corrupt — back up and start fresh
|
| 396 |
print(f"[SYNC] Config JSON is corrupt: {e}")
|
| 397 |
backup = config_path.with_suffix(f".corrupt_{int(time.time())}")
|
| 398 |
try:
|
|
|
|
| 447 |
data.setdefault("agents", {}).setdefault("defaults", {}).setdefault("model", {})
|
| 448 |
data.setdefault("session", {})["scope"] = "global"
|
| 449 |
|
| 450 |
+
# ── Provider API-key resolution ──────────────────────────────────
|
| 451 |
# The persisted config may contain template placeholders such as
|
| 452 |
# __OPENROUTER_API_KEY__ or __NVIDIA_API_KEY__. Resolve those from
|
| 453 |
# the current HF Space environment instead of treating them as
|
|
|
|
| 469 |
if pid not in restored_providers:
|
| 470 |
if env_key:
|
| 471 |
restored_providers[pid] = {**provider_defaults[pid], "apiKey": env_key}
|
| 472 |
+
print(f"[SYNC] ✓ Recreated {pid} provider from {env_name}")
|
| 473 |
continue
|
| 474 |
current_key = str(restored_providers[pid].get("apiKey", "")).strip()
|
| 475 |
if env_key and (not current_key or current_key.startswith("${") or current_key.startswith("__")):
|
| 476 |
restored_providers[pid]["apiKey"] = env_key
|
| 477 |
+
print(f"[SYNC] ✓ Resolved {pid} API key from {env_name}")
|
| 478 |
elif not env_key and (not current_key or current_key.startswith("${") or current_key.startswith("__")):
|
| 479 |
del restored_providers[pid]
|
| 480 |
print(f"[SYNC] Removed provider '{pid}' (no API key available in {env_name})")
|
|
|
|
| 482 |
if OPENCLAW_DEFAULT_MODEL:
|
| 483 |
data["agents"]["defaults"]["model"]["primary"] = OPENCLAW_DEFAULT_MODEL
|
| 484 |
|
| 485 |
+
# ── ACP (Agent Client Protocol) — native Claude Code integration ──
|
| 486 |
data["acp"] = {
|
| 487 |
"enabled": True,
|
| 488 |
"backend": "acpx",
|
|
|
|
| 501 |
plugin_allow.append("a2a-gateway")
|
| 502 |
data["plugins"]["allow"] = plugin_allow
|
| 503 |
|
| 504 |
+
# ── acpx Plugin Configuration (ACP backend) ──
|
| 505 |
data["plugins"]["entries"]["acpx"] = {
|
| 506 |
"enabled": True,
|
| 507 |
"config": {
|
|
|
|
| 509 |
}
|
| 510 |
}
|
| 511 |
|
| 512 |
+
# ── Coding Agent Plugin Configuration ──
|
| 513 |
CODING_TARGET_SPACE = os.environ.get("CODING_AGENT_TARGET_SPACE", "")
|
| 514 |
CODING_TARGET_DATASET = os.environ.get("CODING_AGENT_TARGET_DATASET", "")
|
| 515 |
if CODING_TARGET_SPACE:
|
|
|
|
| 528 |
elif isinstance(data["plugins"]["entries"]["telegram"], dict):
|
| 529 |
data["plugins"]["entries"]["telegram"]["enabled"] = True
|
| 530 |
|
| 531 |
+
# ── A2A Gateway Plugin Configuration (only if A2A_PEERS is set) ──
|
| 532 |
if A2A_PEERS:
|
| 533 |
peers = []
|
| 534 |
for peer_url in A2A_PEERS.split(","):
|
|
|
|
| 540 |
"name": name,
|
| 541 |
"agentCardUrl": f"{peer_url}/.well-known/agent-card.json"
|
| 542 |
})
|
| 543 |
+
print(f"[SYNC] A2A peer: {name} → {peer_url}")
|
| 544 |
|
| 545 |
data["plugins"]["entries"]["a2a-gateway"] = {
|
| 546 |
"enabled": True,
|
|
|
|
| 558 |
}
|
| 559 |
print(f"[SYNC] A2A gateway configured: name={AGENT_NAME}, port=18800, peers={len(peers)}")
|
| 560 |
|
| 561 |
+
# ── Telegram channel defaults (open DM policy for HF Spaces) ──
|
| 562 |
+
# Personal bot on HF Spaces — no need for strict pairing.
|
| 563 |
tg_ch = data.setdefault("channels", {}).setdefault("telegram", {})
|
| 564 |
tg_ch["dmPolicy"] = "open"
|
| 565 |
tg_ch["allowFrom"] = ["*"]
|
| 566 |
tg_ch["configWrites"] = True
|
| 567 |
print("[SYNC] Set channels.telegram: dmPolicy=open, allowFrom=[*], configWrites=true")
|
| 568 |
|
| 569 |
+
# ── Telegram API base auto-probe ──────────────────────────────
|
| 570 |
+
# Probe is done in run_openclaw() — sets TELEGRAM_API_ROOT env var
|
| 571 |
# for the telegram-proxy.cjs preload script to intercept fetch().
|
| 572 |
|
| 573 |
with open(config_path, "w") as f:
|
| 574 |
json.dump(data, f, indent=2)
|
| 575 |
print("[SYNC] Config patched and saved.")
|
| 576 |
|
| 577 |
+
# ── Deploy workspace templates (coding agent identity) ──
|
| 578 |
workspace_dir = Path(OPENCLAW_HOME) / "workspace"
|
| 579 |
workspace_dir.mkdir(parents=True, exist_ok=True)
|
| 580 |
templates_dir = Path("/home/node/workspace-templates")
|
|
|
|
| 618 |
except Exception as e:
|
| 619 |
print(f"[SYNC] listing failed: {e}")
|
| 620 |
|
| 621 |
+
# ── Background sync loop ──────────────────────────────────────────
|
| 622 |
|
| 623 |
def background_sync_loop(self, stop_event):
|
| 624 |
print(f"[SYNC] Background sync started (interval={SYNC_INTERVAL}s)")
|
| 625 |
while not stop_event.is_set():
|
| 626 |
if stop_event.wait(timeout=SYNC_INTERVAL):
|
| 627 |
break
|
| 628 |
+
print(f"[SYNC] ── Periodic sync triggered at {datetime.now().isoformat()} ──")
|
| 629 |
self.save_to_repo()
|
| 630 |
|
| 631 |
+
# ── OpenClaw migration / repair ───────────────────────────────────
|
| 632 |
|
| 633 |
def run_doctor_fix(self):
|
| 634 |
"""Run OpenClaw doctor --fix in headless mode before starting the gateway."""
|
|
|
|
| 647 |
print("[SYNC] Doctor output will be streamed below.")
|
| 648 |
env = os.environ.copy()
|
| 649 |
env["NODE_ENV"] = "production"
|
| 650 |
+
env["CI"] = "1"
|
| 651 |
+
env["TERM"] = "dumb"
|
| 652 |
+
|
| 653 |
+
# Doctor is a one-shot maintenance command. Do NOT inherit the
|
| 654 |
+
# gateway-only Node preload used by HuggingClaw (for example the
|
| 655 |
+
# token-redirect helper). That preload starts background monitoring
|
| 656 |
+
# for remote agents and can keep Doctor's Node event loop alive.
|
| 657 |
+
# The gateway receives the normal environment later in run_openclaw().
|
| 658 |
+
node_options = env.pop("NODE_OPTIONS", "")
|
| 659 |
+
if node_options:
|
| 660 |
+
print("[SYNC] Doctor: removed inherited NODE_OPTIONS preload")
|
| 661 |
+
|
| 662 |
timeout = int(os.environ.get("DOCTOR_TIMEOUT", "300"))
|
| 663 |
try:
|
| 664 |
+
process = subprocess.Popen(
|
| 665 |
+
doctor_cmd,
|
| 666 |
+
cwd=str(APP_DIR),
|
| 667 |
+
env=env,
|
| 668 |
+
stdin=subprocess.DEVNULL,
|
| 669 |
+
)
|
| 670 |
start_time = time.time()
|
| 671 |
while True:
|
| 672 |
return_code = process.poll()
|
|
|
|
| 674 |
elapsed = time.time() - start_time
|
| 675 |
print(f"[SYNC] Doctor finished in {elapsed:.1f}s with exit code {return_code}")
|
| 676 |
if return_code == 0:
|
| 677 |
+
print("[SYNC] ✓ OpenClaw doctor --fix completed")
|
| 678 |
return True
|
| 679 |
+
print(f"[SYNC] ✗ openclaw doctor --fix failed with exit code {return_code}")
|
| 680 |
return False
|
| 681 |
if time.time() - start_time > timeout:
|
| 682 |
+
print(f"[SYNC] ✗ Doctor exceeded DOCTOR_TIMEOUT={timeout}s. Stopping it so the Space does not remain stuck.")
|
| 683 |
try:
|
| 684 |
process.terminate()
|
| 685 |
process.wait(timeout=10)
|
|
|
|
| 690 |
return False
|
| 691 |
time.sleep(0.5)
|
| 692 |
except Exception as e:
|
| 693 |
+
print(f"[SYNC] ✗ Failed to run doctor --fix: {e}")
|
| 694 |
traceback.print_exc()
|
| 695 |
return False
|
| 696 |
|
| 697 |
+
# ── Application runner ─────────────────────────────────────────────
|
| 698 |
|
| 699 |
def run_openclaw(self):
|
| 700 |
log_file = OPENCLAW_HOME / "workspace" / "startup.log"
|
|
|
|
| 732 |
# Pass entire environment to OpenClaw (all API keys are already in os.environ)
|
| 733 |
env = os.environ.copy()
|
| 734 |
|
| 735 |
+
# ── Telegram API base probe ──────────────────────────────────────
|
| 736 |
# Determine working Telegram API endpoint and set env var for
|
| 737 |
# telegram-proxy.cjs to intercept fetch() calls.
|
| 738 |
if TELEGRAM_API_BASE:
|
|
|
|
| 747 |
print(f"[TELEGRAM] Set TELEGRAM_API_ROOT={tg_root}")
|
| 748 |
print(f"[TELEGRAM] telegram-proxy.cjs will redirect fetch() calls")
|
| 749 |
elif tg_root:
|
| 750 |
+
print("[TELEGRAM] Official API reachable — no proxy needed")
|
| 751 |
else:
|
| 752 |
+
print("[TELEGRAM] No reachable endpoint found — Telegram will not work")
|
| 753 |
try:
|
| 754 |
# Use Popen without shell to avoid pipe issues
|
| 755 |
+
# auth disabled in config — no token needed
|
| 756 |
process = subprocess.Popen(
|
| 757 |
entry_cmd,
|
| 758 |
cwd=str(APP_DIR),
|
|
|
|
| 776 |
continue
|
| 777 |
# Skip progress bars and download noise
|
| 778 |
if any(skip in stripped for skip in [
|
| 779 |
+
'Downloading', 'Fetching', '%|', 'â”', '───',
|
| 780 |
'Already cached', 'Using cache', 'tokenizer',
|
| 781 |
'.safetensors', 'model-', 'shard',
|
| 782 |
]):
|
|
|
|
| 799 |
traceback.print_exc()
|
| 800 |
return None
|
| 801 |
|
| 802 |
+
# ── Main ──────────────────────────────────────â���€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€
|
| 803 |
|
| 804 |
def main():
|
| 805 |
try:
|
|
|
|
| 842 |
t0 = time.time()
|
| 843 |
process = sync.run_openclaw()
|
| 844 |
print(f"[TIMER] run_openclaw launch: {time.time() - t0:.1f}s")
|
| 845 |
+
print(f"[TIMER] Total startup (init → app launched): {time.time() - t_main_start:.1f}s")
|
| 846 |
|
| 847 |
# 4. Start conversation-loop (RUN_ORCHESTRATOR=1, runs on God Space)
|
| 848 |
# OFFICE_MODE=1 is for frontend serving only (Home Space)
|
|
|
|
| 883 |
conv_thread = threading.Thread(target=run_conversation_loop_forever, daemon=True)
|
| 884 |
conv_thread.start()
|
| 885 |
else:
|
| 886 |
+
print("[SYNC] RUN_ORCHESTRATOR not set — skipping conversation-loop")
|
| 887 |
|
| 888 |
# Signal handler
|
| 889 |
def handle_signal(sig, frame):
|
|
|
|
| 933 |
|
| 934 |
|
| 935 |
if __name__ == "__main__":
|
| 936 |
+
main()
|