Powerman9x commited on
Commit
c398d2f
·
verified ·
1 Parent(s): 7492305

Update scripts/sync_hf.py

Browse files
Files changed (1) hide show
  1. scripts/sync_hf.py +85 -68
scripts/sync_hf.py CHANGED
@@ -1,14 +1,14 @@
1
  #!/usr/bin/env python3
2
  """
3
- OpenClaw HF Spaces Persistence — Full Directory Sync
4
  =====================================================
5
 
6
  Simplified persistence: upload/download the entire ~/.openclaw directory
7
  as-is to/from a Hugging Face Dataset repo.
8
 
9
- - Startup: snapshot_download → ~/.openclaw
10
- - Periodic: upload_folder → dataset openclaw_data/
11
- - Shutdown: final upload_folder → dataset openclaw_data/
12
  """
13
 
14
  import os
@@ -40,7 +40,7 @@ _logging.getLogger("filelock").setLevel(_logging.WARNING)
40
 
41
  from huggingface_hub import HfApi, snapshot_download
42
 
43
- # ── Logging helper ──────────────────────────────────────────────────────────
44
 
45
  class TeeLogger:
46
  """Duplicate output to stream and file."""
@@ -57,7 +57,7 @@ class TeeLogger:
57
  def fileno(self):
58
  return self.stream.fileno()
59
 
60
- # ── Configuration ───────────────────────────────────────────────────────────
61
 
62
  HF_TOKEN = os.environ.get("HF_TOKEN")
63
  OPENCLAW_HOME = Path.home() / ".openclaw"
@@ -76,7 +76,7 @@ GATEWAY_TOKEN = os.environ.get("GATEWAY_TOKEN", "huggingclaw")
76
  AGENT_NAME = os.environ.get("AGENT_NAME", "HuggingClaw")
77
  A2A_PEERS = os.environ.get("A2A_PEERS", "") # comma-separated peer URLs
78
 
79
- # Default model — only set if user explicitly provides it; otherwise let OpenClaw decide.
80
  OPENCLAW_DEFAULT_MODEL = os.environ.get("OPENCLAW_DEFAULT_MODEL", "")
81
 
82
  # HF Spaces built-in env vars (auto-set by HF runtime)
@@ -91,16 +91,16 @@ AUTO_CREATE_DATASET = os.environ.get("AUTO_CREATE_DATASET", "false").lower() in
91
  # This ensures each duplicated Space gets its own dataset automatically.
92
  HF_REPO_ID = os.environ.get("OPENCLAW_DATASET_REPO", "")
93
  if not HF_REPO_ID and SPACE_ID:
94
- # SPACE_ID = "username/SpaceName" → derive "username/SpaceName-data"
95
  HF_REPO_ID = f"{SPACE_ID}-data"
96
- print(f"[SYNC] OPENCLAW_DATASET_REPO not set — auto-derived from SPACE_ID: {HF_REPO_ID}")
97
  elif not HF_REPO_ID and HF_TOKEN:
98
  # Fallback: no SPACE_ID (local Docker), derive from HF_TOKEN username
99
  try:
100
  _api = HfApi(token=HF_TOKEN)
101
  _username = _api.whoami()["name"]
102
  HF_REPO_ID = f"{_username}/HuggingClaw-data"
103
- print(f"[SYNC] OPENCLAW_DATASET_REPO not set — auto-derived from HF_TOKEN: {HF_REPO_ID}")
104
  del _api, _username
105
  except Exception as e:
106
  print(f"[SYNC] WARNING: Could not derive username from HF_TOKEN: {e}")
@@ -112,7 +112,7 @@ log_dir.mkdir(parents=True, exist_ok=True)
112
  sys.stdout = TeeLogger(log_dir / "sync.log", sys.stdout)
113
  sys.stderr = sys.stdout
114
 
115
- # ── Telegram API Base Auto-Probe ────────────────────────────────────────────
116
  #
117
  # HF Spaces blocks DNS for api.telegram.org. grammY uses Node 22's built-in
118
  # fetch (undici) which bypasses dns.lookup patching and /etc/hosts.
@@ -123,7 +123,7 @@ sys.stderr = sys.stdout
123
  # 2. telegram-proxy.cjs (loaded via NODE_OPTIONS --require) intercepts
124
  # globalThis.fetch() and rewrites api.telegram.org URLs to the mirror.
125
  #
126
- # This works without a bot token — we just test HTTP reachability.
127
  # If a bot token IS available, we do a full getMe verification.
128
 
129
  # User can force a specific base via env var (skip auto-probe)
@@ -140,7 +140,7 @@ def probe_telegram_api(timeout: int = 8) -> str:
140
  """Probe Telegram API endpoints and return the first reachable one.
141
 
142
  First checks if official api.telegram.org is reachable (HTTP level).
143
- If not, tries mirrors. No bot token required — just tests connectivity.
144
  Returns the working base URL (without trailing slash), or "" if all fail.
145
  """
146
  ctx = ssl.create_default_context()
@@ -149,22 +149,22 @@ def probe_telegram_api(timeout: int = 8) -> str:
149
  try:
150
  req = urllib.request.Request(url, method="GET")
151
  resp = urllib.request.urlopen(req, timeout=timeout, context=ctx)
152
- print(f"[TELEGRAM] ✓ Reachable: {base} (HTTP {resp.status})")
153
  return base.rstrip("/")
154
  except urllib.error.HTTPError as e:
155
  # HTTP error (4xx/5xx) still means the host IS reachable
156
- print(f"[TELEGRAM] ✓ Reachable: {base} (HTTP {e.code})")
157
  return base.rstrip("/")
158
  except Exception as e:
159
  reason = str(e)[:80]
160
- print(f"[TELEGRAM] ✗ Unreachable: {base} ({reason})")
161
  continue
162
 
163
  print("[TELEGRAM] WARNING: All API endpoints unreachable!")
164
  return ""
165
 
166
 
167
- # ── Sync Manager ────────────────────────────────────────────────────────────
168
 
169
  class OpenClawFullSync:
170
  """Upload/download the entire ~/.openclaw directory to HF Dataset."""
@@ -186,7 +186,7 @@ class OpenClawFullSync:
186
  self.api = HfApi(token=HF_TOKEN)
187
  self.dataset_exists = self._ensure_repo_exists()
188
 
189
- # ── Repo management ────────────────────────────────────────────────
190
 
191
  def _ensure_repo_exists(self):
192
  """Check if dataset repo exists; auto-create only when AUTO_CREATE_DATASET=true AND HF_TOKEN is set."""
@@ -200,23 +200,23 @@ class OpenClawFullSync:
200
  print(f"[SYNC] Set AUTO_CREATE_DATASET=true to auto-create.")
201
  print(f"[SYNC] Persistence disabled (app will still run normally).")
202
  return False
203
- print(f"[SYNC] Dataset repo NOT found: {HF_REPO_ID} — creating...")
204
  try:
205
  self.api.create_repo(
206
  repo_id=HF_REPO_ID,
207
  repo_type="dataset",
208
  private=True,
209
  )
210
- print(f"[SYNC] ✓ Dataset repo created: {HF_REPO_ID}")
211
  return True
212
  except Exception as e:
213
- print(f"[SYNC] ✗ Failed to create dataset repo: {e}")
214
  return False
215
 
216
- # ── Restore (startup) ─────────────────────────────────────────────
217
 
218
  def load_from_repo(self):
219
- """Download from dataset → ~/.openclaw"""
220
  if not self.enabled:
221
  print("[SYNC] Persistence disabled - skipping restore")
222
  self._ensure_default_config()
@@ -229,7 +229,7 @@ class OpenClawFullSync:
229
  self._patch_config()
230
  return
231
 
232
- print(f"[SYNC] ▶ Restoring ~/.openclaw from dataset {HF_REPO_ID} ...")
233
  OPENCLAW_HOME.mkdir(parents=True, exist_ok=True)
234
 
235
  try:
@@ -259,22 +259,22 @@ class OpenClawFullSync:
259
  dest = OPENCLAW_HOME / rel
260
  dest.parent.mkdir(parents=True, exist_ok=True)
261
  shutil.copy2(str(item), str(dest))
262
- print("[SYNC] ✓ Restore completed.")
263
  else:
264
  print("[SYNC] Downloaded snapshot but dir not found. Starting fresh.")
265
 
266
  except Exception as e:
267
- print(f"[SYNC] ✗ Restore failed: {e}")
268
  traceback.print_exc()
269
 
270
  # Patch config after restore
271
  self._patch_config()
272
  self._debug_list_files()
273
 
274
- # ── Save (periodic + shutdown) ─────────────────────────────────────
275
 
276
  def save_to_repo(self):
277
- """Upload entire ~/.openclaw directory → dataset (all files, no filtering)"""
278
  if not self.enabled:
279
  return
280
  if not OPENCLAW_HOME.exists():
@@ -286,7 +286,7 @@ class OpenClawFullSync:
286
  print(f"[SYNC] Dataset {HF_REPO_ID} unavailable - skipping save")
287
  return
288
 
289
- print(f"[SYNC] ▶ Uploading ~/.openclaw → dataset {HF_REPO_ID}/{DATASET_PATH}/ ...")
290
 
291
  try:
292
  # Count files to upload (no per-file logging to reduce noise)
@@ -310,16 +310,16 @@ class OpenClawFullSync:
310
  repo_id=HF_REPO_ID,
311
  repo_type="dataset",
312
  token=HF_TOKEN,
313
- commit_message=f"Sync .openclaw — {datetime.now().isoformat()}",
314
  ignore_patterns=[
315
- "*.log", # Log files (sync.log, startup.log) — regenerated on boot
316
- "*.lock", # Lock files — stale after restart
317
  "*.tmp", # Temp files
318
  "*.pid", # PID files
319
  "__pycache__", # Python cache
320
  ],
321
  )
322
- print(f"[SYNC] ✓ Upload completed at {datetime.now().isoformat()}")
323
 
324
  # Verify (summary only)
325
  try:
@@ -330,10 +330,10 @@ class OpenClawFullSync:
330
  pass
331
 
332
  except Exception as e:
333
- print(f"[SYNC] ✗ Upload failed: {e}")
334
  traceback.print_exc()
335
 
336
- # ── Config helpers ─────────────────────────────────────────────────
337
 
338
  def _ensure_default_config(self):
339
  config_path = OPENCLAW_HOME / "openclaw.json"
@@ -349,7 +349,7 @@ class OpenClawFullSync:
349
  # Set gateway token
350
  if "gateway" in cfg:
351
  cfg["gateway"]["auth"] = {"token": GATEWAY_TOKEN}
352
- # Don't manage providers — leave them for OpenClaw to handle.
353
  # Just remove template entries with unresolved ${...} placeholders
354
  # so OpenClaw starts clean and discovers providers from env vars.
355
  providers = cfg.get("models", {}).get("providers", {})
@@ -392,7 +392,7 @@ class OpenClawFullSync:
392
  data = json.load(f)
393
  print("[SYNC] Config parsed OK.")
394
  except (json.JSONDecodeError, Exception) as e:
395
- # Config is corrupt — back up and start fresh
396
  print(f"[SYNC] Config JSON is corrupt: {e}")
397
  backup = config_path.with_suffix(f".corrupt_{int(time.time())}")
398
  try:
@@ -447,7 +447,7 @@ class OpenClawFullSync:
447
  data.setdefault("agents", {}).setdefault("defaults", {}).setdefault("model", {})
448
  data.setdefault("session", {})["scope"] = "global"
449
 
450
- # ── Provider API-key resolution ──────────────────────────────────
451
  # The persisted config may contain template placeholders such as
452
  # __OPENROUTER_API_KEY__ or __NVIDIA_API_KEY__. Resolve those from
453
  # the current HF Space environment instead of treating them as
@@ -469,12 +469,12 @@ class OpenClawFullSync:
469
  if pid not in restored_providers:
470
  if env_key:
471
  restored_providers[pid] = {**provider_defaults[pid], "apiKey": env_key}
472
- print(f"[SYNC] ✓ Recreated {pid} provider from {env_name}")
473
  continue
474
  current_key = str(restored_providers[pid].get("apiKey", "")).strip()
475
  if env_key and (not current_key or current_key.startswith("${") or current_key.startswith("__")):
476
  restored_providers[pid]["apiKey"] = env_key
477
- print(f"[SYNC] ✓ Resolved {pid} API key from {env_name}")
478
  elif not env_key and (not current_key or current_key.startswith("${") or current_key.startswith("__")):
479
  del restored_providers[pid]
480
  print(f"[SYNC] Removed provider '{pid}' (no API key available in {env_name})")
@@ -482,7 +482,7 @@ class OpenClawFullSync:
482
  if OPENCLAW_DEFAULT_MODEL:
483
  data["agents"]["defaults"]["model"]["primary"] = OPENCLAW_DEFAULT_MODEL
484
 
485
- # ── ACP (Agent Client Protocol) — native Claude Code integration ──
486
  data["acp"] = {
487
  "enabled": True,
488
  "backend": "acpx",
@@ -501,7 +501,7 @@ class OpenClawFullSync:
501
  plugin_allow.append("a2a-gateway")
502
  data["plugins"]["allow"] = plugin_allow
503
 
504
- # ── acpx Plugin Configuration (ACP backend) ──
505
  data["plugins"]["entries"]["acpx"] = {
506
  "enabled": True,
507
  "config": {
@@ -509,7 +509,7 @@ class OpenClawFullSync:
509
  }
510
  }
511
 
512
- # ── Coding Agent Plugin Configuration ──
513
  CODING_TARGET_SPACE = os.environ.get("CODING_AGENT_TARGET_SPACE", "")
514
  CODING_TARGET_DATASET = os.environ.get("CODING_AGENT_TARGET_DATASET", "")
515
  if CODING_TARGET_SPACE:
@@ -528,7 +528,7 @@ class OpenClawFullSync:
528
  elif isinstance(data["plugins"]["entries"]["telegram"], dict):
529
  data["plugins"]["entries"]["telegram"]["enabled"] = True
530
 
531
- # ── A2A Gateway Plugin Configuration (only if A2A_PEERS is set) ──
532
  if A2A_PEERS:
533
  peers = []
534
  for peer_url in A2A_PEERS.split(","):
@@ -540,7 +540,7 @@ class OpenClawFullSync:
540
  "name": name,
541
  "agentCardUrl": f"{peer_url}/.well-known/agent-card.json"
542
  })
543
- print(f"[SYNC] A2A peer: {name} → {peer_url}")
544
 
545
  data["plugins"]["entries"]["a2a-gateway"] = {
546
  "enabled": True,
@@ -558,23 +558,23 @@ class OpenClawFullSync:
558
  }
559
  print(f"[SYNC] A2A gateway configured: name={AGENT_NAME}, port=18800, peers={len(peers)}")
560
 
561
- # ── Telegram channel defaults (open DM policy for HF Spaces) ──
562
- # Personal bot on HF Spaces — no need for strict pairing.
563
  tg_ch = data.setdefault("channels", {}).setdefault("telegram", {})
564
  tg_ch["dmPolicy"] = "open"
565
  tg_ch["allowFrom"] = ["*"]
566
  tg_ch["configWrites"] = True
567
  print("[SYNC] Set channels.telegram: dmPolicy=open, allowFrom=[*], configWrites=true")
568
 
569
- # ── Telegram API base auto-probe ──────────────────────────────
570
- # Probe is done in run_openclaw() — sets TELEGRAM_API_ROOT env var
571
  # for the telegram-proxy.cjs preload script to intercept fetch().
572
 
573
  with open(config_path, "w") as f:
574
  json.dump(data, f, indent=2)
575
  print("[SYNC] Config patched and saved.")
576
 
577
- # ── Deploy workspace templates (coding agent identity) ──
578
  workspace_dir = Path(OPENCLAW_HOME) / "workspace"
579
  workspace_dir.mkdir(parents=True, exist_ok=True)
580
  templates_dir = Path("/home/node/workspace-templates")
@@ -618,17 +618,17 @@ class OpenClawFullSync:
618
  except Exception as e:
619
  print(f"[SYNC] listing failed: {e}")
620
 
621
- # ── Background sync loop ──────────────────────────────────────────
622
 
623
  def background_sync_loop(self, stop_event):
624
  print(f"[SYNC] Background sync started (interval={SYNC_INTERVAL}s)")
625
  while not stop_event.is_set():
626
  if stop_event.wait(timeout=SYNC_INTERVAL):
627
  break
628
- print(f"[SYNC] ── Periodic sync triggered at {datetime.now().isoformat()} ──")
629
  self.save_to_repo()
630
 
631
- # ── OpenClaw migration / repair ───────────────────────────────────
632
 
633
  def run_doctor_fix(self):
634
  """Run OpenClaw doctor --fix in headless mode before starting the gateway."""
@@ -647,9 +647,26 @@ class OpenClawFullSync:
647
  print("[SYNC] Doctor output will be streamed below.")
648
  env = os.environ.copy()
649
  env["NODE_ENV"] = "production"
 
 
 
 
 
 
 
 
 
 
 
 
650
  timeout = int(os.environ.get("DOCTOR_TIMEOUT", "300"))
651
  try:
652
- process = subprocess.Popen(doctor_cmd, cwd=str(APP_DIR), env=env)
 
 
 
 
 
653
  start_time = time.time()
654
  while True:
655
  return_code = process.poll()
@@ -657,12 +674,12 @@ class OpenClawFullSync:
657
  elapsed = time.time() - start_time
658
  print(f"[SYNC] Doctor finished in {elapsed:.1f}s with exit code {return_code}")
659
  if return_code == 0:
660
- print("[SYNC] ✓ OpenClaw doctor --fix completed")
661
  return True
662
- print(f"[SYNC] ✗ openclaw doctor --fix failed with exit code {return_code}")
663
  return False
664
  if time.time() - start_time > timeout:
665
- print(f"[SYNC] ✗ Doctor exceeded DOCTOR_TIMEOUT={timeout}s. Stopping it so the Space does not remain stuck.")
666
  try:
667
  process.terminate()
668
  process.wait(timeout=10)
@@ -673,11 +690,11 @@ class OpenClawFullSync:
673
  return False
674
  time.sleep(0.5)
675
  except Exception as e:
676
- print(f"[SYNC] ✗ Failed to run doctor --fix: {e}")
677
  traceback.print_exc()
678
  return False
679
 
680
- # ── Application runner ─────────────────────────────────────────────
681
 
682
  def run_openclaw(self):
683
  log_file = OPENCLAW_HOME / "workspace" / "startup.log"
@@ -715,7 +732,7 @@ class OpenClawFullSync:
715
  # Pass entire environment to OpenClaw (all API keys are already in os.environ)
716
  env = os.environ.copy()
717
 
718
- # ── Telegram API base probe ──────────────────────────────────────
719
  # Determine working Telegram API endpoint and set env var for
720
  # telegram-proxy.cjs to intercept fetch() calls.
721
  if TELEGRAM_API_BASE:
@@ -730,12 +747,12 @@ class OpenClawFullSync:
730
  print(f"[TELEGRAM] Set TELEGRAM_API_ROOT={tg_root}")
731
  print(f"[TELEGRAM] telegram-proxy.cjs will redirect fetch() calls")
732
  elif tg_root:
733
- print("[TELEGRAM] Official API reachable — no proxy needed")
734
  else:
735
- print("[TELEGRAM] No reachable endpoint found — Telegram will not work")
736
  try:
737
  # Use Popen without shell to avoid pipe issues
738
- # auth disabled in config — no token needed
739
  process = subprocess.Popen(
740
  entry_cmd,
741
  cwd=str(APP_DIR),
@@ -759,7 +776,7 @@ class OpenClawFullSync:
759
  continue
760
  # Skip progress bars and download noise
761
  if any(skip in stripped for skip in [
762
- 'Downloading', 'Fetching', '%|', '━', '───',
763
  'Already cached', 'Using cache', 'tokenizer',
764
  '.safetensors', 'model-', 'shard',
765
  ]):
@@ -782,7 +799,7 @@ class OpenClawFullSync:
782
  traceback.print_exc()
783
  return None
784
 
785
- # ── Main ────────────────────────────────────────────────────────────────────
786
 
787
  def main():
788
  try:
@@ -825,7 +842,7 @@ def main():
825
  t0 = time.time()
826
  process = sync.run_openclaw()
827
  print(f"[TIMER] run_openclaw launch: {time.time() - t0:.1f}s")
828
- print(f"[TIMER] Total startup (init → app launched): {time.time() - t_main_start:.1f}s")
829
 
830
  # 4. Start conversation-loop (RUN_ORCHESTRATOR=1, runs on God Space)
831
  # OFFICE_MODE=1 is for frontend serving only (Home Space)
@@ -866,7 +883,7 @@ def main():
866
  conv_thread = threading.Thread(target=run_conversation_loop_forever, daemon=True)
867
  conv_thread.start()
868
  else:
869
- print("[SYNC] RUN_ORCHESTRATOR not set — skipping conversation-loop")
870
 
871
  # Signal handler
872
  def handle_signal(sig, frame):
@@ -916,4 +933,4 @@ def main():
916
 
917
 
918
  if __name__ == "__main__":
919
- main()
 
1
  #!/usr/bin/env python3
2
  """
3
+ OpenClaw HF Spaces Persistence — Full Directory Sync
4
  =====================================================
5
 
6
  Simplified persistence: upload/download the entire ~/.openclaw directory
7
  as-is to/from a Hugging Face Dataset repo.
8
 
9
+ - Startup: snapshot_download → ~/.openclaw
10
+ - Periodic: upload_folder → dataset openclaw_data/
11
+ - Shutdown: final upload_folder → dataset openclaw_data/
12
  """
13
 
14
  import os
 
40
 
41
  from huggingface_hub import HfApi, snapshot_download
42
 
43
+ # ── Logging helper ──────────────────────────────────────────────────────────
44
 
45
  class TeeLogger:
46
  """Duplicate output to stream and file."""
 
57
  def fileno(self):
58
  return self.stream.fileno()
59
 
60
+ # ── Configuration ───────────────────────────────────────────────────────────
61
 
62
  HF_TOKEN = os.environ.get("HF_TOKEN")
63
  OPENCLAW_HOME = Path.home() / ".openclaw"
 
76
  AGENT_NAME = os.environ.get("AGENT_NAME", "HuggingClaw")
77
  A2A_PEERS = os.environ.get("A2A_PEERS", "") # comma-separated peer URLs
78
 
79
+ # Default model — only set if user explicitly provides it; otherwise let OpenClaw decide.
80
  OPENCLAW_DEFAULT_MODEL = os.environ.get("OPENCLAW_DEFAULT_MODEL", "")
81
 
82
  # HF Spaces built-in env vars (auto-set by HF runtime)
 
91
  # This ensures each duplicated Space gets its own dataset automatically.
92
  HF_REPO_ID = os.environ.get("OPENCLAW_DATASET_REPO", "")
93
  if not HF_REPO_ID and SPACE_ID:
94
+ # SPACE_ID = "username/SpaceName" → derive "username/SpaceName-data"
95
  HF_REPO_ID = f"{SPACE_ID}-data"
96
+ print(f"[SYNC] OPENCLAW_DATASET_REPO not set — auto-derived from SPACE_ID: {HF_REPO_ID}")
97
  elif not HF_REPO_ID and HF_TOKEN:
98
  # Fallback: no SPACE_ID (local Docker), derive from HF_TOKEN username
99
  try:
100
  _api = HfApi(token=HF_TOKEN)
101
  _username = _api.whoami()["name"]
102
  HF_REPO_ID = f"{_username}/HuggingClaw-data"
103
+ print(f"[SYNC] OPENCLAW_DATASET_REPO not set — auto-derived from HF_TOKEN: {HF_REPO_ID}")
104
  del _api, _username
105
  except Exception as e:
106
  print(f"[SYNC] WARNING: Could not derive username from HF_TOKEN: {e}")
 
112
  sys.stdout = TeeLogger(log_dir / "sync.log", sys.stdout)
113
  sys.stderr = sys.stdout
114
 
115
+ # ── Telegram API Base Auto-Probe ────────────────────────────────────────────
116
  #
117
  # HF Spaces blocks DNS for api.telegram.org. grammY uses Node 22's built-in
118
  # fetch (undici) which bypasses dns.lookup patching and /etc/hosts.
 
123
  # 2. telegram-proxy.cjs (loaded via NODE_OPTIONS --require) intercepts
124
  # globalThis.fetch() and rewrites api.telegram.org URLs to the mirror.
125
  #
126
+ # This works without a bot token — we just test HTTP reachability.
127
  # If a bot token IS available, we do a full getMe verification.
128
 
129
  # User can force a specific base via env var (skip auto-probe)
 
140
  """Probe Telegram API endpoints and return the first reachable one.
141
 
142
  First checks if official api.telegram.org is reachable (HTTP level).
143
+ If not, tries mirrors. No bot token required — just tests connectivity.
144
  Returns the working base URL (without trailing slash), or "" if all fail.
145
  """
146
  ctx = ssl.create_default_context()
 
149
  try:
150
  req = urllib.request.Request(url, method="GET")
151
  resp = urllib.request.urlopen(req, timeout=timeout, context=ctx)
152
+ print(f"[TELEGRAM] ✓ Reachable: {base} (HTTP {resp.status})")
153
  return base.rstrip("/")
154
  except urllib.error.HTTPError as e:
155
  # HTTP error (4xx/5xx) still means the host IS reachable
156
+ print(f"[TELEGRAM] ✓ Reachable: {base} (HTTP {e.code})")
157
  return base.rstrip("/")
158
  except Exception as e:
159
  reason = str(e)[:80]
160
+ print(f"[TELEGRAM] ✗ Unreachable: {base} ({reason})")
161
  continue
162
 
163
  print("[TELEGRAM] WARNING: All API endpoints unreachable!")
164
  return ""
165
 
166
 
167
+ # ── Sync Manager ────────────────────────────────────────────────────────────
168
 
169
  class OpenClawFullSync:
170
  """Upload/download the entire ~/.openclaw directory to HF Dataset."""
 
186
  self.api = HfApi(token=HF_TOKEN)
187
  self.dataset_exists = self._ensure_repo_exists()
188
 
189
+ # ── Repo management ────────────────────────────────────────────────
190
 
191
  def _ensure_repo_exists(self):
192
  """Check if dataset repo exists; auto-create only when AUTO_CREATE_DATASET=true AND HF_TOKEN is set."""
 
200
  print(f"[SYNC] Set AUTO_CREATE_DATASET=true to auto-create.")
201
  print(f"[SYNC] Persistence disabled (app will still run normally).")
202
  return False
203
+ print(f"[SYNC] Dataset repo NOT found: {HF_REPO_ID} — creating...")
204
  try:
205
  self.api.create_repo(
206
  repo_id=HF_REPO_ID,
207
  repo_type="dataset",
208
  private=True,
209
  )
210
+ print(f"[SYNC] ✓ Dataset repo created: {HF_REPO_ID}")
211
  return True
212
  except Exception as e:
213
+ print(f"[SYNC] ✗ Failed to create dataset repo: {e}")
214
  return False
215
 
216
+ # ── Restore (startup) ──────────────â��€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€
217
 
218
  def load_from_repo(self):
219
+ """Download from dataset → ~/.openclaw"""
220
  if not self.enabled:
221
  print("[SYNC] Persistence disabled - skipping restore")
222
  self._ensure_default_config()
 
229
  self._patch_config()
230
  return
231
 
232
+ print(f"[SYNC] â–¶ Restoring ~/.openclaw from dataset {HF_REPO_ID} ...")
233
  OPENCLAW_HOME.mkdir(parents=True, exist_ok=True)
234
 
235
  try:
 
259
  dest = OPENCLAW_HOME / rel
260
  dest.parent.mkdir(parents=True, exist_ok=True)
261
  shutil.copy2(str(item), str(dest))
262
+ print("[SYNC] ✓ Restore completed.")
263
  else:
264
  print("[SYNC] Downloaded snapshot but dir not found. Starting fresh.")
265
 
266
  except Exception as e:
267
+ print(f"[SYNC] ✗ Restore failed: {e}")
268
  traceback.print_exc()
269
 
270
  # Patch config after restore
271
  self._patch_config()
272
  self._debug_list_files()
273
 
274
+ # ── Save (periodic + shutdown) ─────────────────────────────────────
275
 
276
  def save_to_repo(self):
277
+ """Upload entire ~/.openclaw directory → dataset (all files, no filtering)"""
278
  if not self.enabled:
279
  return
280
  if not OPENCLAW_HOME.exists():
 
286
  print(f"[SYNC] Dataset {HF_REPO_ID} unavailable - skipping save")
287
  return
288
 
289
+ print(f"[SYNC] ▶ Uploading ~/.openclaw → dataset {HF_REPO_ID}/{DATASET_PATH}/ ...")
290
 
291
  try:
292
  # Count files to upload (no per-file logging to reduce noise)
 
310
  repo_id=HF_REPO_ID,
311
  repo_type="dataset",
312
  token=HF_TOKEN,
313
+ commit_message=f"Sync .openclaw — {datetime.now().isoformat()}",
314
  ignore_patterns=[
315
+ "*.log", # Log files (sync.log, startup.log) — regenerated on boot
316
+ "*.lock", # Lock files — stale after restart
317
  "*.tmp", # Temp files
318
  "*.pid", # PID files
319
  "__pycache__", # Python cache
320
  ],
321
  )
322
+ print(f"[SYNC] ✓ Upload completed at {datetime.now().isoformat()}")
323
 
324
  # Verify (summary only)
325
  try:
 
330
  pass
331
 
332
  except Exception as e:
333
+ print(f"[SYNC] ✗ Upload failed: {e}")
334
  traceback.print_exc()
335
 
336
+ # ── Config helpers ─────────────────────────────────────────────────
337
 
338
  def _ensure_default_config(self):
339
  config_path = OPENCLAW_HOME / "openclaw.json"
 
349
  # Set gateway token
350
  if "gateway" in cfg:
351
  cfg["gateway"]["auth"] = {"token": GATEWAY_TOKEN}
352
+ # Don't manage providers — leave them for OpenClaw to handle.
353
  # Just remove template entries with unresolved ${...} placeholders
354
  # so OpenClaw starts clean and discovers providers from env vars.
355
  providers = cfg.get("models", {}).get("providers", {})
 
392
  data = json.load(f)
393
  print("[SYNC] Config parsed OK.")
394
  except (json.JSONDecodeError, Exception) as e:
395
+ # Config is corrupt — back up and start fresh
396
  print(f"[SYNC] Config JSON is corrupt: {e}")
397
  backup = config_path.with_suffix(f".corrupt_{int(time.time())}")
398
  try:
 
447
  data.setdefault("agents", {}).setdefault("defaults", {}).setdefault("model", {})
448
  data.setdefault("session", {})["scope"] = "global"
449
 
450
+ # ── Provider API-key resolution ──────────────────────────────────
451
  # The persisted config may contain template placeholders such as
452
  # __OPENROUTER_API_KEY__ or __NVIDIA_API_KEY__. Resolve those from
453
  # the current HF Space environment instead of treating them as
 
469
  if pid not in restored_providers:
470
  if env_key:
471
  restored_providers[pid] = {**provider_defaults[pid], "apiKey": env_key}
472
+ print(f"[SYNC] ✓ Recreated {pid} provider from {env_name}")
473
  continue
474
  current_key = str(restored_providers[pid].get("apiKey", "")).strip()
475
  if env_key and (not current_key or current_key.startswith("${") or current_key.startswith("__")):
476
  restored_providers[pid]["apiKey"] = env_key
477
+ print(f"[SYNC] ✓ Resolved {pid} API key from {env_name}")
478
  elif not env_key and (not current_key or current_key.startswith("${") or current_key.startswith("__")):
479
  del restored_providers[pid]
480
  print(f"[SYNC] Removed provider '{pid}' (no API key available in {env_name})")
 
482
  if OPENCLAW_DEFAULT_MODEL:
483
  data["agents"]["defaults"]["model"]["primary"] = OPENCLAW_DEFAULT_MODEL
484
 
485
+ # ── ACP (Agent Client Protocol) — native Claude Code integration ──
486
  data["acp"] = {
487
  "enabled": True,
488
  "backend": "acpx",
 
501
  plugin_allow.append("a2a-gateway")
502
  data["plugins"]["allow"] = plugin_allow
503
 
504
+ # ── acpx Plugin Configuration (ACP backend) ──
505
  data["plugins"]["entries"]["acpx"] = {
506
  "enabled": True,
507
  "config": {
 
509
  }
510
  }
511
 
512
+ # ── Coding Agent Plugin Configuration ──
513
  CODING_TARGET_SPACE = os.environ.get("CODING_AGENT_TARGET_SPACE", "")
514
  CODING_TARGET_DATASET = os.environ.get("CODING_AGENT_TARGET_DATASET", "")
515
  if CODING_TARGET_SPACE:
 
528
  elif isinstance(data["plugins"]["entries"]["telegram"], dict):
529
  data["plugins"]["entries"]["telegram"]["enabled"] = True
530
 
531
+ # ── A2A Gateway Plugin Configuration (only if A2A_PEERS is set) ──
532
  if A2A_PEERS:
533
  peers = []
534
  for peer_url in A2A_PEERS.split(","):
 
540
  "name": name,
541
  "agentCardUrl": f"{peer_url}/.well-known/agent-card.json"
542
  })
543
+ print(f"[SYNC] A2A peer: {name} → {peer_url}")
544
 
545
  data["plugins"]["entries"]["a2a-gateway"] = {
546
  "enabled": True,
 
558
  }
559
  print(f"[SYNC] A2A gateway configured: name={AGENT_NAME}, port=18800, peers={len(peers)}")
560
 
561
+ # ── Telegram channel defaults (open DM policy for HF Spaces) ──
562
+ # Personal bot on HF Spaces — no need for strict pairing.
563
  tg_ch = data.setdefault("channels", {}).setdefault("telegram", {})
564
  tg_ch["dmPolicy"] = "open"
565
  tg_ch["allowFrom"] = ["*"]
566
  tg_ch["configWrites"] = True
567
  print("[SYNC] Set channels.telegram: dmPolicy=open, allowFrom=[*], configWrites=true")
568
 
569
+ # ── Telegram API base auto-probe ──────────────────────────────
570
+ # Probe is done in run_openclaw() — sets TELEGRAM_API_ROOT env var
571
  # for the telegram-proxy.cjs preload script to intercept fetch().
572
 
573
  with open(config_path, "w") as f:
574
  json.dump(data, f, indent=2)
575
  print("[SYNC] Config patched and saved.")
576
 
577
+ # ── Deploy workspace templates (coding agent identity) ──
578
  workspace_dir = Path(OPENCLAW_HOME) / "workspace"
579
  workspace_dir.mkdir(parents=True, exist_ok=True)
580
  templates_dir = Path("/home/node/workspace-templates")
 
618
  except Exception as e:
619
  print(f"[SYNC] listing failed: {e}")
620
 
621
+ # ── Background sync loop ──────────────────────────────────────────
622
 
623
  def background_sync_loop(self, stop_event):
624
  print(f"[SYNC] Background sync started (interval={SYNC_INTERVAL}s)")
625
  while not stop_event.is_set():
626
  if stop_event.wait(timeout=SYNC_INTERVAL):
627
  break
628
+ print(f"[SYNC] ── Periodic sync triggered at {datetime.now().isoformat()} ──")
629
  self.save_to_repo()
630
 
631
+ # ── OpenClaw migration / repair ───────────────────────────────────
632
 
633
  def run_doctor_fix(self):
634
  """Run OpenClaw doctor --fix in headless mode before starting the gateway."""
 
647
  print("[SYNC] Doctor output will be streamed below.")
648
  env = os.environ.copy()
649
  env["NODE_ENV"] = "production"
650
+ env["CI"] = "1"
651
+ env["TERM"] = "dumb"
652
+
653
+ # Doctor is a one-shot maintenance command. Do NOT inherit the
654
+ # gateway-only Node preload used by HuggingClaw (for example the
655
+ # token-redirect helper). That preload starts background monitoring
656
+ # for remote agents and can keep Doctor's Node event loop alive.
657
+ # The gateway receives the normal environment later in run_openclaw().
658
+ node_options = env.pop("NODE_OPTIONS", "")
659
+ if node_options:
660
+ print("[SYNC] Doctor: removed inherited NODE_OPTIONS preload")
661
+
662
  timeout = int(os.environ.get("DOCTOR_TIMEOUT", "300"))
663
  try:
664
+ process = subprocess.Popen(
665
+ doctor_cmd,
666
+ cwd=str(APP_DIR),
667
+ env=env,
668
+ stdin=subprocess.DEVNULL,
669
+ )
670
  start_time = time.time()
671
  while True:
672
  return_code = process.poll()
 
674
  elapsed = time.time() - start_time
675
  print(f"[SYNC] Doctor finished in {elapsed:.1f}s with exit code {return_code}")
676
  if return_code == 0:
677
+ print("[SYNC] ✓ OpenClaw doctor --fix completed")
678
  return True
679
+ print(f"[SYNC] ✗ openclaw doctor --fix failed with exit code {return_code}")
680
  return False
681
  if time.time() - start_time > timeout:
682
+ print(f"[SYNC] ✗ Doctor exceeded DOCTOR_TIMEOUT={timeout}s. Stopping it so the Space does not remain stuck.")
683
  try:
684
  process.terminate()
685
  process.wait(timeout=10)
 
690
  return False
691
  time.sleep(0.5)
692
  except Exception as e:
693
+ print(f"[SYNC] ✗ Failed to run doctor --fix: {e}")
694
  traceback.print_exc()
695
  return False
696
 
697
+ # ── Application runner ─────────────────────────────────────────────
698
 
699
  def run_openclaw(self):
700
  log_file = OPENCLAW_HOME / "workspace" / "startup.log"
 
732
  # Pass entire environment to OpenClaw (all API keys are already in os.environ)
733
  env = os.environ.copy()
734
 
735
+ # ── Telegram API base probe ──────────────────────────────────────
736
  # Determine working Telegram API endpoint and set env var for
737
  # telegram-proxy.cjs to intercept fetch() calls.
738
  if TELEGRAM_API_BASE:
 
747
  print(f"[TELEGRAM] Set TELEGRAM_API_ROOT={tg_root}")
748
  print(f"[TELEGRAM] telegram-proxy.cjs will redirect fetch() calls")
749
  elif tg_root:
750
+ print("[TELEGRAM] Official API reachable — no proxy needed")
751
  else:
752
+ print("[TELEGRAM] No reachable endpoint found — Telegram will not work")
753
  try:
754
  # Use Popen without shell to avoid pipe issues
755
+ # auth disabled in config — no token needed
756
  process = subprocess.Popen(
757
  entry_cmd,
758
  cwd=str(APP_DIR),
 
776
  continue
777
  # Skip progress bars and download noise
778
  if any(skip in stripped for skip in [
779
+ 'Downloading', 'Fetching', '%|', '━', '───',
780
  'Already cached', 'Using cache', 'tokenizer',
781
  '.safetensors', 'model-', 'shard',
782
  ]):
 
799
  traceback.print_exc()
800
  return None
801
 
802
+ # ── Main ──────────────────────────────────────â���€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€
803
 
804
  def main():
805
  try:
 
842
  t0 = time.time()
843
  process = sync.run_openclaw()
844
  print(f"[TIMER] run_openclaw launch: {time.time() - t0:.1f}s")
845
+ print(f"[TIMER] Total startup (init → app launched): {time.time() - t_main_start:.1f}s")
846
 
847
  # 4. Start conversation-loop (RUN_ORCHESTRATOR=1, runs on God Space)
848
  # OFFICE_MODE=1 is for frontend serving only (Home Space)
 
883
  conv_thread = threading.Thread(target=run_conversation_loop_forever, daemon=True)
884
  conv_thread.start()
885
  else:
886
+ print("[SYNC] RUN_ORCHESTRATOR not set — skipping conversation-loop")
887
 
888
  # Signal handler
889
  def handle_signal(sig, frame):
 
933
 
934
 
935
  if __name__ == "__main__":
936
+ main()