Add NVIDIA StepFun flash model

.env.example

@@ -105,6 +105,7 @@ LLM_API_KEY=your_api_key_here
 #
 # NVIDIA (NVIDIA_API_KEY):
 #   - nvidia/nemotron-3-super-120b-a12b
+#   - nvidia/stepfun-ai/step-3.7-flash
 #
 # Groq (GROQ_API_KEY):
 #   - groq/mixtral-8x7b-32768
@@ -273,7 +274,7 @@ LLM_API_KEY_FALLBACK_ENABLED=true
 # Optional: explicitly pin model lists per provider for Control UI visibility
 # when provider keys are configured.
 # Format: comma-separated model IDs
-# NVIDIA_MODELS=meta/llama-3.1-70b-instruct,nvidia/llama-3.1-nemotron-70b-instruct
+# NVIDIA_MODELS=meta/llama-3.1-70b-instruct,nvidia/llama-3.1-nemotron-70b-instruct,nvidia/stepfun-ai/step-3.7-flash
 # OPENAI_MODELS=gpt-4o-mini,gpt-4.1
 # GROQ_MODELS=llama-3.3-70b-versatile,deepseek-r1-distill-llama-70b
 #      independently and in parallel.
@@ -395,6 +396,22 @@ OPENCLAW_CONFIG_SETTLE_SECONDS=3
 # Minimum gap between sessions-triggered immediate syncs (seconds). Default: 30.
 SESSIONS_MIN_SYNC_GAP=30
 
+# Max seconds a one-off sync waits for another sync process to release its lock.
+# Startup background sync still keeps retrying; this mainly prevents shutdown or
+# gateway-restart saves from hanging forever behind a stuck upload. Default: 20.
+SYNC_LOCK_TIMEOUT=20
+
+# Shutdown/restart one-shot sync upload budgets. These are intentionally much
+# larger than SYNC_LOCK_TIMEOUT so 40-50 MB files are not killed mid-upload just
+# because another sync lock check needed to be bounded. Set to 0 to disable the
+# outer command timeout completely. Defaults: 120 seconds each.
+SYNC_SETTLED_TIMEOUT=120
+SYNC_FINAL_TIMEOUT=120
+
+# Lock wait used only by shutdown/restart one-shot syncs. Keep this short
+# because start.sh stops the background sync loop before these commands run.
+SYNC_ONE_SHOT_LOCK_TIMEOUT=5
+
 # Webhooks: Standard POST notifications for lifecycle events
 # WEBHOOK_URL=https://your-webhook-endpoint.com/log
 

README.md

@@ -188,6 +188,10 @@ HuggingClaw automatically syncs your workspace (chats, settings, sessions) to a
 | `OPENCLAW_CONFIG_WATCH_INTERVAL` | `1` | How often to check `openclaw.json` for immediate settings sync |
 | `OPENCLAW_CONFIG_SETTLE_SECONDS` | `3` | How long `openclaw.json` must stay valid and unchanged before syncing |
 | `SESSIONS_MIN_SYNC_GAP` | `30` | Minimum seconds between session-triggered immediate syncs |
+| `SYNC_LOCK_TIMEOUT` | `20` | Max seconds one-off syncs wait for another sync lock before failing clearly |
+| `SYNC_SETTLED_TIMEOUT` | `120` | Shutdown/restart settled-sync upload budget; set `0` to disable the outer timeout |
+| `SYNC_FINAL_TIMEOUT` | `120` | Shutdown/restart final catch-up sync upload budget; set `0` to disable the outer timeout |
+| `SYNC_ONE_SHOT_LOCK_TIMEOUT` | `5` | Short lock wait for shutdown/restart one-shot syncs after the background loop is stopped |
 
 ## 📦 Ephemeral Package Re-install *(Optional)*
 

env-builder.js

@@ -142,7 +142,8 @@ const MODEL_CATALOGS = {
     "NVIDIA": [
       "nvidia/nemotron-3-super-120b-a12b",
       "nvidia/nemotron-4-340b-instruct",
-      "nvidia/llama-3.1-nemotron-70b-instruct"
+      "nvidia/llama-3.1-nemotron-70b-instruct",
+      "nvidia/stepfun-ai/step-3.7-flash"
     ],
     "KiloCode": [
       "kilocode/anthropic/claude-opus-4.7",
@@ -376,7 +377,8 @@ const MODEL_CATALOGS = {
   "NVIDIA_MODELS": [
     "nvidia/nemotron-3-super-120b-a12b",
     "nvidia/nemotron-4-340b-instruct",
-    "nvidia/llama-3.1-nemotron-70b-instruct"
+    "nvidia/llama-3.1-nemotron-70b-instruct",
+    "nvidia/stepfun-ai/step-3.7-flash"
   ],
   "KILOCODE_MODELS": [
     "kilocode/anthropic/claude-opus-4.7",

jupyter-devdata-sync.py

@@ -167,6 +167,10 @@ def _matches_prefix(parts: tuple[str, ...], prefix: tuple[str, ...]) -> bool:
     return len(parts) >= len(prefix) and parts[:len(prefix)] == prefix
 
 
+def _is_prefix_of(parts: tuple[str, ...], full_path: tuple[str, ...]) -> bool:
+    return len(parts) <= len(full_path) and full_path[:len(parts)] == parts
+
+
 def should_skip(p: Path):
     # Reserved sync helper files are not user data. Old datasets may still have
     # these markers, but fresh snapshots no longer create them.
@@ -188,15 +192,52 @@ def should_skip(p: Path):
     # Skip any component whose name looks like a secret file/dir.
     return any(_name_is_secret(part) for part in parts)
 
+
+def iter_sync_tree(root: Path):
+    """Yield syncable DevData paths without descending into excluded trees."""
+    if not root.exists():
+        return
+
+    for dirpath, dirnames, filenames in os.walk(root):
+        dir_path = Path(dirpath)
+        try:
+            dir_rel = dir_path.relative_to(root)
+        except ValueError:
+            dir_rel = Path()
+
+        kept_dirnames: list[str] = []
+        for dirname in sorted(dirnames):
+            rel = dir_rel / dirname
+            child = dir_path / dirname
+            rel_parts = rel.parts
+            # Do not prune ancestors of explicitly allowed Jupyter settings
+            # paths. should_skip(.local/share/jupyter) is true by design for
+            # files under that tree, but we must still descend through the
+            # parent dirs to reach lab/user-settings and lab/workspaces.
+            allowed_ancestor = any(
+                _is_prefix_of(rel_parts, prefix) for prefix in JUPYTER_DATA_ALLOW_PREFIXES
+            )
+            if child.is_symlink() or (should_skip(rel) and not allowed_ancestor):
+                continue
+            kept_dirnames.append(dirname)
+        dirnames[:] = kept_dirnames
+
+        for dirname in kept_dirnames:
+            yield dir_path / dirname
+
+        for filename in sorted(filenames):
+            rel = dir_rel / filename
+            child = dir_path / filename
+            if child.is_symlink() or should_skip(rel):
+                continue
+            yield child
+
+
 def snapshot(src: Path, dst: Path) -> tuple[bool, set[str]]:
     had_copy_failures = False
     protected_large_files: set[str] = set()
-    for p in src.rglob("*"):
+    for p in iter_sync_tree(src):
         rel = p.relative_to(src)
-        if should_skip(rel):
-            continue
-        if p.is_symlink():
-            continue
         target = dst / rel
         if p.is_dir():
             # Keep parent directories for files copied later in this snapshot.

openclaw-sync.py

@@ -51,6 +51,7 @@ CONFIG_SETTLE_SECONDS = max(
     float(os.environ.get("OPENCLAW_CONFIG_SETTLE_SECONDS", "3")),
 )
 SESSIONS_MIN_SYNC_GAP = int(os.environ.get("SESSIONS_MIN_SYNC_GAP", "30"))
+SYNC_LOCK_TIMEOUT = max(1.0, float(os.environ.get("SYNC_LOCK_TIMEOUT", "20")))
 HF_TOKEN = os.environ.get("HF_TOKEN", "").strip()
 HF_USERNAME = os.environ.get("HF_USERNAME", "").strip()
 SPACE_AUTHOR_NAME = os.environ.get("SPACE_AUTHOR_NAME", "").strip()
@@ -88,6 +89,22 @@ EXCLUDED_STATE_NAMES = {
     # "WhatsApp enabled but not installing on restart".
     "extensions",
 }
+# Internal restore/snapshot working directories live beside the workspace under
+# /home/node/.openclaw.  If a previous container is killed mid-restore, these
+# hidden directories can be left behind and may contain a full copy of the
+# workspace, including huggingclaw-state/openclaw itself.  Backing them up makes
+# the dataset grow recursively under huggingclaw-state/openclaw and can keep the
+# sync loop busy forever, which in turn blocks session deletes/updates from being
+# pruned remotely.  Treat every such helper path as disposable sync scratch.
+INTERNAL_TEMP_STATE_NAMES = {
+    ".workspace-restore-staging",
+    ".workspace-restore-old",
+    ".openclaw-staging",
+}
+INTERNAL_TEMP_NAME_PREFIXES = (
+    ".workspace-restore-",
+    ".openclaw-staging",
+)
 SESSIONS_ROOT = OPENCLAW_HOME / "agents"
 WHATSAPP_CREDS_DIR = OPENCLAW_HOME / "credentials" / "whatsapp" / "default"
 WHATSAPP_BACKUP_DIR = STATE_DIR / "credentials" / "whatsapp" / "default"
@@ -100,6 +117,20 @@ WorkspaceMarker: TypeAlias = tuple[int, int, int, str]
 # Set True when prune fails so the next sync pass bypasses the fingerprint
 # early-exit and retries the prune even if no local files changed.
 _prune_needed: bool = False
+_remote_temp_prune_done: bool = False
+
+# Workspace-relative temp paths managed by this sync script.  Keep this narrow:
+# user projects may legitimately contain folders with similar names, and those
+# must keep syncing normally.  Only the script-owned state/restore scratch
+# locations are skipped and pruned.
+INTERNAL_TEMP_WORKSPACE_PREFIXES: tuple[tuple[str, ...], ...] = (
+    (".workspace-restore-staging",),
+    (".workspace-restore-old",),
+    ("huggingclaw-state", ".openclaw-staging"),
+    ("huggingclaw-state", "openclaw", ".workspace-restore-staging"),
+    ("huggingclaw-state", "openclaw", ".workspace-restore-old"),
+    ("huggingclaw-state", "openclaw", ".openclaw-staging"),
+)
 
 
 def write_status(status: str, message: str) -> None:
@@ -133,6 +164,86 @@ def _remove_path(path: Path) -> None:
         path.unlink(missing_ok=True)
 
 
+def _is_internal_temp_name(name: str) -> bool:
+    return name in INTERNAL_TEMP_STATE_NAMES or any(
+        name.startswith(prefix) for prefix in INTERNAL_TEMP_NAME_PREFIXES
+    )
+
+
+def _has_internal_temp_part(path: str) -> bool:
+    parts = Path(path).parts
+    return any(_matches_prefix(parts, prefix) for prefix in INTERNAL_TEMP_WORKSPACE_PREFIXES)
+
+
+def _matches_prefix(parts: tuple[str, ...], prefix: tuple[str, ...]) -> bool:
+    return len(parts) >= len(prefix) and parts[:len(prefix)] == prefix
+
+
+def _should_skip_state_entry_name(name: str) -> bool:
+    return (
+        name in EXCLUDED_STATE_NAMES
+        or name in EXCLUDED_SYNC_DIRS
+        or _is_internal_temp_name(name)
+    )
+
+
+def _should_skip_sync_path(rel_parts: tuple[str, ...]) -> bool:
+    return any(part in EXCLUDED_SYNC_DIRS for part in rel_parts) or any(
+        _matches_prefix(rel_parts, prefix) for prefix in INTERNAL_TEMP_WORKSPACE_PREFIXES
+    )
+
+
+def _iter_sync_tree(root: Path):
+    """Yield syncable paths without descending into ignored/temp directories."""
+    if not root.exists():
+        return
+
+    for dirpath, dirnames, filenames in os.walk(root):
+        dir_path = Path(dirpath)
+        try:
+            dir_rel_parts = dir_path.relative_to(root).parts
+        except ValueError:
+            dir_rel_parts = ()
+        if dir_rel_parts == (".",):
+            dir_rel_parts = ()
+        dirnames[:] = sorted(
+            name for name in dirnames
+            if not _should_skip_sync_path(dir_rel_parts + (name,))
+        )
+
+        for dirname in dirnames:
+            yield dir_path / dirname
+        for filename in sorted(filenames):
+            if _should_skip_sync_path(dir_rel_parts + (filename,)):
+                continue
+            yield dir_path / filename
+
+
+def _iter_sync_files(root: Path):
+    for path in _iter_sync_tree(root):
+        if path.is_file():
+            yield path
+
+
+def cleanup_internal_temp_paths() -> None:
+    """Remove stale sync/restore scratch dirs that must never be backed up."""
+    for root in (OPENCLAW_HOME, STATE_DIR):
+        if not root.exists() or not root.is_dir():
+            continue
+        try:
+            children = list(root.iterdir())
+        except OSError:
+            continue
+        for child in children:
+            if not _is_internal_temp_name(child.name):
+                continue
+            try:
+                _remove_path(child)
+                print(f"Removed stale sync scratch path: {child}")
+            except OSError as exc:
+                print(f"Warning: could not remove stale sync scratch path {child}: {exc}")
+
+
 def _copy_hot_directory_snapshot(source_path: Path, tmp_path: Path) -> bool:
     """Best-effort mirror of a hot directory into *tmp_path*.
 
@@ -259,6 +370,7 @@ def copy_state_entry_with_retry(source_path: Path, backup_path: Path, attempts:
 def snapshot_state_into_workspace() -> bool:
     had_copy_failures = False
     try:
+        cleanup_internal_temp_paths()
         STATE_DIR.mkdir(parents=True, exist_ok=True)
         # Atomic snapshot: copy to a staging dir first, then rename.
         # This prevents a half-written (or empty) backup if we crash mid-copy,
@@ -274,7 +386,7 @@ def snapshot_state_into_workspace() -> bool:
         skipped_entries: list[tuple[str, Exception]] = []
         copied_entry_names: set[str] = set()
         for source_path in OPENCLAW_HOME.iterdir():
-            if source_path.name in EXCLUDED_STATE_NAMES:
+            if _should_skip_state_entry_name(source_path.name):
                 continue
 
             backup_path = staging_dir / source_path.name
@@ -300,7 +412,9 @@ def snapshot_state_into_workspace() -> bool:
         # dataset on the next sync.  Only remove an entry from staging when the
         # source has genuinely been deleted from OPENCLAW_HOME.
         for staged_path in list(staging_dir.iterdir()):
-            if staged_path.name in EXCLUDED_STATE_NAMES:
+            if _should_skip_state_entry_name(staged_path.name):
+                if staged_path.exists() or staged_path.is_symlink():
+                    _remove_path(staged_path)
                 continue
             if staged_path.name in copied_entry_names:
                 continue
@@ -425,13 +539,8 @@ def locally_existing_large_files(root: Path) -> set[str]:
     protected: set[str] = set()
     if not root.exists():
         return protected
-    for path in root.rglob("*"):
-        if not path.is_file():
-            continue
+    for path in _iter_sync_files(root):
         rel = path.relative_to(root).as_posix()
-        parts = Path(rel).parts
-        if any(part in EXCLUDED_SYNC_DIRS for part in parts):
-            continue
         try:
             if path.stat().st_size > MAX_FILE_SIZE_BYTES:
                 protected.add(rel)
@@ -441,6 +550,7 @@ def locally_existing_large_files(root: Path) -> set[str]:
 
 
 def restore_embedded_state() -> None:
+    cleanup_internal_temp_paths()
     state_backup_root = STATE_DIR / "openclaw"
 
     # Migration fix: old backups stored state in ".huggingclaw-state/openclaw"
@@ -462,7 +572,7 @@ def restore_embedded_state() -> None:
     if state_backup_root.is_dir():
         for source_path in state_backup_root.iterdir():
             name = source_path.name
-            if name in EXCLUDED_STATE_NAMES:
+            if _should_skip_state_entry_name(name):
                 if source_path.is_dir():
                     shutil.rmtree(source_path, ignore_errors=True)
                 else:
@@ -527,7 +637,7 @@ def ensure_repo_exists() -> str:
 
 def _should_exclude(rel_posix: str, path: Path) -> bool:
     parts = Path(rel_posix).parts
-    if any(part in EXCLUDED_SYNC_DIRS for part in parts):
+    if _should_skip_sync_path(parts):
         return True
     if path.is_file():
         try:
@@ -558,17 +668,12 @@ def metadata_marker(root: Path) -> WorkspaceMarker:
     total_size = 0
     newest_mtime = 0
     metadata_hasher = hashlib.sha256()
-    for path in sorted(root.rglob("*")):
-        if not path.is_file():
-            continue
+    for path in _iter_sync_files(root):
         rel = path.relative_to(root).as_posix()
         # BUG FIX: use directory-only exclusion here (not size-based) so that
         # deleting a large file changes the marker and triggers a sync/prune
         # pass.  Large files are still excluded from the upload snapshot via
         # _should_exclude; this only controls change-detection.
-        parts = Path(rel).parts
-        if any(part in EXCLUDED_SYNC_DIRS for part in parts):
-            continue
         try:
             stat = path.stat()
         except OSError:
@@ -576,14 +681,17 @@ def metadata_marker(root: Path) -> WorkspaceMarker:
         file_count += 1
         size = int(stat.st_size)
         mtime_ns = int(stat.st_mtime_ns)
+        ctime_ns = int(stat.st_ctime_ns)
         total_size += size
-        newest_mtime = max(newest_mtime, mtime_ns)
+        newest_mtime = max(newest_mtime, mtime_ns, ctime_ns)
         metadata_hasher.update(rel.encode("utf-8"))
         metadata_hasher.update(b"\0")
         metadata_hasher.update(str(size).encode("ascii"))
         metadata_hasher.update(b"\0")
         metadata_hasher.update(str(mtime_ns).encode("ascii"))
         metadata_hasher.update(b"\0")
+        metadata_hasher.update(str(ctime_ns).encode("ascii"))
+        metadata_hasher.update(b"\0")
     return (file_count, total_size, newest_mtime, metadata_hasher.hexdigest())
 
 
@@ -592,15 +700,12 @@ def fingerprint_dir(root: Path) -> str:
     if not root.exists():
         return hasher.hexdigest()
 
-    for path in sorted(p for p in root.rglob("*") if p.is_file()):
+    for path in _iter_sync_files(root):
         rel = path.relative_to(root).as_posix()
         # BUG FIX: use directory-only exclusion (not size-based) so that
         # creating or deleting a large file changes the fingerprint.
         # Large files are hashed by path + metadata only (no content read)
         # to avoid reading gigabytes for a change-detection hash.
-        parts = Path(rel).parts
-        if any(part in EXCLUDED_SYNC_DIRS for part in parts):
-            continue
         hasher.update(rel.encode("utf-8"))
         try:
             stat = path.stat()
@@ -628,7 +733,7 @@ def fingerprint_dir(root: Path) -> str:
 
 def create_snapshot_dir(source_root: Path) -> Path:
     staging_root = Path(tempfile.mkdtemp(prefix="huggingclaw-sync-"))
-    for path in sorted(source_root.rglob("*")):
+    for path in _iter_sync_tree(source_root):
         rel = path.relative_to(source_root)
         rel_posix = rel.as_posix()
         if _should_exclude(rel_posix, path):
@@ -679,15 +784,19 @@ def prune_remote_deleted_files(
         and path not in protected_paths
         and not any(path == prefix or path.startswith(prefix + "/") for prefix in skip_prefixes)
     ]
-    if not stale_files:
+    _commit_delete_batches(repo_id, stale_files, "Prune", "stale file(s) after workspace sync")
+
+
+def _commit_delete_batches(repo_id: str, paths: list[str], action: str, reason: str) -> None:
+    if not paths:
         return
 
-    total = len(stale_files)
+    total = len(paths)
     num_batches = (total + PRUNE_BATCH_SIZE - 1) // PRUNE_BATCH_SIZE
     for batch_idx in range(num_batches):
-        batch = stale_files[batch_idx * PRUNE_BATCH_SIZE:(batch_idx + 1) * PRUNE_BATCH_SIZE]
+        batch = paths[batch_idx * PRUNE_BATCH_SIZE:(batch_idx + 1) * PRUNE_BATCH_SIZE]
         batch_label = f"{batch_idx + 1}/{num_batches}" if num_batches > 1 else ""
-        msg = f"Prune {len(batch)} stale file(s) after workspace sync"
+        msg = f"{action} {len(batch)} {reason}"
         if batch_label:
             msg += f" (batch {batch_label})"
         operations = [CommitOperationDelete(path_in_repo=path) for path in batch]
@@ -698,7 +807,25 @@ def prune_remote_deleted_files(
             commit_message=msg,
         )
         if num_batches > 1:
-            print(f"Pruned batch {batch_label}: {len(batch)} file(s)")
+            print(f"{action} batch {batch_label}: {len(batch)} file(s)")
+
+
+def prune_remote_internal_temp_files(repo_id: str) -> None:
+    """Remove old recursive scratch files from the HF backup even if nothing changed locally."""
+    if HF_API is None:
+        return
+
+    remote_files = list(HF_API.list_repo_files(repo_id=repo_id, repo_type="dataset"))
+    internal_temp_files = [
+        path for path in remote_files
+        if path != ".gitattributes" and _has_internal_temp_part(path)
+    ]
+    _commit_delete_batches(
+        repo_id,
+        internal_temp_files,
+        "Prune internal sync scratch",
+        "file(s) from backup",
+    )
 
 
 def restore_workspace() -> bool:
@@ -795,10 +922,20 @@ def _sync_once_unlocked(
         write_status("disabled", "HF_TOKEN is not configured.")
         return (last_fingerprint or "", last_marker or (0, 0, 0, ""))
 
-    global _prune_needed
+    global _prune_needed, _remote_temp_prune_done
 
     had_snapshot_copy_failures = snapshot_state_into_workspace()
     repo_id = ensure_repo_exists()
+    if not _remote_temp_prune_done:
+        try:
+            prune_remote_internal_temp_files(repo_id)
+            _remote_temp_prune_done = True
+        except Exception as temp_prune_exc:
+            # Do not block normal user-data sync if the cleanup commit fails;
+            # leave the flag false so the next pass retries before taking the
+            # metadata/fingerprint early exit again.
+            print(f"Warning: could not prune internal sync scratch files: {temp_prune_exc}")
+            _prune_needed = True
     current_marker = metadata_marker(WORKSPACE)
     # Session watcher uses content digests and can detect same-size rewrites
     # whose workspace metadata marker is unchanged.  In that case, force the
@@ -879,7 +1016,18 @@ def sync_once(
 ) -> tuple[str, WorkspaceMarker]:
     SYNC_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True)
     with SYNC_LOCK_FILE.open("w", encoding="utf-8") as lock_handle:
-        fcntl.flock(lock_handle, fcntl.LOCK_EX)
+        deadline = time.monotonic() + SYNC_LOCK_TIMEOUT
+        while True:
+            try:
+                fcntl.flock(lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                break
+            except BlockingIOError:
+                if time.monotonic() >= deadline:
+                    raise TimeoutError(
+                        f"Timed out waiting {SYNC_LOCK_TIMEOUT:.0f}s for workspace sync lock."
+                    )
+                if STOP_EVENT.wait(0.25):
+                    raise TimeoutError("Stopped while waiting for workspace sync lock.")
         try:
             return _sync_once_unlocked(
                 last_fingerprint,

start.sh

@@ -657,7 +657,7 @@ _DEFAULT_XAI_MODELS="xai/grok-4.20,xai/grok-4.3,xai/grok-4.1,xai/grok-latest,xai
 _DEFAULT_COHERE_MODELS="cohere/command-a,cohere/command-a-03-2025,cohere/command-a-reasoning-08-2025,cohere/command-r-plus-08-2024"
 _DEFAULT_TOGETHER_MODELS="together/moonshotai/Kimi-K2.6,together/deepseek-ai/DeepSeek-V4-Pro,together/Qwen/Qwen3-235B-A22B-Instruct-2507-tput,together/meta-llama/Llama-3.3-70B-Instruct-Turbo"
 _DEFAULT_CEREBRAS_MODELS="cerebras/zai-glm-4.7,cerebras/gpt-oss-120b,cerebras/deepseek-r1,cerebras/qwen3-32b"
-_DEFAULT_NVIDIA_MODELS="nvidia/nemotron-3-super-120b-a12b,nvidia/nemotron-4-340b-instruct,nvidia/llama-3.1-nemotron-70b-instruct"
+_DEFAULT_NVIDIA_MODELS="nvidia/nemotron-3-super-120b-a12b,nvidia/nemotron-4-340b-instruct,nvidia/llama-3.1-nemotron-70b-instruct,nvidia/stepfun-ai/step-3.7-flash"
 _DEFAULT_KILOCODE_MODELS="kilocode/kilo/auto,kilocode/anthropic/claude-opus-4.7,kilocode/openai/gpt-5.4,kilocode/google/gemini-2.5-pro"
 _DEFAULT_MOONSHOT_MODELS="moonshot/kimi-k2.6,moonshot/kimi-k2.6-thinking,moonshot/kimi-k2-thinking"
 _DEFAULT_MINIMAX_MODELS="minimax/MiniMax-M2.7,minimax/minimax-m1.5"
@@ -1414,32 +1414,63 @@ if [ -n "${WEBHOOK_URL:-}" ]; then
 fi
 
 # ── Trap SIGTERM for graceful shutdown ──
+stop_background_sync_loop() {
+  [ -n "${SYNC_LOOP_PID:-}" ] || return 0
+
+  if ! kill -0 "$SYNC_LOOP_PID" 2>/dev/null; then
+    SYNC_LOOP_PID=""
+    return 0
+  fi
+
+  kill "$SYNC_LOOP_PID" 2>/dev/null || true
+  # Wait for the Python process to actually exit so its fcntl lock is released.
+  # A fixed short sleep was not enough when the process was inside a HF upload,
+  # which made the following one-shot syncs burn their timeout waiting on the
+  # lock and left sessions/config changes unsaved.
+  for _sync_stop_i in 1 2 3 4 5 6 7 8 9 10; do
+    if ! kill -0 "$SYNC_LOOP_PID" 2>/dev/null; then
+      SYNC_LOOP_PID=""
+      unset _sync_stop_i
+      return 0
+    fi
+    sleep 0.2
+  done
+
+  echo "Warning: workspace sync loop did not stop after SIGTERM; forcing it to release the sync lock."
+  kill -9 "$SYNC_LOOP_PID" 2>/dev/null || true
+  sleep 0.2
+  SYNC_LOOP_PID=""
+  unset _sync_stop_i
+}
+
+run_openclaw_sync_with_timeout() {
+  local timeout_seconds="$1"
+  local command_name="$2"
+  local lock_timeout_seconds="$3"
+  if [ "${timeout_seconds}" = "0" ]; then
+    env SYNC_LOCK_TIMEOUT="$lock_timeout_seconds" \
+      python3 /home/node/app/openclaw-sync.py "$command_name"
+  else
+    timeout --kill-after=10s "${timeout_seconds}s" env SYNC_LOCK_TIMEOUT="$lock_timeout_seconds" \
+      python3 /home/node/app/openclaw-sync.py "$command_name"
+  fi
+}
+
 graceful_shutdown() {
   echo "Shutting down..."
   if [ -f "/home/node/app/openclaw-sync.py" ] && [ -n "${HF_TOKEN:-}" ]; then
     echo "Saving state before exit..."
-    # BUG FIX: kill the background sync loop *before* running the shutdown
-    # syncs.  The loop holds the sync lock while uploading; if it is
-    # mid-upload when sync-once-settled runs, the 8-second timeout fires
-    # before the lock is released and the settled-sync is silently skipped.
-    # Killing the loop first releases the lock immediately (fcntl.flock is
-    # released on process exit) so sync-once-settled can acquire it cleanly.
-    if [ -n "${SYNC_LOOP_PID:-}" ]; then
-      kill "$SYNC_LOOP_PID" 2>/dev/null || true
-      # Give Python a moment to flush and release the lock file.
-      # Reduced from 0.5 s → 0.3 s to reclaim time for the upload.
-      sleep 0.3
-    fi
+    stop_background_sync_loop
     # Pass 1: wait for config to settle then upload — avoids pushing a
-    # half-written JSON config to the dataset.  Timeout raised from 8 s
-    # to 15 s so the 3-second settle window + actual upload both fit
-    # within the budget (old 8 s left only 5 s for the upload itself).
-    timeout 15s python3 /home/node/app/openclaw-sync.py sync-once-settled || \
+    # half-written JSON config to the dataset.  The per-command lock wait is
+    # intentionally shorter than the outer timeout so a stuck lock fails clearly
+    # and leaves time for the final catch-up pass.
+    run_openclaw_sync_with_timeout "${SYNC_SETTLED_TIMEOUT:-120}" sync-once-settled "${SYNC_ONE_SHOT_LOCK_TIMEOUT:-5}" || \
       echo "Warning: could not complete settled shutdown sync"
     # Pass 2: catch any writes that arrived after the settled sync completed.
     # BUG FIX: added timeout (previously unbounded) so HF container kill
     # can't interrupt a hung upload and lose all data silently.
-    timeout 8s python3 /home/node/app/openclaw-sync.py sync-once || \
+    run_openclaw_sync_with_timeout "${SYNC_FINAL_TIMEOUT:-120}" sync-once "${SYNC_ONE_SHOT_LOCK_TIMEOUT:-5}" || \
       echo "Warning: could not complete final shutdown sync"
   elif [ -f "/home/node/app/openclaw-sync.py" ]; then
     echo "HF_TOKEN not set; skipping shutdown backup sync."
@@ -2439,27 +2470,21 @@ sync_before_gateway_restart() {
 
   echo "Gateway stopped; saving latest OpenClaw state before restart..."
   # Kill the background sync loop before syncing — same reason as in
-  # graceful_shutdown: the loop holds the fcntl.flock while uploading;
-  # if it is mid-upload when sync-once-settled runs, the lock contention
-  # will eat into (or exhaust) the 15s timeout budget, silently skipping
-  # the upload.  Killing the loop releases the lock immediately (fcntl.flock
-  # is released on process exit) so sync-once-settled can acquire it cleanly.
-  if [ -n "${SYNC_LOOP_PID:-}" ]; then
-    kill "$SYNC_LOOP_PID" 2>/dev/null || true
-    sleep 0.3
-    SYNC_LOOP_PID=""
-  fi
+  # graceful_shutdown: the loop holds the fcntl.flock while uploading.  Wait for
+  # it to exit (and force-kill as a last resort) before the one-shot syncs so
+  # gateway restarts cannot hang behind a stale lock.
+  stop_background_sync_loop
   # Pass 1: wait for config to settle then upload — avoids pushing a
   # half-written JSON config to the dataset.  Timeout added (was unbounded)
   # so a slow HF upload cannot stall gateway restarts indefinitely.
-  timeout 15s python3 /home/node/app/openclaw-sync.py sync-once-settled || \
+  run_openclaw_sync_with_timeout "${SYNC_SETTLED_TIMEOUT:-120}" sync-once-settled "${SYNC_ONE_SHOT_LOCK_TIMEOUT:-5}" || \
     echo "Warning: could not sync settled state before gateway restart"
   # Pass 2: catch any writes that arrived after the settled sync completed
   # (e.g. session state flushed by OpenClaw just before exit).
   # BUG FIX: this second pass was missing from the restart path (it existed
   # only in graceful_shutdown), so last-second writes were lost on watchdog
   # restarts — causing sessions to disappear after OpenClaw auto-restarts.
-  timeout 8s python3 /home/node/app/openclaw-sync.py sync-once || \
+  run_openclaw_sync_with_timeout "${SYNC_FINAL_TIMEOUT:-120}" sync-once "${SYNC_ONE_SHOT_LOCK_TIMEOUT:-5}" || \
     echo "Warning: could not complete final sync before gateway restart"
 }