Update hf-sync-manager.mjs

hf-sync-manager.mjs

@@ -1,205 +1,434 @@
 // ============================================================
-// HF Memory Sync Manager - Definitive Edition
+// HF Sync Manager - Dataset as Runtime Storage
 //
-// Based on actual OpenClaw disk layout (from official docs):
+// Concept: HF Dataset IS the persistent storage layer.
+// OpenClaw writes to local disk. We mirror to Dataset every 30min.
+// On container restart: restore from Dataset before gateway starts.
 //
-//   ~/.openclaw/openclaw.json          <- gateway config
-//   ~/.openclaw/workspace/             <- agent brain (THIS is memory)
-//     AGENTS.md                        <- operating instructions
-//     SOUL.md                          <- persona / tone
-//     USER.md                          <- user profile
-//     IDENTITY.md                       <- name / emoji
-//     TOOLS.md                          <- env conventions
-//     HEARTBEAT.md                     <- periodic tasks
-//     MEMORY.md                        <- long-term memory (iron rules)
-//     memory/YYYY-MM-DD.md             <- daily logs
-//     memory/archive/                  <- logs older than 30 days
-//     checklists/                      <- op checklists
-//     docs/                            <- on-demand reference docs
-//   ~/.openclaw/agents/<id>/sessions/  <- DO NOT sync (session transcripts)
-//   ~/.openclaw/agents/<id>/qmd/       <- DO NOT sync (sqlite index, rebuilt on boot)
+// What gets synced (everything OpenClaw writes at runtime):
+//   ~/.openclaw/workspace/    <- agent brain, memory, persona files
+//   ~/.openclaw/openclaw.json <- gateway config (JSON-validated)
 //
-// Repo structure:
-//   settings/openclaw.json             <- single copy, validated, overwrite
-//   workspace/                         <- full workspace mirror
-//     AGENTS.md / SOUL.md / MEMORY.md / memory/ …
-//   README.md                          <- auto-generated index
+// What is intentionally excluded:
+//   ~/.openclaw/agents/*/sessions/  <- session transcripts (large, rebuild ok)
+//   ~/.openclaw/agents/*/qmd/       <- sqlite vector index (rebuilt on boot)
+//   ~/.openclaw/canvas/             <- UI runtime files
 //
-// Rules:
-//   TRACK 1 - settings:
-//     Boot:    repo -> local (skip if repo JSON invalid)
-//     Upload:  local -> repo (skip if local JSON invalid)
-//     One file, one copy, always overwrite
+// Boot sequence (entrypoint.sh calls this before gateway starts):
+//   1. Clone/pull Dataset repo
+//   2. Restore workspace/ from repo
+//   3. Gateway starts with full memory intact
 //
-//   TRACK 2 - workspace:
-//     Boot:    repo/workspace/ -> ~/.openclaw/workspace/ (full restore)
-//     Upload:  ~/.openclaw/workspace/ -> repo/workspace/ every 3h
-//     Never downloads during runtime (prevents memory flooding)
-//
-// Multi-bot:
-//   Set BOT1_PROFILE / BOT2_PROFILE / BOT3_PROFILE ... in HF Space Variables.
-//   The script will automatically use the first defined BOTx_PROFILE as OPENCLAW_PROFILE.
-//   OpenClaw uses ~/.openclaw/workspace-<profile>/ per profile.
-//   Repo stores each profile under workspace-<profile>/.
+// Runtime loop:
+//   Every 30 min: workspace + config -> Dataset repo -> git push
 // ============================================================
 
-import fs   from "node:fs";
+import fs from "node:fs";
 import path from "node:path";
 import { execSync } from "node:child_process";
 
-// -- config ----------------------------------
-
-var HOME         = process.env.OPENCLAW_HOME || process.env.HOME || "/home/user";
-var OPENCLAW_DIR = path.join(HOME, ".openclaw");
-var HF_TOKEN     = (process.env.HF_TOKEN      || "").trim();
-var DATASET_ID   = (process.env.HF_DATASET_ID  || "").trim();
-var REPO_DIR     = "/tmp/hf-memory-sync";
-
-// --- Multi-bot support ----------------------
-// Automatically detect BOTx_PROFILE from HF Variables
-var PROFILE = "";
-for (let i = 1; i <= 10; i++) { // 支持最多 10 个 bot，可自由扩展
-    let envVar = process.env[`BOT${i}_PROFILE`];
-    if (envVar && envVar.trim()) {
-        PROFILE = envVar.trim();
-        break; // 取第一个存在的 BOTx_PROFILE
-    }
-}
+// -- config ---------------------------------------------------
 
-// OPENCLAW_PROFILE fallback (legacy)
-if (!PROFILE) {
-    PROFILE = (process.env.OPENCLAW_PROFILE || "").trim();
-}
+var HOME          = process.env.OPENCLAW_HOME || process.env.HOME || "/home/user";
+var OPENCLAW_DIR  = path.join(HOME, ".openclaw");
+var WORKSPACE_DIR = path.join(OPENCLAW_DIR, "workspace");
+var HF_TOKEN      = (process.env.HF_TOKEN || "").trim();
+var DATASET_ID    = (process.env.HF_DATASET_ID || "").trim();
+var REPO_DIR      = "/tmp/oc-dataset";
 
-// Compute workspace dirs
-var WORKSPACE_DIR = PROFILE
-    ? path.join(OPENCLAW_DIR, "workspace-" + PROFILE)
-    : path.join(OPENCLAW_DIR, "workspace");
-
-var REPO_WORKSPACE_FOLDER = PROFILE ? "workspace-" + PROFILE : "workspace";
-
-var UPLOAD_INTERVAL = 3 * 60 * 60 * 1000; // 3 hours
-
-// -- workspace files OpenClaw actually uses -----------------
-var WORKSPACE_SYNC = [
-"AGENTS.md",
-"SOUL.md",
-"USER.md",
-"IDENTITY.md",
-"TOOLS.md",
-"HEARTBEAT.md",
-"BOOT.md",
-"MEMORY.md",
-"memory",       // directory: memory/YYYY-MM-DD.md + memory/archive/
-"checklists",   // directory: op checklists
-"docs"          // directory: on-demand reference docs
-];
+// Upload every 30 min during runtime
+var UPLOAD_INTERVAL = 30 * 60 * 1000;
 
-// Never touch these (sessions and search index are runtime-only)
-var NEVER_SYNC = [
-"agents",       // ~/.openclaw/agents/<id>/sessions/ and qmd/ - runtime only
-"canvas",       // canvas UI files - runtime only
-".git",
-"node_modules"
+// Workspace files OpenClaw actually uses (official docs)
+var WORKSPACE_FILES = [
+  "AGENTS.md",
+  "SOUL.md",
+  "USER.md",
+  "IDENTITY.md",
+  "TOOLS.md",
+  "HEARTBEAT.md",
+  "BOOT.md",
+  "MEMORY.md",
+  "memory",
+  "checklists",
+  "docs"
 ];
 
-// -- helpers ----------------------------------
+// -- helpers --------------------------------------------------
 
 function log(msg) {
-    console.log("[hf-sync] " + msg);
+  console.log("[hf-sync] " + msg);
 }
 
 function sleep(ms) {
-    return new Promise(function(resolve) { setTimeout(resolve, ms); });
+  return new Promise(function(resolve) { setTimeout(resolve, ms); });
 }
 
 function repoUrl() {
-    return "https://user:" + HF_TOKEN +
-           "@huggingface.co/datasets/" + DATASET_ID;
+  return "https://user:" + HF_TOKEN + "@huggingface.co/datasets/" + DATASET_ID;
 }
 
 function git(args) {
-    return execSync("git " + args, {
-        cwd:     REPO_DIR,
-        stdio:   "pipe",
-        timeout: 120000,
-        env:     Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
-    }).toString().trim();
+  return execSync("git " + args, {
+    cwd: REPO_DIR,
+    stdio: "pipe",
+    timeout: 120000,
+    env: Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
+  }).toString().trim();
 }
 
 function isValidJson(filePath) {
-    try {
-        if (!fs.existsSync(filePath)) return false;
-        var content = fs.readFileSync(filePath, "utf-8").trim();
-        if (!content) return false;
-        JSON.parse(content);
-        return true;
-    } catch (e) {
-        return false;
-    }
+  try {
+    if (!fs.existsSync(filePath)) return false;
+    var content = fs.readFileSync(filePath, "utf-8").trim();
+    if (!content) return false;
+    JSON.parse(content);
+    return true;
+  } catch (e) {
+    return false;
+  }
 }
 
-function copyItem(src, dst) {
-    if (fs.statSync(src).isDirectory()) {
-        execSync("cp -a " + src + " " + dst, { stdio: "pipe" });
-    } else {
-        fs.mkdirSync(path.dirname(dst), { recursive: true });
-        fs.copyFileSync(src, dst);
-    }
+function copyDir(src, dst) {
+  execSync("cp -a " + src + "/. " + dst + "/", { stdio: "pipe" });
+}
+
+function copyFile(src, dst) {
+  fs.mkdirSync(path.dirname(dst), { recursive: true });
+  fs.copyFileSync(src, dst);
 }
 
-// -- git repo setup --------------------------
+// -- git repo setup ------------------------------------------
 
 function ensureRepo() {
-    if (fs.existsSync(path.join(REPO_DIR, ".git"))) return;
-    execSync("rm -rf " + REPO_DIR, { stdio: "pipe" });
+  if (fs.existsSync(path.join(REPO_DIR, ".git"))) return;
+  execSync("rm -rf " + REPO_DIR, { stdio: "pipe" });
+  try {
+    execSync(
+      "git clone --depth 1 " + repoUrl() + " " + REPO_DIR,
+      {
+        stdio: "pipe",
+        timeout: 120000,
+        env: Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
+      }
+    );
+    log("Repo cloned from Dataset");
+  } catch (e) {
+    log("Clone failed, initializing empty repo: " + e.message);
+    fs.mkdirSync(REPO_DIR, { recursive: true });
+    execSync("git init " + REPO_DIR, { stdio: "pipe" });
+    git("remote add origin " + repoUrl());
+  }
+  try {
+    git("config user.email openclaw@hf.space");
+    git("config user.name OpenClaw-Sync");
+    git("config pull.rebase false");
+  } catch (e) { /* non-fatal */ }
+}
+
+function pullLatest() {
+  try { git("fetch --quiet origin"); } catch (e) { /* ignore */ }
+  try { git("pull --quiet --no-rebase origin main"); }
+  catch (e) {
+    try { git("pull --quiet --no-rebase origin master"); }
+    catch (e2) { /* repo empty, first push */ }
+  }
+}
+
+function commitAndPush() {
+  try {
+    git("add -A");
+    var changed = git("diff --cached --name-only");
+    if (!changed) return 0;
+    var count = changed.split("\n").filter(Boolean).length;
+    var ts    = new Date().toISOString().replace("T", " ").substring(0, 19);
+    git('commit -m "runtime-sync ' + ts + '"');
+    try { git("push --quiet origin HEAD:main"); }
+    catch (e) { git("push --quiet origin HEAD:master"); }
+    return count;
+  } catch (e) {
+    log("Push failed: " + e.message);
+    return 0;
+  }
+}
+
+// -- README ---------------------------------------------------
+
+function writeReadme() {
+  var ts = new Date().toISOString().replace("T", " ").substring(0, 19);
+  var content = [
+    "# OpenClaw Runtime Dataset",
+    "",
+    "This repository is the persistent storage layer for an OpenClaw gateway",
+    "running on Hugging Face Spaces. It is auto-managed - do not edit manually.",
+    "",
+    "Last sync: " + ts,
+    "",
+    "## Structure",
+    "",
+    "`",
+    "openclaw.json        <- gateway config (JSON-validated, auto-generated)",
+    "workspace/           <- agent brain (everything OpenClaw remembers)",
+    "  AGENTS.md          <- operating instructions",
+    "  SOUL.md            <- persona / tone / language",
+    "  USER.md            <- user profile",
+    "  IDENTITY.md        <- agent name and emoji",
+    "  TOOLS.md           <- environment conventions",
+    "  HEARTBEAT.md       <- periodic task instructions",
+    "  MEMORY.md          <- long-term memory (iron rules)",
+    "  memory/            <- daily logs: YYYY-MM-DD.md",
+    "    archive/         <- logs older than 30 days",
+    "  checklists/        <- operation checklists",
+    "  docs/              <- on-demand reference docs",
+    "`",
+    "",
+    "## What is NOT stored here",
+    "",
+    "- Session transcripts (`agents/*/sessions/`) - large, not needed for memory",
+    "- Vector search index (`agents/*/qmd/`) - rebuilt automatically on boot",
+    "- Canvas UI files - runtime only",
+    "",
+    "## Container restart recovery",
+    "",
+    "On every container start, the sync manager pulls this repo and restores",
+    "the workspace before OpenClaw boots. Memory is never lost across restarts."
+  ].join("\n") + "\n";
+
+  fs.writeFileSync(path.join(REPO_DIR, "README.md"), content, "utf-8");
+}
+
+// -- seed default workspace files ----------------------------
+
+function seedWorkspaceIfEmpty() {
+  fs.mkdirSync(WORKSPACE_DIR, { recursive: true });
+  fs.mkdirSync(path.join(WORKSPACE_DIR, "memory"), { recursive: true });
+
+  var soulPath = path.join(WORKSPACE_DIR, "SOUL.md");
+  if (!fs.existsSync(soulPath)) {
+    fs.writeFileSync(soulPath, [
+      "# Soul",
+      "",
+      "You are a helpful, warm, and concise AI assistant.",
+      "",
+      "## Language",
+      "",
+      "Always respond in Simplified Chinese by default.",
+      "Switch to the user's language only if they explicitly write in another language.",
+      "",
+      "## Tone",
+      "",
+      "- Friendly and natural, not overly formal",
+      "- Get to the point quickly",
+      "- Ask clarifying questions when needed"
+    ].join("\n") + "\n", "utf-8");
+    log("[workspace] Seeded SOUL.md (Chinese language configured)");
+  }
+
+  var agentsPath = path.join(WORKSPACE_DIR, "AGENTS.md");
+  if (!fs.existsSync(agentsPath)) {
+    fs.writeFileSync(agentsPath, [
+      "# Agent Instructions",
+      "",
+      "## Boot sequence",
+      "",
+      "1. Read SOUL.md - persona and language rules",
+      "2. Read USER.md if present - user profile",
+      "3. Read MEMORY.md - long-term memory and iron rules",
+      "4. Read today's and yesterday's memory/YYYY-MM-DD.md if present",
+      "",
+      "## Memory rules",
+      "",
+      "- Save important facts to MEMORY.md",
+      "- Log daily context to memory/YYYY-MM-DD.md",
+      "- Do not repeat information the user has already told you"
+    ].join("\n") + "\n", "utf-8");
+    log("[workspace] Seeded AGENTS.md");
+  }
+
+  var memoryPath = path.join(WORKSPACE_DIR, "MEMORY.md");
+  if (!fs.existsSync(memoryPath)) {
+    fs.writeFileSync(memoryPath, [
+      "# Long-term Memory",
+      "",
+      "<!-- OpenClaw writes important facts here. -->",
+      "<!-- Loaded at the start of every main session. -->"
+    ].join("\n") + "\n", "utf-8");
+    log("[workspace] Seeded MEMORY.md");
+  }
+}
+
+// ============================================================
+// BOOT RESTORE
+// Called once before gateway starts. Pulls Dataset -> local disk.
+// ============================================================
+
+function bootRestore() {
+  if (!HF_TOKEN || !DATASET_ID) {
+    log("HF_TOKEN or HF_DATASET_ID not set - running without Dataset backup");
+    seedWorkspaceIfEmpty();
+    return;
+  }
+
+  log("Boot restore from Dataset: " + DATASET_ID);
+
+  try {
+    ensureRepo();
+    pullLatest();
+  } catch (e) {
+    log("Repo init failed (non-fatal, starting fresh): " + e.message);
+    seedWorkspaceIfEmpty();
+    return;
+  }
+
+  // Restore workspace
+  var repoWorkspace = path.join(REPO_DIR, "workspace");
+  if (fs.existsSync(repoWorkspace)) {
+    fs.mkdirSync(WORKSPACE_DIR, { recursive: true });
     try {
-        execSync(
-            "git clone --depth 1 " + repoUrl() + " " + REPO_DIR,
-            {
-                stdio:   "pipe",
-                timeout: 120000,
-                env:     Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
-            }
-        );
+      copyDir(repoWorkspace, WORKSPACE_DIR);
+      var count = fs.readdirSync(WORKSPACE_DIR).length;
+      log("Workspace restored: " + count + " item(s) -> " + WORKSPACE_DIR);
     } catch (e) {
-        fs.mkdirSync(REPO_DIR, { recursive: true });
-        execSync("git init " + REPO_DIR, { stdio: "pipe" });
-        git("remote add origin " + repoUrl());
+      log("Workspace restore failed: " + e.message);
     }
-    try {
-        git("config user.email openclaw@hf.space");
-        git("config user.name OpenClaw-Sync");
-        git("config pull.rebase false");
-    } catch (e) { /* non-fatal */ }
-}
+  } else {
+    log("No workspace in Dataset yet - starting fresh");
+  }
 
-function pullLatest() {
-    try { git("fetch --quiet origin"); } catch (e) { /* ignore */ }
+  // Restore openclaw.json ONLY if repo has valid one AND local doesn't exist yet
+  var repoConfig = path.join(REPO_DIR, "openclaw.json");
+  var localConfig = path.join(OPENCLAW_DIR, "openclaw.json");
+  if (!fs.existsSync(localConfig) && isValidJson(repoConfig)) {
     try {
-        git("pull --quiet --no-rebase origin main");
+      fs.mkdirSync(OPENCLAW_DIR, { recursive: true });
+      copyFile(repoConfig, localConfig);
+      log("Config restored from Dataset (was missing locally)");
     } catch (e) {
-        try { git("pull --quiet --no-rebase origin master"); } catch (e2) { /* no remote yet */ }
+      log("Config restore failed: " + e.message);
     }
+  }
+
+  // Seed workspace files that don't exist yet
+  seedWorkspaceIfEmpty();
+
+  log("Boot restore complete.");
 }
 
-function commitAndPush(label) {
-    try {
-        git("add -A");
-        var changed = git("diff --cached --name-only");
-        if (!changed) return 0;
-        var count = changed.split("\n").filter(Boolean).length;
-        var ts    = new Date().toISOString().replace("T", " ").substring(0, 19);
-        var tag   = PROFILE ? "[" + PROFILE + "]" : "";
-        git("commit -m \"sync" + tag + " " + ts + "\"");
-        try {
-            git("push --quiet origin HEAD:main");
-        } catch (e) {
-            git("push --quiet origin HEAD:master");
+// ============================================================
+// UPLOAD CYCLE
+// Runs every 30 min. Pushes local state to Dataset.
+// Never downloads during runtime.
+// ============================================================
+
+function runUploadCycle() {
+  if (!HF_TOKEN || !DATASET_ID) return;
+
+  var ts = new Date().toISOString().replace("T", " ").substring(0, 19);
+  log("Upload cycle at " + ts + "...");
+
+  try {
+    ensureRepo();
+    pullLatest();
+
+    // Upload workspace
+    var repoWorkspace = path.join(REPO_DIR, "workspace");
+    if (fs.existsSync(WORKSPACE_DIR)) {
+      var existing = WORKSPACE_FILES.filter(function(name) {
+        return fs.existsSync(path.join(WORKSPACE_DIR, name));
+      });
+
+      if (existing.length > 0) {
+        // Clear and re-copy workspace in repo
+        if (fs.existsSync(repoWorkspace)) {
+          execSync("rm -rf " + repoWorkspace, { stdio: "pipe" });
         }
-        return count;
-    } catch (e) {
-        log("Commit/push failed: " + e.message);
-        return 0;
+        fs.mkdirSync(repoWorkspace, { recursive: true });
+
+        existing.forEach(function(name) {
+          var src = path.join(WORKSPACE_DIR, name);
+          var dst = path.join(repoWorkspace, name);
+          try {
+            var stat = fs.statSync(src);
+            if (stat.isDirectory()) {
+              fs.mkdirSync(dst, { recursive: true });
+              copyDir(src, dst);
+            } else {
+              copyFile(src, dst);
+            }
+          } catch (e) {
+            log("Copy error [" + name + "]: " + e.message);
+          }
+        });
+        log("Workspace uploaded: " + existing.join(", "));
+      } else {
+        log("No workspace files found yet - skipping workspace upload");
+      }
     }
-}
+
+    // Upload openclaw.json (validated)
+    var localConfig = path.join(OPENCLAW_DIR, "openclaw.json");
+    if (isValidJson(localConfig)) {
+      copyFile(localConfig, path.join(REPO_DIR, "openclaw.json"));
+      log("Config uploaded (JSON valid)");
+    } else {
+      log("Config skipped (invalid JSON)");
+    }
+
+    writeReadme();
+
+    var changed = commitAndPush();
+    if (changed > 0) {
+      log("Upload done: " + changed + " file(s) pushed to Dataset");
+    } else {
+      log("Upload done: no changes");
+    }
+
+  } catch (e) {
+    log("Upload failed (will retry in 30 min): " + e.message);
+  }
+}
+
+// ============================================================
+// MAIN
+// ============================================================
+
+async function main() {
+  log("Starting...");
+  log("OPENCLAW_DIR  : " + OPENCLAW_DIR);
+  log("WORKSPACE_DIR : " + WORKSPACE_DIR);
+  log("HF_DATASET_ID : " + (DATASET_ID || "NOT SET"));
+  log("Upload every  : " + (UPLOAD_INTERVAL / 60000) + " min");
+
+  fs.mkdirSync(OPENCLAW_DIR,  { recursive: true });
+  fs.mkdirSync(WORKSPACE_DIR, { recursive: true });
+
+  try {
+    execSync("git config --global user.email openclaw@hf.space", { stdio: "pipe" });
+    execSync("git config --global user.name OpenClaw-Sync", { stdio: "pipe" });
+    execSync("git config --global http.postBuffer 52428800", { stdio: "pipe" });
+    execSync("git config --global pull.rebase false", { stdio: "pipe" });
+  } catch (e) { /* non-fatal */ }
+
+  // Step 1: restore from Dataset (runs before gateway is fully up)
+  bootRestore();
+
+  // Step 2: upload loop (every 30 min, never downloads)
+  if (HF_TOKEN && DATASET_ID) {
+    log("Upload loop starting (first run in 10 min)...");
+    (async function loop() {
+      await sleep(10 * 60 * 1000);
+      while (true) {
+        runUploadCycle();
+        await sleep(UPLOAD_INTERVAL);
+      }
+    })().catch(function(e) { log("Upload loop crashed: " + e.message); });
+  } else {
+    log("Upload disabled: set HF_TOKEN and HF_DATASET_ID in Space Secrets");
+  }
+
+  log("Ready.");
+}
+
+main().catch(function(e) {
+  console.error("[hf-sync] Fatal: " + e.message);
+});