Update hf-sync-manager.mjs

hf-sync-manager.mjs

@@ -1,277 +1,191 @@
 // ============================================================
-// HF Memory Sync Manager - Production Grade
+// HF Memory Sync Manager - Definitive Edition
 //
-// TRACK 1 - Settings (openclaw.json):
-//   Boot:    repo/settings/openclaw.json -> local disk
-//            ONLY if repo file passes JSON validation
-//            If invalid: keep local file, log warning
-//   Upload:  local disk -> repo/settings/openclaw.json
-//            ONLY if local file passes JSON validation
-//            Single copy, always overwrite, never accumulate
+// Based on actual OpenClaw disk layout (from official docs):
 //
-// TRACK 2 - Memory (conversations, history, context, agents, logs):
-//   Boot:    repo/data/* -> local disk (full restore, all sessions)
-//   Upload:  local disk -> repo/data/* every 3h
-//            Upload only, NEVER download during runtime
+//   ~/.openclaw/openclaw.json          <- gateway config
+//   ~/.openclaw/workspace/             <- agent brain (THIS is memory)
+//     AGENTS.md                        <- operating instructions
+//     SOUL.md                          <- persona / tone
+//     USER.md                          <- user profile
+//     IDENTITY.md                       <- name / emoji
+//     TOOLS.md                         <- env conventions
+//     HEARTBEAT.md                     <- periodic tasks
+//     MEMORY.md                        <- long-term memory (iron rules)
+//     memory/YYYY-MM-DD.md             <- daily logs
+//     memory/archive/                  <- logs older than 30 days
+//     checklists/                      <- op checklists
+//     docs/                            <- on-demand reference docs
+//   ~/.openclaw/agents/<id>/sessions/  <- DO NOT sync (session transcripts)
+//   ~/.openclaw/agents/<id>/qmd/       <- DO NOT sync (sqlite index, rebuilt on boot)
+//
+// Repo structure:
+//   settings/openclaw.json             <- single copy, validated, overwrite
+//   workspace/                         <- full workspace mirror
+//     AGENTS.md / SOUL.md / MEMORY.md / memory/ …
+//   README.md                          <- auto-generated index
+//
+// Rules:
+//   TRACK 1 - settings:
+//     Boot:    repo -> local (skip if repo JSON invalid)
+//     Upload:  local -> repo (skip if local JSON invalid)
+//     One file, one copy, always overwrite
+//
+//   TRACK 2 - workspace:
+//     Boot:    repo/workspace/ -> ~/.openclaw/workspace/ (full restore)
+//     Upload:  ~/.openclaw/workspace/ -> repo/workspace/ every 3h
+//     Never downloads during runtime (prevents memory flooding)
+//
+// Multi-bot:
+//   Set OPENCLAW_PROFILE env var in HF Space Variables.
+//   OpenClaw uses ~/.openclaw/workspace-<profile>/ per profile.
+//   Repo stores each profile under workspace-<profile>/.
 // ============================================================
 
 import fs   from "node:fs";
 import path from "node:path";
 import { execSync } from "node:child_process";
 
-// -- config ---------------------------------------------------
-
-var HOME            = process.env.OPENCLAW_HOME || process.env.HOME || "/home/user";
-var OPENCLAW_DIR    = path.join(HOME, ".openclaw");
-var HF_TOKEN        = (process.env.HF_TOKEN      || "").trim();
-var DATASET_ID      = (process.env.HF_DATASET_ID  || "").trim();
-var REPO_DIR        = "/tmp/hf-memory-sync";
-var UPLOAD_INTERVAL = 3 * 60 * 60 * 1000;
-
-var SETTINGS_FILE = "openclaw.json";
-var REPO_SETTINGS = "settings";
-
-var MEMORY_TARGETS = [
-  "memory",
-  "history",
-  "logs",
-  "conversations",
-  "context.json",
-  "agents"
+// -- config ----------------------------------
+
+var HOME         = process.env.OPENCLAW_HOME || process.env.HOME || "/home/user";
+var OPENCLAW_DIR = path.join(HOME, ".openclaw");
+var HF_TOKEN     = (process.env.HF_TOKEN      || "").trim();
+var DATASET_ID   = (process.env.HF_DATASET_ID  || "").trim();
+var REPO_DIR     = "/tmp/hf-memory-sync";
+
+// OPENCLAW_PROFILE: if set, workspace is ~/.openclaw/workspace-<profile>/
+// This is OpenClaw's native multi-bot mechanism
+var PROFILE      = (process.env.OPENCLAW_PROFILE || "").trim();
+var WORKSPACE_DIR = PROFILE
+? path.join(OPENCLAW_DIR, "workspace-" + PROFILE)
+: path.join(OPENCLAW_DIR, "workspace");
+
+// Repo folder name for this profile's workspace
+var REPO_WORKSPACE_FOLDER = PROFILE ? "workspace-" + PROFILE : "workspace";
+
+var UPLOAD_INTERVAL = 3 * 60 * 60 * 1000; // 3 hours
+
+// -- workspace files OpenClaw actually uses -----------------
+var WORKSPACE_SYNC = [
+"AGENTS.md",
+"SOUL.md",
+"USER.md",
+"IDENTITY.md",
+"TOOLS.md",
+"HEARTBEAT.md",
+"BOOT.md",
+"MEMORY.md",
+"memory",       // directory: memory/YYYY-MM-DD.md + memory/archive/
+"checklists",   // directory: op checklists
+"docs"          // directory: on-demand reference docs
 ];
 
+// Never touch these (sessions and search index are runtime-only)
 var NEVER_SYNC = [
-  "openclaw.json",
-  "openclaw.json.bak",
-  ".sync-state.json",
-  "settings.json",
-  "config.yaml",
-  "config.json",
-  ".env",
-  "node_modules",
-  ".git",
-  "canvas"
+"agents",       // ~/.openclaw/agents/<id>/sessions/ and qmd/ - runtime only
+"canvas",       // canvas UI files - runtime only
+".git",
+"node_modules"
 ];
 
-// -- helpers --------------------------------------------------
+// -- helpers ----------------------------------
 
 function log(msg) {
-  console.log("[hf-sync] " + msg);
+console.log("[hf-sync] " + msg);
 }
 
 function sleep(ms) {
-  return new Promise(function(resolve) { setTimeout(resolve, ms); });
+return new Promise(function(resolve) { setTimeout(resolve, ms); });
 }
 
 function repoUrl() {
-  return "https://user:" + HF_TOKEN +
-    "@huggingface.co/datasets/" + DATASET_ID;
+return "https://user:" + HF_TOKEN +
+"@huggingface.co/datasets/" + DATASET_ID;
 }
 
 function git(args) {
-  return execSync("git " + args, {
-    cwd:     REPO_DIR,
-    stdio:   "pipe",
-    timeout: 120000,
-    env:     Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
-  }).toString().trim();
+return execSync("git " + args, {
+cwd:     REPO_DIR,
+stdio:   "pipe",
+timeout: 120000,
+env:     Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
+}).toString().trim();
 }
 
-function isMemoryTarget(name) {
-  var i;
-  for (i = 0; i < NEVER_SYNC.length; i++) {
-    if (name === NEVER_SYNC[i]) return false;
-  }
-  for (i = 0; i < MEMORY_TARGETS.length; i++) {
-    if (name === MEMORY_TARGETS[i]) return true;
-  }
-  return false;
-}
-
-// FIX: validate JSON before any config file operation
-// Returns true if file exists and contains valid JSON
 function isValidJson(filePath) {
-  try {
-    if (!fs.existsSync(filePath)) return false;
-    var content = fs.readFileSync(filePath, "utf-8").trim();
-    if (!content) return false;
-    JSON.parse(content);
-    return true;
-  } catch (e) {
-    return false;
-  }
+try {
+if (!fs.existsSync(filePath)) return false;
+var content = fs.readFileSync(filePath, "utf-8").trim();
+if (!content) return false;
+JSON.parse(content);
+return true;
+} catch (e) {
+return false;
+}
 }
 
 function copyItem(src, dst) {
-  if (fs.statSync(src).isDirectory()) {
-    execSync("cp -r " + src + " " + dst, { stdio: "pipe" });
-  } else {
-    fs.mkdirSync(path.dirname(dst), { recursive: true });
-    fs.copyFileSync(src, dst);
-  }
+if (fs.statSync(src).isDirectory()) {
+execSync("cp -a " + src + " " + dst, { stdio: "pipe" });
+} else {
+fs.mkdirSync(path.dirname(dst), { recursive: true });
+fs.copyFileSync(src, dst);
+}
 }
 
-// -- git repo setup ------------------------------------------
+// -- git repo setup --------------------------
 
 function ensureRepo() {
-  if (fs.existsSync(path.join(REPO_DIR, ".git"))) return;
-  execSync("rm -rf " + REPO_DIR, { stdio: "pipe" });
-  try {
-    execSync(
-      "git clone --depth 1 " + repoUrl() + " " + REPO_DIR,
-      {
-        stdio:   "pipe",
-        timeout: 120000,
-        env:     Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
-      }
-    );
-  } catch (e) {
-    fs.mkdirSync(REPO_DIR, { recursive: true });
-    execSync("git init " + REPO_DIR, { stdio: "pipe" });
-    git("remote add origin " + repoUrl());
-  }
-  try {
-    git("config user.email openclaw@hf.space");
-    git("config user.name OpenClaw-Memory-Sync");
-    git("config pull.rebase false");
-  } catch (e) { /* non-fatal */ }
+if (fs.existsSync(path.join(REPO_DIR, ".git"))) return;
+execSync("rm -rf " + REPO_DIR, { stdio: "pipe" });
+try {
+execSync(
+"git clone --depth 1 " + repoUrl() + " " + REPO_DIR,
+{
+stdio:   "pipe",
+timeout: 120000,
+env:     Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
 }
-
-function pullLatest() {
-  try { git("fetch --quiet origin"); } catch (e) { /* ignore */ }
-  try {
-    git("pull --quiet --no-rebase origin main");
-  } catch (e) {
-    try { git("pull --quiet --no-rebase origin master"); } catch (e2) { /* no remote yet */ }
-  }
+);
+} catch (e) {
+fs.mkdirSync(REPO_DIR, { recursive: true });
+execSync("git init " + REPO_DIR, { stdio: "pipe" });
+git("remote add origin " + repoUrl());
 }
-
-function commitAndPush(label) {
-  try {
-    git("add -A");
-    var changed = git("diff --cached --name-only");
-    if (!changed) return 0;
-    var count = changed.split("\n").filter(Boolean).length;
-    var ts    = new Date().toISOString().replace("T", " ").substring(0, 19);
-    git("commit -m \"" + label + " " + ts + "\"");
-    try {
-      git("push --quiet origin HEAD:main");
-    } catch (e) {
-      git("push --quiet origin HEAD:master");
-    }
-    return count;
-  } catch (e) {
-    log("Commit/push failed: " + e.message);
-    return 0;
-  }
+try {
+git("config user.email openclaw@hf.space");
+git("config user.name OpenClaw-Sync");
+git("config pull.rebase false");
+} catch (e) { /* non-fatal */ }
 }
 
-// ============================================================
-// TRACK 1: Settings
-// ============================================================
-
-function settingsBootRestore() {
-  var repoFile  = path.join(REPO_DIR, REPO_SETTINGS, SETTINGS_FILE);
-  var localFile = path.join(OPENCLAW_DIR, SETTINGS_FILE);
-
-  if (!fs.existsSync(repoFile)) {
-    log("[settings] No settings in repo yet, keeping local defaults");
-    return false;
-  }
-
-  // FIX: validate repo file before overwriting local config
-  if (!isValidJson(repoFile)) {
-    log("[settings] WARN: repo openclaw.json is invalid JSON, skipping restore to protect local config");
-    log("[settings] Corrupted repo file will be overwritten on next upload");
-    return false;
-  }
-
-  try {
-    fs.mkdirSync(OPENCLAW_DIR, { recursive: true });
-    // Backup local before overwriting
-    if (fs.existsSync(localFile)) {
-      fs.copyFileSync(localFile, localFile + ".pre-restore-bak");
-    }
-    fs.copyFileSync(repoFile, localFile);
-    log("[settings] Boot restore done: openclaw.json <- repo (validated OK)");
-    return true;
-  } catch (e) {
-    log("[settings] Boot restore failed: " + e.message);
-    return false;
-  }
+function pullLatest() {
+try { git("fetch --quiet origin"); } catch (e) { /* ignore */ }
+try {
+git("pull --quiet --no-rebase origin main");
+} catch (e) {
+try { git("pull --quiet --no-rebase origin master"); } catch (e2) { /* no remote yet */ }
 }
-
-function settingsUpload() {
-  var localFile = path.join(OPENCLAW_DIR, SETTINGS_FILE);
-
-  // FIX: validate local file before uploading to repo
-  if (!isValidJson(localFile)) {
-    log("[settings] WARN: local openclaw.json is invalid JSON, skipping upload");
-    return 0;
-  }
-
-  var repoDir  = path.join(REPO_DIR, REPO_SETTINGS);
-  var repoFile = path.join(repoDir, SETTINGS_FILE);
-
-  fs.mkdirSync(repoDir, { recursive: true });
-  // Single copy, direct overwrite - no version accumulation
-  fs.copyFileSync(localFile, repoFile);
-  log("[settings] Uploaded openclaw.json -> repo (validated OK)");
-  return 1;
 }
 
-// ============================================================
-// TRACK 2: Memory
-// ============================================================
-
-function memoryBootRestore() {
-  var src = path.join(REPO_DIR, "data");
-  if (!fs.existsSync(src)) return 0;
-
-  var items  = fs.readdirSync(src);
-  var copied = 0;
-
-  items.forEach(function(name) {
-    var s = path.join(src, name);
-    var d = path.join(OPENCLAW_DIR, name);
-    try {
-      fs.mkdirSync(OPENCLAW_DIR, { recursive: true });
-      copyItem(s, d);
-      copied++;
-    } catch (e) {
-      log("[memory] Restore error [" + name + "]: " + e.message);
-    }
-  });
-
-  return copied;
+function commitAndPush(label) {
+try {
+git("add -A");
+var changed = git("diff --cached --name-only");
+if (!changed) return 0;
+var count = changed.split("\n").filter(Boolean).length;
+var ts    = new Date().toISOString().replace("T", " ").substring(0, 19);
+var tag   = PROFILE ? "[" + PROFILE + "]" : "";
+git("commit -m \"sync" + tag + " " + ts + "\"");
+try {
+git("push --quiet origin HEAD:main");
+} catch (e) {
+git("push --quiet origin HEAD:master");
 }
-
-function memoryUpload() {
-  var dest = path.join(REPO_DIR, "data");
-  execSync("rm -rf " + dest, { stdio: "pipe" });
-  fs.mkdirSync(dest, { recursive: true });
-
-  if (!fs.existsSync(OPENCLAW_DIR)) return 0;
-
-  var items  = fs.readdirSync(OPENCLAW_DIR);
-  var copied = 0;
-
-  items.forEach(function(name) {
-    if (!isMemoryTarget(name)) return;
-    var src = path.join(OPENCLAW_DIR, name);
-    var dst = path.join(dest, name);
-    try {
-      copyItem(src, dst);
-      copied++;
-    } catch (e) {
-      log("[memory] Copy error [" + name + "]: " + e.message);
-    }
-  });
-
-  return copied;
+return count;
+} catch (e) {
+log("Commit/push failed: " + e.message);
+return 0;
 }
-
-function scanMemoryDisk() {
-  if (!fs.existsSync(OPENCLAW_DIR)) return [];
-  return fs.readdirSync(OPENCLAW_DIR).filter(isMemoryTarget);
 }