Update hf-sync-manager.mjs

hf-sync-manager.mjs

@@ -1,101 +1,380 @@
-import fs from "node:fs";
+// ============================================================
+// HF Sync Manager - Full Mirror Edition
+//
+// Mirrors ~/.openclaw/ to HF Dataset every 30 min.
+//
+// CRITICAL RULES:
+// 1. NEVER restore openclaw.json from Dataset.
+//    openclaw.json is owned by setup-hf-config.mjs only.
+//    Restoring it causes an infinite restart loop:
+//    user saves -> gateway restarts -> we overwrite their
+//    settings with old Dataset copy -> OpenClaw restarts again.
+//
+// 2. RUNNING_FLAG is never deleted on process exit.
+//    It lives in /tmp which is only cleared on true container
+//    restart. Gateway restarts do NOT clear /tmp.
+//    This correctly distinguishes the two restart types.
+//
+// Excluded from sync:
+//   openclaw.json          (owned by setup, never touch)
+//   openclaw.json.bak      (backup files)
+//   agents/*/sessions/     (session transcripts, runtime only)
+//   agents/*/qmd/          (vector index, rebuilt on boot)
+//   canvas/                (UI runtime files)
+//   *.bak *.tmp *.log
+// ============================================================
+
+import fs   from "node:fs";
 import path from "node:path";
 import { execSync } from "node:child_process";
 
-var HOME = process.env.OPENCLAW_HOME || process.env.HOME || "/home/user";
-var OPENCLAW_DIR = path.join(HOME,".openclaw");
-var MEMORY_DIR = path.join(OPENCLAW_DIR,"dataset-memory");
-var HF_TOKEN = (process.env.HF_TOKEN||"").trim();
-var DATASET_ID = (process.env.HF_DATASET_ID||"").trim();
-var REPO_DIR = "/tmp/hf-memory-sync";
-var UPLOAD_INTERVAL = 3*60*60*1000;
+// –– config ————————————————
+
+var HOME         = process.env.OPENCLAW_HOME || process.env.HOME || "/home/user";
+var OPENCLAW_DIR = path.join(HOME, ".openclaw");
+var WORKSPACE    = path.join(OPENCLAW_DIR, "workspace");
+var HF_TOKEN     = (process.env.HF_TOKEN      || "").trim();
+var DATASET_ID   = (process.env.HF_DATASET_ID  || "").trim();
+var REPO_DIR     = "/tmp/oc-dataset";
+var INTERVAL     = 30 * 60 * 1000; // 30 minutes
+
+// This flag file distinguishes container restart from gateway restart.
+// Container restart: /tmp is wiped, flag is gone -> do full restore.
+// Gateway restart:   /tmp survives, flag exists -> skip restore.
+// IMPORTANT: we never delete this flag on process exit.
+var RUNNING_FLAG = "/tmp/.hf-sync-running";
+
+// Files/dirs to EXCLUDE from sync in any direction
+var EXCLUDE_NAMES = [
+"openclaw.json",     // NEVER touch - owned by setup-hf-config.mjs
+"openclaw.json.bak", // backup files
+"sessions",          // session transcripts inside agents/*/sessions/
+"qmd",              // sqlite vector index inside agents/*/qmd/
+"canvas",            // UI runtime files
+".git"
+];
+var EXCLUDE_EXT = [".bak", ".tmp", ".log"];
 
-var TRACK_NAMES = ["conversations","memory","logs"];
-var botMemory = { conversations:[], memory:[], logs:[] };
+// –– helpers ———————————————–
 
-function log(msg){ console.log("[hf-sync] "+msg); }
-function repoUrl(){ return "https://user:"+HF_TOKEN+"@huggingface.co/datasets/"+DATASET_ID; }
-function git(args,cwd){ return execSync("git "+args,{ cwd:cwd||REPO_DIR, stdio:"pipe", timeout:120000, env:Object.assign({},process.env,{GIT_TERMINAL_PROMPT:"0"}) }).toString().trim(); }
-function sleep(ms){ return new Promise(resolve=>setTimeout(resolve,ms)); }
+function log(msg) { console.log("[hf-sync] " + msg); }
 
-function ensureRepo(){
-  if(fs.existsSync(path.join(REPO_DIR,".git"))) return;
-  execSync("rm -rf "+REPO_DIR,{stdio:"pipe"});
-  try{ git("clone --depth 1 "+repoUrl()+" "+REPO_DIR,"/tmp"); } catch(e){ git("init",REPO_DIR); git("remote add origin "+repoUrl(),REPO_DIR); }
-  try{ git("config user.email openclaw@hf.space"); git("config user.name OpenClaw-Memory-Sync"); git("config pull.rebase false"); } catch(e){ }
+function sleep(ms) {
+return new Promise(function(resolve) { setTimeout(resolve, ms); });
 }
 
-function pullLatest(){
-  try{ git("fetch --quiet origin"); git("pull --quiet --no-rebase origin main"); }
-  catch(e){ try{ git("pull --quiet --no-rebase origin master"); } catch(e2){ } }
+function repoUrl() {
+return "https://user:" + HF_TOKEN +
+"@huggingface.co/datasets/" + DATASET_ID;
 }
 
-function commitAndPush(){
-  try{
-    git("add -A");
-    var changed = git("diff --cached --name-only");
-    if(!changed) return 0;
-    var ts = new Date().toISOString().replace("T"," ").substring(0,19);
-    var count = changed.split("\n").filter(Boolean).length;
-    git('commit -m "memory-sync '+ts+'"');
-    try{ git("push --quiet origin HEAD:main"); } catch(e){ git("push --quiet origin HEAD:master"); }
-    return count;
-  }catch(e){ log("Push failed: "+e.message); return 0; }
+function git(args) {
+return execSync("git " + args, {
+cwd:     REPO_DIR,
+stdio:   "pipe",
+timeout: 120000,
+env:     Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
+}).toString().trim();
 }
 
-function readJsonl(filePath,targetArray){ if(!fs.existsSync(filePath)) return; fs.readFileSync(filePath,"utf-8").split("\n").filter(Boolean).forEach(line=>{ try{ targetArray.push(JSON.parse(line)) }catch(e){} }); }
-function writeJsonl(filePath,dataArray){ fs.writeFileSync(filePath,dataArray.map(x=>JSON.stringify(x)).join("\n"),"utf-8"); }
+function shouldExclude(name) {
+var i;
+for (i = 0; i < EXCLUDE_NAMES.length; i++) {
+if (name === EXCLUDE_NAMES[i]) return true;
+}
+for (i = 0; i < EXCLUDE_EXT.length; i++) {
+if (name.indexOf(EXCLUDE_EXT[i]) >= 0) return true;
+}
+return false;
+}
 
-function todayDir(){ var d=new Date(); return d.getFullYear()+"-"+String(d.getMonth()+1).padStart(2,"0")+"-"+String(d.getDate()).padStart(2,"0"); }
-function getRepoDateDir(dateStr){ var dir=path.join(REPO_DIR,"dataset-memory",dateStr); fs.mkdirSync(dir,{recursive:true}); return dir; }
+// –– git repo setup ––––––––––––––––––––
 
-function loadMemoryFromRepo(){
-  var repoBase = path.join(REPO_DIR,"dataset-memory");
-  if(!fs.existsSync(repoBase)) return;
-  var dateDirs = fs.readdirSync(repoBase).filter(f=>fs.statSync(path.join(repoBase,f)).isDirectory());
-  dateDirs.forEach(dateDir=>{ TRACK_NAMES.forEach(track=>{ readJsonl(path.join(repoBase,dateDir,track+".jsonl"),botMemory[track]); }); });
-  log("Loaded: conversations="+botMemory.conversations.length+" memory="+botMemory.memory.length+" logs="+botMemory.logs.length);
+function ensureRepo() {
+if (fs.existsSync(path.join(REPO_DIR, ".git"))) return;
+execSync("rm -rf " + REPO_DIR, { stdio: "pipe" });
+try {
+execSync("git clone --depth 1 " + repoUrl() + " " + REPO_DIR, {
+stdio: "pipe", timeout: 120000,
+env: Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
+});
+log("Repo cloned");
+} catch (e) {
+log("Clone failed, init empty: " + e.message);
+fs.mkdirSync(REPO_DIR, { recursive: true });
+execSync("git init " + REPO_DIR, { stdio: "pipe" });
+git("remote add origin " + repoUrl());
+}
+try {
+git("config user.email openclaw@hf.space");
+git("config user.name OpenClaw-Sync");
+git("config pull.rebase false");
+} catch (e) { /* non-fatal */ }
 }
 
-function saveMemoryToRepo(){
-  var dateStr=todayDir(); var destDir=getRepoDateDir(dateStr); TRACK_NAMES.forEach(track=>writeJsonl(path.join(destDir,track+".jsonl"),botMemory[track]));
+function pull() {
+try { git("fetch --quiet origin"); } catch (e) { /* ignore */ }
+try { git("pull --quiet --no-rebase origin main"); }
+catch (e) {
+try { git("pull --quiet --no-rebase origin master"); }
+catch (e2) { /* empty repo */ }
+}
 }
 
-function bootRestore(){
-  if(!HF_TOKEN || !DATASET_ID){ log("HF_TOKEN or HF_DATASET_ID not set, skipping restore"); return; }
-  log("Boot restore starting from "+DATASET_ID+"...");
-  try{ ensureRepo(); pullLatest(); loadMemoryFromRepo(); log("Boot restore done."); }catch(e){ log("Boot restore failed (non-fatal): "+e.message); }
+function push() {
+try {
+git("add -A");
+var changed = git("diff --cached --name-only");
+if (!changed) return 0;
+var n  = changed.split("\n").filter(Boolean).length;
+var ts = new Date().toISOString().replace("T", " ").substring(0, 19);
+git('commit -m "sync ' + ts + '"');
+try { git("push --quiet origin HEAD:main"); }
+catch (e) { git("push --quiet origin HEAD:master"); }
+return n;
+} catch (e) {
+log("Push failed: " + e.message);
+return 0;
+}
 }
 
-async function startUploadLoop(){
-  while(true){
-    await sleep(UPLOAD_INTERVAL);
-    try{ log("Uploading memory to repo..."); ensureRepo(); pullLatest(); saveMemoryToRepo(); var count=commitAndPush(); log("Upload done: "+count+" file(s) changed"); }catch(e){ log("Upload error: "+e.message); }
-  }
+// –– directory mirror: local -> repo ———————–
+
+function mirrorToRepo(srcDir, dstDir) {
+if (!fs.existsSync(srcDir)) return 0;
+fs.mkdirSync(dstDir, { recursive: true });
+var count = 0;
+fs.readdirSync(srcDir).forEach(function(name) {
+if (shouldExclude(name)) return;
+var src = path.join(srcDir, name);
+var dst = path.join(dstDir, name);
+try {
+if (fs.statSync(src).isDirectory()) {
+count += mirrorToRepo(src, dst);
+} else {
+fs.copyFileSync(src, dst);
+count++;
+}
+} catch (e) {
+log("Copy error [" + name + "]: " + e.message);
+}
+});
+return count;
 }
 
-async function main(){
-  log("Starting...");
-  log("OPENCLAW_DIR : "+OPENCLAW_DIR);
-  log("HF_DATASET_ID: "+(DATASET_ID||"NOT SET"));
-  log("Upload interval: "+(UPLOAD_INTERVAL/3600000)+"h");
+// –– directory mirror: repo -> local ———————–
+// Note: shouldExclude ensures openclaw.json is never restored
 
-  fs.mkdirSync(MEMORY_DIR,{recursive:true});
-  try{
-    execSync("git config --global user.email openclaw@hf.space",{stdio:"pipe"});
-    execSync("git config --global user.name OpenClaw-Memory-Sync",{stdio:"pipe"});
-    execSync("git config --global http.postBuffer 52428800",{stdio:"pipe"});
-    execSync("git config --global pull.rebase false",{stdio:"pipe"});
-  }catch(e){}
+function mirrorFromRepo(srcDir, dstDir) {
+if (!fs.existsSync(srcDir)) return 0;
+fs.mkdirSync(dstDir, { recursive: true });
+var count = 0;
+fs.readdirSync(srcDir).forEach(function(name) {
+if (shouldExclude(name)) return; // never restore openclaw.json
+var src = path.join(srcDir, name);
+var dst = path.join(dstDir, name);
+try {
+if (fs.statSync(src).isDirectory()) {
+count += mirrorFromRepo(src, dst);
+} else {
+fs.copyFileSync(src, dst);
+count++;
+}
+} catch (e) {
+log("Restore error [" + name + "]: " + e.message);
+}
+});
+return count;
+}
+
+// –– seed default workspace files –––––––––––––
+
+function seedWorkspaceIfNeeded() {
+fs.mkdirSync(WORKSPACE, { recursive: true });
+fs.mkdirSync(path.join(WORKSPACE, "memory"), { recursive: true });
+
+var soul = path.join(WORKSPACE, "SOUL.md");
+if (!fs.existsSync(soul)) {
+fs.writeFileSync(soul, [
+"# Soul",
+"",
+"You are a helpful, warm, concise AI assistant.",
+"",
+"## Language",
+"",
+"Default language: Simplified Chinese.",
+"Always reply in Chinese unless the user writes in another language first.",
+"",
+"## Tone",
+"",
+"- Natural and friendly, not overly formal",
+"- Concise and to the point",
+"- Ask one clarifying question at a time when needed"
+].join("\n") + "\n", "utf-8");
+log("Seeded workspace/SOUL.md");
+}
+
+var agents = path.join(WORKSPACE, "AGENTS.md");
+if (!fs.existsSync(agents)) {
+fs.writeFileSync(agents, [
+"# Agent Instructions",
+"",
+"## Boot sequence",
+"",
+"1. Read SOUL.md - language and persona",
+"2. Read USER.md if present - user profile",
+"3. Read MEMORY.md - long-term facts and rules",
+"4. Read today and yesterday memory/YYYY-MM-DD.md if present",
+"",
+"## Memory rules",
+"",
+"- Write important facts to MEMORY.md when asked",
+"- Log daily context to memory/YYYY-MM-DD.md",
+"- Do not ask for information already provided"
+].join("\n") + "\n", "utf-8");
+log("Seeded workspace/AGENTS.md");
+}
+
+var mem = path.join(WORKSPACE, "MEMORY.md");
+if (!fs.existsSync(mem)) {
+fs.writeFileSync(mem, [
+"# Long-term Memory",
+"",
+"<!-- OpenClaw writes important facts here. -->",
+"<!-- Loaded at the start of every session. -->"
+].join("\n") + "\n", "utf-8");
+log("Seeded workspace/MEMORY.md");
+}
+}
 
-  bootRestore();
+// –– boot restore (container restart only) —————–
 
-  if(HF_TOKEN && DATASET_ID){
-    log("Upload loop started (first run in "+(UPLOAD_INTERVAL/3600000)+"h)");
-    startUploadLoop().catch(e=>{ log("Upload loop error: "+e.message); });
+function bootRestore() {
+if (!HF_TOKEN || !DATASET_ID) {
+log("No HF_TOKEN/HF_DATASET_ID - skipping Dataset restore");
+seedWorkspaceIfNeeded();
+return;
+}
+
+log("Boot restore from " + DATASET_ID + "…");
+try {
+ensureRepo();
+pull();
+} catch (e) {
+log("Repo init failed (" + e.message + ") - starting fresh");
+seedWorkspaceIfNeeded();
+return;
+}
+
+var repoOC = path.join(REPO_DIR, "openclaw");
+if (fs.existsSync(repoOC)) {
+// mirrorFromRepo skips openclaw.json automatically via shouldExclude
+var n = mirrorFromRepo(repoOC, OPENCLAW_DIR);
+log("Boot restore done: " + n + " file(s) restored (openclaw.json excluded)");
+} else {
+log("No data in Dataset yet - starting fresh");
+}
+
+seedWorkspaceIfNeeded();
+log("Boot restore complete");
+}
+
+// –– sync cycle (every 30 min) —————————–
+
+function runSync() {
+if (!HF_TOKEN || !DATASET_ID) return;
+
+var ts = new Date().toISOString().replace("T", " ").substring(0, 19);
+log("Sync at " + ts + "…");
+
+try {
+ensureRepo();
+pull();
+
+var repoOC = path.join(REPO_DIR, "openclaw");
+if (fs.existsSync(repoOC)) {
+  execSync("rm -rf " + repoOC, { stdio: "pipe" });
+}
+fs.mkdirSync(repoOC, { recursive: true });
+
+// mirrorToRepo also skips openclaw.json via shouldExclude
+// We explicitly copy openclaw.json separately only for backup
+var n = mirrorToRepo(OPENCLAW_DIR, repoOC);
+
+// Upload openclaw.json separately without excluding it from upload
+// (we exclude it from RESTORE, not from upload)
+var localCfg = path.join(OPENCLAW_DIR, "openclaw.json");
+if (fs.existsSync(localCfg)) {
+  try {
+    var content = fs.readFileSync(localCfg, "utf-8").trim();
+    JSON.parse(content); // validate
+    fs.copyFileSync(localCfg, path.join(repoOC, "openclaw.json"));
+    n++;
+    log("openclaw.json uploaded (validated)");
+  } catch (e) {
+    log("openclaw.json skipped (invalid JSON): " + e.message);
   }
+}
+
+log("Mirrored " + n + " file(s) to Dataset");
 
-  log("Ready.");
+var pushed = push();
+if (pushed > 0) {
+  log("Sync done: " + pushed + " file(s) pushed");
+} else {
+  log("Sync done: no changes");
 }
+} catch (e) {
+log("Sync failed (retry in 30 min): " + e.message);
+}
+}
+
+// –– main –––––––––––––––––––––––––
+
+(async function() {
+log("Starting…");
+log("OPENCLAW_DIR : " + OPENCLAW_DIR);
+log("DATASET      : " + (DATASET_ID || "NOT SET"));
+log("RUNNING_FLAG : " + RUNNING_FLAG);
+
+fs.mkdirSync(OPENCLAW_DIR, { recursive: true });
+fs.mkdirSync(WORKSPACE,    { recursive: true });
+
+try {
+execSync("git config --global user.email openclaw@hf.space", { stdio: "pipe" });
+execSync("git config --global user.name OpenClaw-Sync",      { stdio: "pipe" });
+execSync("git config --global http.postBuffer 52428800",     { stdio: "pipe" });
+execSync("git config --global pull.rebase false",            { stdio: "pipe" });
+} catch (e) { /* non-fatal */ }
+
+// Detect restart type:
+// /tmp survives gateway restart but is wiped on container restart.
+// We NEVER delete this flag - only container restart clears it.
+var isGatewayRestart = fs.existsSync(RUNNING_FLAG);
 
-main().catch(e=>{ console.error("[hf-sync] Fatal: "+e.message); });
+if (isGatewayRestart) {
+log("Gateway restart detected - skipping boot restore");
+log("(openclaw.json and workspace already on disk, nothing to restore)");
+} else {
+log("Container start detected - running full boot restore");
+bootRestore();
+}
+
+// Write flag. Never delete it (not on SIGTERM, not on exit).
+// Container restart is the only thing that clears /tmp.
+fs.writeFileSync(RUNNING_FLAG, String(process.pid), "utf-8");
+log("Running flag set: " + RUNNING_FLAG);
+
+if (!HF_TOKEN || !DATASET_ID) {
+log("Sync disabled: set HF_TOKEN and HF_DATASET_ID in Space Secrets");
+return;
+}
+
+log("Sync loop starting (first run in 10 min)…");
+await sleep(10 * 60 * 1000);
+
+while (true) {
+runSync();
+await sleep(INTERVAL);
+}
+})().catch(function(e) { console.error("[hf-sync] Fatal: " + e.message); });