Update hf-sync-manager.mjs

hf-sync-manager.mjs

@@ -1,324 +1,101 @@
-// ============================================================
-// HF Sync Manager - Full Mirror Edition
-//
-// Mirrors ~/.openclaw/ to HF Dataset every 30 min.
-// Boot restore: Dataset -> local (container restart only).
-// Gateway restart detection via /tmp flag file.
-//
-// Excluded: agents/*/sessions, agents/*/qmd, canvas, .bak, .tmp
-// ============================================================
-
-import fs   from "node:fs";
+import fs from "node:fs";
 import path from "node:path";
 import { execSync } from "node:child_process";
 
-var HOME         = process.env.OPENCLAW_HOME || process.env.HOME || "/home/user";
-var OPENCLAW_DIR = path.join(HOME, ".openclaw");
-var WORKSPACE    = path.join(OPENCLAW_DIR, "workspace");
-var HF_TOKEN     = (process.env.HF_TOKEN      || "").trim();
-var DATASET_ID   = (process.env.HF_DATASET_ID  || "").trim();
-var REPO_DIR     = "/tmp/oc-dataset";
-var INTERVAL     = 30 * 60 * 1000;
-var RUNNING_FLAG = "/tmp/.hf-sync-running";
-
-var EXCLUDE     = ["sessions", "qmd", "canvas", ".git"];
-var EXCLUDE_EXT = [".bak", ".tmp", ".log"];
-
-// –––––––––––––––––––––––––––––
-
-function log(msg) { console.log("[hf-sync] " + msg); }
-
-function sleep(ms) {
-  return new Promise(function(resolve) { setTimeout(resolve, ms); });
-}
-
-function repoUrl() {
-  return "https://user:" + HF_TOKEN +
-    "@huggingface.co/datasets/" + DATASET_ID;
+var HOME = process.env.OPENCLAW_HOME || process.env.HOME || "/home/user";
+var OPENCLAW_DIR = path.join(HOME,".openclaw");
+var MEMORY_DIR = path.join(OPENCLAW_DIR,"dataset-memory");
+var HF_TOKEN = (process.env.HF_TOKEN||"").trim();
+var DATASET_ID = (process.env.HF_DATASET_ID||"").trim();
+var REPO_DIR = "/tmp/hf-memory-sync";
+var UPLOAD_INTERVAL = 3*60*60*1000;
+
+var TRACK_NAMES = ["conversations","memory","logs"];
+var botMemory = { conversations:[], memory:[], logs:[] };
+
+function log(msg){ console.log("[hf-sync] "+msg); }
+function repoUrl(){ return "https://user:"+HF_TOKEN+"@huggingface.co/datasets/"+DATASET_ID; }
+function git(args,cwd){ return execSync("git "+args,{ cwd:cwd||REPO_DIR, stdio:"pipe", timeout:120000, env:Object.assign({},process.env,{GIT_TERMINAL_PROMPT:"0"}) }).toString().trim(); }
+function sleep(ms){ return new Promise(resolve=>setTimeout(resolve,ms)); }
+
+function ensureRepo(){
+  if(fs.existsSync(path.join(REPO_DIR,".git"))) return;
+  execSync("rm -rf "+REPO_DIR,{stdio:"pipe"});
+  try{ git("clone --depth 1 "+repoUrl()+" "+REPO_DIR,"/tmp"); } catch(e){ git("init",REPO_DIR); git("remote add origin "+repoUrl(),REPO_DIR); }
+  try{ git("config user.email openclaw@hf.space"); git("config user.name OpenClaw-Memory-Sync"); git("config pull.rebase false"); } catch(e){ }
 }
 
-function git(args) {
-  return execSync("git " + args, {
-    cwd:     REPO_DIR,
-    stdio:   "pipe",
-    timeout: 120000,
-    env:     Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
-  }).toString().trim();
+function pullLatest(){
+  try{ git("fetch --quiet origin"); git("pull --quiet --no-rebase origin main"); }
+  catch(e){ try{ git("pull --quiet --no-rebase origin master"); } catch(e2){ } }
 }
 
-function shouldExclude(name) {
-  var i;
-  for (i = 0; i < EXCLUDE.length; i++) {
-    if (name === EXCLUDE[i]) return true;
-  }
-  for (i = 0; i < EXCLUDE_EXT.length; i++) {
-    if (name.indexOf(EXCLUDE_EXT[i]) >= 0) return true;
-  }
-  return false;
-}
-
-// –––––––––––––––––––––––––––––
-
-function ensureRepo() {
-  if (fs.existsSync(path.join(REPO_DIR, ".git"))) return;
-  execSync("rm -rf " + REPO_DIR, { stdio: "pipe" });
-  try {
-    execSync("git clone --depth 1 " + repoUrl() + " " + REPO_DIR, {
-      stdio: "pipe", timeout: 120000,
-      env: Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
-    });
-    log("Repo cloned");
-  } catch (e) {
-    log("Clone failed, init empty: " + e.message);
-    fs.mkdirSync(REPO_DIR, { recursive: true });
-    execSync("git init " + REPO_DIR, { stdio: "pipe" });
-    git("remote add origin " + repoUrl());
-  }
-  try {
-    git("config user.email openclaw@hf.space");
-    git("config user.name OpenClaw-Sync");
-    git("config pull.rebase false");
-  } catch (e) { /* non-fatal */ }
-}
-
-function pull() {
-  try { git("fetch --quiet origin"); } catch (e) { /* ignore */ }
-  try { git("pull --quiet --no-rebase origin main"); }
-  catch (e) {
-    try { git("pull --quiet --no-rebase origin master"); }
-    catch (e2) { /* empty repo */ }
-  }
-}
-
-function push() {
-  try {
+function commitAndPush(){
+  try{
     git("add -A");
     var changed = git("diff --cached --name-only");
-    if (!changed) return 0;
-    var n  = changed.split("\n").filter(Boolean).length;
-    var ts = new Date().toISOString().replace("T", " ").substring(0, 19);
-    git('commit -m "sync ' + ts + '"');
-    try { git("push --quiet origin HEAD:main"); }
-    catch (e) { git("push --quiet origin HEAD:master"); }
-    return n;
-  } catch (e) {
-    log("Push failed: " + e.message);
-    return 0;
-  }
+    if(!changed) return 0;
+    var ts = new Date().toISOString().replace("T"," ").substring(0,19);
+    var count = changed.split("\n").filter(Boolean).length;
+    git('commit -m "memory-sync '+ts+'"');
+    try{ git("push --quiet origin HEAD:main"); } catch(e){ git("push --quiet origin HEAD:master"); }
+    return count;
+  }catch(e){ log("Push failed: "+e.message); return 0; }
 }
 
-// –––––––––––––––––––––––––––––
+function readJsonl(filePath,targetArray){ if(!fs.existsSync(filePath)) return; fs.readFileSync(filePath,"utf-8").split("\n").filter(Boolean).forEach(line=>{ try{ targetArray.push(JSON.parse(line)) }catch(e){} }); }
+function writeJsonl(filePath,dataArray){ fs.writeFileSync(filePath,dataArray.map(x=>JSON.stringify(x)).join("\n"),"utf-8"); }
 
-function mirrorDirToRepo(srcDir, dstDir) {
-  if (!fs.existsSync(srcDir)) return 0;
-  fs.mkdirSync(dstDir, { recursive: true });
-  var count = 0;
-  fs.readdirSync(srcDir).forEach(function(name) {
-    if (shouldExclude(name)) return;
-    var src = path.join(srcDir, name);
-    var dst = path.join(dstDir, name);
-    try {
-      if (fs.statSync(src).isDirectory()) {
-        count += mirrorDirToRepo(src, dst);
-      } else {
-        fs.copyFileSync(src, dst);
-        count++;
-      }
-    } catch (e) {
-      log("Copy error [" + name + "]: " + e.message);
-    }
-  });
-  return count;
-}
+function todayDir(){ var d=new Date(); return d.getFullYear()+"-"+String(d.getMonth()+1).padStart(2,"0")+"-"+String(d.getDate()).padStart(2,"0"); }
+function getRepoDateDir(dateStr){ var dir=path.join(REPO_DIR,"dataset-memory",dateStr); fs.mkdirSync(dir,{recursive:true}); return dir; }
 
-function mirrorDirFromRepo(srcDir, dstDir) {
-  if (!fs.existsSync(srcDir)) return 0;
-  fs.mkdirSync(dstDir, { recursive: true });
-  var count = 0;
-  fs.readdirSync(srcDir).forEach(function(name) {
-    var src = path.join(srcDir, name);
-    var dst = path.join(dstDir, name);
-    try {
-      if (fs.statSync(src).isDirectory()) {
-        count += mirrorDirFromRepo(src, dst);
-      } else {
-        fs.copyFileSync(src, dst);
-        count++;
-      }
-    } catch (e) {
-      log("Restore error [" + name + "]: " + e.message);
-    }
-  });
-  return count;
+function loadMemoryFromRepo(){
+  var repoBase = path.join(REPO_DIR,"dataset-memory");
+  if(!fs.existsSync(repoBase)) return;
+  var dateDirs = fs.readdirSync(repoBase).filter(f=>fs.statSync(path.join(repoBase,f)).isDirectory());
+  dateDirs.forEach(dateDir=>{ TRACK_NAMES.forEach(track=>{ readJsonl(path.join(repoBase,dateDir,track+".jsonl"),botMemory[track]); }); });
+  log("Loaded: conversations="+botMemory.conversations.length+" memory="+botMemory.memory.length+" logs="+botMemory.logs.length);
 }
 
-// –––––––––––––––––––––––––––––
-
-function seedWorkspaceIfNeeded() {
-  fs.mkdirSync(WORKSPACE, { recursive: true });
-  fs.mkdirSync(path.join(WORKSPACE, "memory"), { recursive: true });
-
-  var soul = path.join(WORKSPACE, "SOUL.md");
-  if (!fs.existsSync(soul)) {
-    fs.writeFileSync(soul, [
-      "# Soul",
-      "",
-      "You are a helpful, warm, concise AI assistant.",
-      "",
-      "## Language",
-      "",
-      "Default language: Simplified Chinese.",
-      "Always reply in Chinese unless the user writes in another language first.",
-      "",
-      "## Tone",
-      "",
-      "- Natural and friendly, not overly formal",
-      "- Concise and to the point",
-      "- Ask one clarifying question at a time when needed"
-    ].join("\n") + "\n", "utf-8");
-    log("Seeded workspace/SOUL.md");
-  }
-
-  var agents = path.join(WORKSPACE, "AGENTS.md");
-  if (!fs.existsSync(agents)) {
-    fs.writeFileSync(agents, [
-      "# Agent Instructions",
-      "",
-      "## Boot sequence",
-      "",
-      "1. Read SOUL.md - language and persona",
-      "2. Read USER.md if present - user profile",
-      "3. Read MEMORY.md - long-term facts and rules",
-      "4. Read today and yesterday memory/YYYY-MM-DD.md if present",
-      "",
-      "## Memory rules",
-      "",
-      "- Write important facts to MEMORY.md when asked",
-      "- Log daily context to memory/YYYY-MM-DD.md",
-      "- Do not ask for information already provided"
-    ].join("\n") + "\n", "utf-8");
-    log("Seeded workspace/AGENTS.md");
-  }
-
-  var mem = path.join(WORKSPACE, "MEMORY.md");
-  if (!fs.existsSync(mem)) {
-    fs.writeFileSync(mem, [
-      "# Long-term Memory",
-      "",
-      "<!-- OpenClaw writes important facts here. -->",
-      "<!-- Loaded at the start of every session. -->"
-    ].join("\n") + "\n", "utf-8");
-    log("Seeded workspace/MEMORY.md");
-  }
+function saveMemoryToRepo(){
+  var dateStr=todayDir(); var destDir=getRepoDateDir(dateStr); TRACK_NAMES.forEach(track=>writeJsonl(path.join(destDir,track+".jsonl"),botMemory[track]));
 }
 
-// –––––––––––––––––––––––––––––
-
-function bootRestore() {
-  if (!HF_TOKEN || !DATASET_ID) {
-    log("No HF_TOKEN/HF_DATASET_ID - skipping Dataset restore");
-    seedWorkspaceIfNeeded();
-    return;
-  }
-
-  log("Boot restore from " + DATASET_ID + "…");
-  try {
-    ensureRepo();
-    pull();
-  } catch (e) {
-    log("Repo init failed (" + e.message + ") - starting fresh");
-    seedWorkspaceIfNeeded();
-    return;
-  }
-
-  var repoOC = path.join(REPO_DIR, "openclaw");
-  if (fs.existsSync(repoOC)) {
-    var n = mirrorDirFromRepo(repoOC, OPENCLAW_DIR);
-    log("Boot restore done: " + n + " file(s) -> " + OPENCLAW_DIR);
-  } else {
-    log("No data in Dataset yet - starting fresh");
-  }
-
-  seedWorkspaceIfNeeded();
-  log("Boot restore complete");
+function bootRestore(){
+  if(!HF_TOKEN || !DATASET_ID){ log("HF_TOKEN or HF_DATASET_ID not set, skipping restore"); return; }
+  log("Boot restore starting from "+DATASET_ID+"...");
+  try{ ensureRepo(); pullLatest(); loadMemoryFromRepo(); log("Boot restore done."); }catch(e){ log("Boot restore failed (non-fatal): "+e.message); }
 }
 
-function runSync() {
-  if (!HF_TOKEN || !DATASET_ID) return;
-
-  var ts = new Date().toISOString().replace("T", " ").substring(0, 19);
-  log("Sync at " + ts + "…");
-
-  try {
-    ensureRepo();
-    pull();
-
-    var repoOC = path.join(REPO_DIR, "openclaw");
-    if (fs.existsSync(repoOC)) {
-      execSync("rm -rf " + repoOC, { stdio: "pipe" });
-    }
-    fs.mkdirSync(repoOC, { recursive: true });
-
-    var n = mirrorDirToRepo(OPENCLAW_DIR, repoOC);
-    log("Mirrored " + n + " file(s)");
-
-    var pushed = push();
-    if (pushed > 0) {
-      log("Sync done: " + pushed + " file(s) pushed");
-    } else {
-      log("Sync done: no changes");
-    }
-  } catch (e) {
-    log("Sync failed (retry in 30 min): " + e.message);
+async function startUploadLoop(){
+  while(true){
+    await sleep(UPLOAD_INTERVAL);
+    try{ log("Uploading memory to repo..."); ensureRepo(); pullLatest(); saveMemoryToRepo(); var count=commitAndPush(); log("Upload done: "+count+" file(s) changed"); }catch(e){ log("Upload error: "+e.message); }
   }
 }
 
-// –––––––––––––––––––––––––––––
-// Main - use IIFE to avoid any hoisting issues
-// –––––––––––––––––––––––––––––
-
-(async function() {
-  log("Starting…");
-  log("OPENCLAW_DIR : " + OPENCLAW_DIR);
-  log("DATASET      : " + (DATASET_ID || "NOT SET"));
-
-  fs.mkdirSync(OPENCLAW_DIR, { recursive: true });
-  fs.mkdirSync(WORKSPACE,    { recursive: true });
-
-  try {
-    execSync("git config --global user.email openclaw@hf.space", { stdio: "pipe" });
-    execSync("git config --global user.name OpenClaw-Sync",      { stdio: "pipe" });
-    execSync("git config --global http.postBuffer 52428800",     { stdio: "pipe" });
-    execSync("git config --global pull.rebase false",            { stdio: "pipe" });
-  } catch (e) { /* non-fatal */ }
-
-  var isGatewayRestart = fs.existsSync(RUNNING_FLAG);
-
-  if (isGatewayRestart) {
-    log("Gateway restart - skipping boot restore (data on disk)");
-  } else {
-    log("Container start - running full boot restore");
-    bootRestore();
+async function main(){
+  log("Starting...");
+  log("OPENCLAW_DIR : "+OPENCLAW_DIR);
+  log("HF_DATASET_ID: "+(DATASET_ID||"NOT SET"));
+  log("Upload interval: "+(UPLOAD_INTERVAL/3600000)+"h");
+
+  fs.mkdirSync(MEMORY_DIR,{recursive:true});
+  try{
+    execSync("git config --global user.email openclaw@hf.space",{stdio:"pipe"});
+    execSync("git config --global user.name OpenClaw-Memory-Sync",{stdio:"pipe"});
+    execSync("git config --global http.postBuffer 52428800",{stdio:"pipe"});
+    execSync("git config --global pull.rebase false",{stdio:"pipe"});
+  }catch(e){}
+
+  bootRestore();
+
+  if(HF_TOKEN && DATASET_ID){
+    log("Upload loop started (first run in "+(UPLOAD_INTERVAL/3600000)+"h)");
+    startUploadLoop().catch(e=>{ log("Upload loop error: "+e.message); });
   }
 
-  fs.writeFileSync(RUNNING_FLAG, String(process.pid), "utf-8");
-
-  process.on("exit",    function() { try { fs.unlinkSync(RUNNING_FLAG); } catch (e) { /* ok */ } });
-  process.on("SIGTERM", function() { try { fs.unlinkSync(RUNNING_FLAG); } catch (e) { /* ok */ } process.exit(0); });
-  process.on("SIGINT",  function() { try { fs.unlinkSync(RUNNING_FLAG); } catch (e) { /* ok */ } process.exit(0); });
-
-  if (!HF_TOKEN || !DATASET_ID) {
-    log("Sync disabled: set HF_TOKEN and HF_DATASET_ID in Secrets");
-    return;
-  }
-
-  log("Sync loop starting (first run in 10 min)…");
-  await sleep(10 * 60 * 1000);
+  log("Ready.");
+}
 
-  while (true) {
-    runSync();
-    await sleep(INTERVAL);
-  }
-})().catch(function(e) { 
-  console.error("[hf-sync] Fatal: " + e.message); 
-});
+main().catch(e=>{ console.error("[hf-sync] Fatal: "+e.message); });