8900 commited on
Commit
e73e1a8
·
verified ·
1 Parent(s): c0df9f7

Update hf-sync-manager.mjs

Browse files
Files changed (1) hide show
  1. hf-sync-manager.mjs +101 -201
hf-sync-manager.mjs CHANGED
@@ -3,17 +3,24 @@
3
  //
4
  // Goal: Dataset = OpenClaw runtime data, always in sync.
5
  //
6
- // Boot: Dataset -> ~/.openclaw/ (full restore)
7
  // Runtime: ~/.openclaw/ -> Dataset every 30 min (full mirror)
8
- // On die: SIGTERM handler uploads immediately before exit
9
  //
10
- // Mirror = exact copy. Whatever OpenClaw writes, Dataset reflects.
11
- // New agents, new skills, new memory - all automatic, no config needed.
 
 
 
 
 
 
 
 
12
  //
13
- // Excluded (runtime-only, auto-rebuilt, no value in backup):
14
- // agents/*/sessions/ - raw session transcripts (large, rebuilt from conv)
15
- // agents/*/qmd/ - sqlite vector index (rebuilt on boot)
16
- // canvas/ - UI temp files
17
  // *.tmp *.log
18
  // ============================================================
19
 
@@ -21,70 +28,54 @@ import fs from "node:fs";
21
  import path from "node:path";
22
  import { execSync } from "node:child_process";
23
 
24
- // –– config ————————————————
25
-
26
  var HOME = process.env.OPENCLAW_HOME || process.env.HOME || "/home/user";
27
  var OPENCLAW_DIR = path.join(HOME, ".openclaw");
28
  var WORKSPACE = path.join(OPENCLAW_DIR, "workspace");
29
- var HF_TOKEN = (process.env.HF_TOKEN || "").trim();
30
- var DATASET_ID = (process.env.HF_DATASET_ID || "").trim();
31
  var REPO_DIR = "/tmp/oc-dataset";
32
- var INTERVAL_MS = 30 * 60 * 1000; // 30 minutes
33
 
34
- // Excluded from both upload and restore
35
- var EXCLUDE_DIRS = ["sessions", "qmd", "canvas", ".git"];
36
  var EXCLUDE_EXTS = [".tmp", ".log"];
37
 
38
- // –– helpers ———————————————–
39
-
40
  function log(msg) {
41
  var ts = new Date().toISOString().substring(11, 19);
42
  console.log("[hf-sync " + ts + "] " + msg);
43
  }
44
-
45
- function sleep(ms) {
46
- return new Promise(function(r) { setTimeout(r, ms); });
47
- }
48
 
49
  function repoUrl() {
50
- return "https://user:" + HF_TOKEN +
51
- "@huggingface.co/datasets/" + DATASET_ID;
52
  }
53
 
54
- function git(args, timeoutMs) {
55
  return execSync("git " + args, {
56
- cwd: REPO_DIR,
57
- stdio: "pipe",
58
- timeout: timeoutMs || 120000,
59
- env: Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
60
  }).toString().trim();
61
  }
62
 
63
  function shouldSkip(name) {
64
  var i;
65
- for (i = 0; i < EXCLUDE_DIRS.length; i++) {
66
- if (name === EXCLUDE_DIRS[i]) return true;
67
- }
68
  for (i = 0; i < EXCLUDE_EXTS.length; i++) {
69
  if (name.slice(-EXCLUDE_EXTS[i].length) === EXCLUDE_EXTS[i]) return true;
70
  }
71
  return false;
72
  }
73
 
74
- // –– git repo setup ––––––––––––––––––––
75
-
76
  function ensureRepo() {
77
  if (fs.existsSync(path.join(REPO_DIR, ".git"))) return;
78
  execSync("rm -rf " + REPO_DIR, { stdio: "pipe" });
79
  try {
80
  execSync("git clone --depth 1 " + repoUrl() + " " + REPO_DIR, {
81
- stdio: "pipe",
82
- timeout: 60000,
83
- env: Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
84
  });
85
- log("Repo cloned from Dataset");
86
  } catch (e) {
87
- log("Clone failed (empty repo?), initializing: " + e.message);
88
  fs.mkdirSync(REPO_DIR, { recursive: true });
89
  execSync("git init " + REPO_DIR, { stdio: "pipe" });
90
  git("remote add origin " + repoUrl());
@@ -93,15 +84,15 @@ try {
93
  git("config user.email openclaw@hf.space");
94
  git("config user.name OpenClaw-Mirror");
95
  git("config pull.rebase false");
96
- } catch (e) { /* non-fatal */ }
97
  }
98
 
99
  function pullLatest() {
100
- try { git("fetch --quiet origin", 30000); } catch (e) { /* ignore */ }
101
  try { git("pull --quiet --no-rebase origin main", 30000); }
102
  catch (e) {
103
  try { git("pull --quiet --no-rebase origin master", 30000); }
104
- catch (e2) { /* empty repo, first push */ }
105
  }
106
  }
107
 
@@ -112,71 +103,50 @@ var changed = git("diff --cached --name-only");
112
  if (!changed) return 0;
113
  var n = changed.split("\n").filter(Boolean).length;
114
  var ts = new Date().toISOString().replace("T", " ").substring(0, 19);
115
- git('commit -m "' + label + " " + ts + '"');
116
  try { git("push --quiet origin HEAD:main", 30000); }
117
  catch (e) { git("push --quiet origin HEAD:master", 30000); }
118
  return n;
119
  } catch (e) {
120
- log("Commit/push failed: " + e.message);
121
  return 0;
122
  }
123
  }
124
 
125
- // –– mirror: local dir -> repo dir ———————––
126
- // Recursively copies everything except excluded names.
127
-
128
  function copyToRepo(src, dst) {
129
  if (!fs.existsSync(src)) return 0;
130
- var stat = fs.statSync(src);
131
- if (!stat.isDirectory()) {
132
  fs.mkdirSync(path.dirname(dst), { recursive: true });
133
  fs.copyFileSync(src, dst);
134
  return 1;
135
  }
136
  fs.mkdirSync(dst, { recursive: true });
137
  var count = 0;
138
- var items = fs.readdirSync(src);
139
- for (var i = 0; i < items.length; i++) {
140
- var name = items[i];
141
- if (shouldSkip(name)) continue;
142
- try {
143
- count += copyToRepo(path.join(src, name), path.join(dst, name));
144
- } catch (e) {
145
- log("Copy error [" + name + "]: " + e.message);
146
- }
147
- }
148
  return count;
149
  }
150
 
151
- // –– mirror: repo dir -> local dir ———————––
152
-
153
  function copyFromRepo(src, dst) {
154
  if (!fs.existsSync(src)) return 0;
155
- var stat = fs.statSync(src);
156
- if (!stat.isDirectory()) {
157
  fs.mkdirSync(path.dirname(dst), { recursive: true });
158
  fs.copyFileSync(src, dst);
159
  return 1;
160
  }
161
  fs.mkdirSync(dst, { recursive: true });
162
  var count = 0;
163
- var items = fs.readdirSync(src);
164
- for (var i = 0; i < items.length; i++) {
165
- var name = items[i];
166
- if (shouldSkip(name)) continue;
167
- try {
168
- count += copyFromRepo(path.join(src, name), path.join(dst, name));
169
- } catch (e) {
170
- log("Restore error [" + name + "]: " + e.message);
171
- }
172
- }
173
  return count;
174
  }
175
 
176
- // –– seed default workspace ––––––––––––––––
177
- // Only creates files that don’t exist yet.
178
- // If files were restored from Dataset, this is a no-op.
179
-
180
  function seedWorkspace() {
181
  fs.mkdirSync(WORKSPACE, { recursive: true });
182
  fs.mkdirSync(path.join(WORKSPACE, "memory"), { recursive: true });
@@ -191,192 +161,135 @@ fs.writeFileSync(soul, [
191
  "Always reply in Chinese unless the user writes in another language first.", "",
192
  "## Tone", "",
193
  "- Natural and friendly, not overly formal",
194
- "- Concise and to the point",
195
- "- Ask one clarifying question at a time when needed"
196
  ].join("\n") + "\n", "utf-8");
197
- log("Seeded workspace/SOUL.md");
198
  }
199
-
200
- var agents = path.join(WORKSPACE, "AGENTS.md");
201
- if (!fs.existsSync(agents)) {
202
- fs.writeFileSync(agents, [
203
- "# Agent Instructions", "",
204
- "## Boot sequence", "",
205
- "1. Read SOUL.md - language and persona",
206
- "2. Read USER.md if present - user profile",
207
- "3. Read MEMORY.md - long-term facts and rules",
208
- "4. Read today and yesterday memory/YYYY-MM-DD.md if present", "",
209
- "## Memory rules", "",
210
- "- Write important facts to MEMORY.md when asked",
211
- "- Log daily context to memory/YYYY-MM-DD.md",
212
- "- Do not ask for information already provided"
213
- ].join("\n") + "\n", "utf-8");
214
- log("Seeded workspace/AGENTS.md");
215
- }
216
-
217
  var mem = path.join(WORKSPACE, "MEMORY.md");
218
  if (!fs.existsSync(mem)) {
219
  fs.writeFileSync(mem, [
220
  "# Long-term Memory", "",
221
- "<!-- OpenClaw writes important facts here. -->",
222
- "<!-- Loaded at the start of every session. -->"
223
  ].join("\n") + "\n", "utf-8");
224
- log("Seeded workspace/MEMORY.md");
225
  }
226
  }
227
 
228
- // –– auto-generated README ———————————
229
-
230
  function writeReadme() {
231
- var ts = new Date().toISOString().replace("T", " ").substring(0, 19);
232
- var repoOC = REPO_DIR;
233
- var dirs = [];
234
- if (fs.existsSync(repoOC)) {
235
- dirs = fs.readdirSync(repoOC).filter(function(n) {
236
- return fs.statSync(path.join(repoOC, n)).isDirectory();
237
  });
238
- }
239
  var lines = [
240
  "# OpenClaw Dataset Mirror",
241
  "",
242
- "Full mirror of `~/.openclaw/`. Auto-managed. Do not edit.",
243
  "",
244
  "Last sync: " + ts,
245
  "",
246
- "## Contents",
247
  "",
248
  "`",
249
- "(root of Dataset = ~/.openclaw/)",
250
- "openclaw.json <- gateway config",
251
- "workspace/ <- agent brain (SOUL.md, MEMORY.md, daily logs)",
252
- "agents/ <- agent state (sessions excluded - too large)",
253
- "skills/ <- installed skills",
254
- "cron/ <- scheduled tasks",
255
- "... (everything else OpenClaw creates)",
 
 
 
 
256
  "`",
257
  "",
258
- "## Directories found: " + dirs.join(", "),
259
- "",
260
- "## What is NOT mirrored",
261
  "",
262
- "- `agents/*/sessions/` - session transcripts (large, not needed for memory)",
263
- "- `agents/*/qmd/` - vector index (auto-rebuilt on boot)",
264
- "- `canvas/` - UI temp files"
265
  ];
266
  fs.writeFileSync(path.join(REPO_DIR, "README.md"), lines.join("\n") + "\n", "utf-8");
267
  }
268
 
269
- // –– BOOT RESTORE ——————————————
270
- // Called once at container start.
271
- // Pulls Dataset -> local disk so OpenClaw starts with all previous data.
272
-
273
  function bootRestore() {
274
  if (!HF_TOKEN || !DATASET_ID) {
275
- log("HF_TOKEN or HF_DATASET_ID not set - skipping restore");
276
  seedWorkspace();
277
  return;
278
  }
279
-
280
- log("Boot restore from Dataset: " + DATASET_ID);
281
-
282
  try {
283
  ensureRepo();
284
  pullLatest();
285
  } catch (e) {
286
- log("Repo init failed (" + e.message + ") - starting fresh");
287
  seedWorkspace();
288
  return;
289
  }
290
-
291
- var repoOC = REPO_DIR;
292
- if (fs.existsSync(repoOC)) {
293
- var n = copyFromRepo(repoOC, OPENCLAW_DIR);
294
- log("Restored " + n + " file(s) -> " + OPENCLAW_DIR);
295
  } else {
296
  log("No data in Dataset yet - starting fresh");
297
  }
298
-
299
- // Seed workspace files that are still missing after restore
300
  seedWorkspace();
301
  log("Boot restore complete");
302
  }
303
 
304
- // –– FULL MIRROR UPLOAD ————————————
305
- // Called every 30 min.
306
- // Takes a complete snapshot of ~/.openclaw/ and pushes to Dataset.
307
- // Dataset ends up looking exactly like OpenClaw’s disk state.
308
-
309
  function runMirror(label) {
310
  if (!HF_TOKEN || !DATASET_ID) return;
311
-
312
  label = label || "mirror";
313
  var ts = new Date().toISOString().replace("T", " ").substring(0, 19);
314
- log("Mirror upload starting [" + label + "] at " + ts + "…");
315
-
316
  try {
317
  ensureRepo();
318
  pullLatest();
319
 
320
- // Clear repo's openclaw snapshot and re-copy everything fresh
321
- // Clear all tracked content (except .git) then re-copy
322
- // This ensures Dataset = exact mirror of OpenClaw disk state
323
- var items = fs.existsSync(REPO_DIR) ? fs.readdirSync(REPO_DIR) : [];
324
- items.forEach(function(name) {
325
  if (name === ".git" || name === "README.md") return;
326
- var p = path.join(REPO_DIR, name);
327
- try { execSync("rm -rf " + p, { stdio: "pipe" }); } catch(e) { /* ignore */ }
328
  });
329
 
330
  var n = copyToRepo(OPENCLAW_DIR, REPO_DIR);
331
- log("Copied " + n + " file(s) from " + OPENCLAW_DIR);
332
 
333
  writeReadme();
334
 
335
  var pushed = commitAndPush(label);
336
  if (pushed > 0) {
337
- log("Mirror done: " + pushed + " file(s) pushed to Dataset");
338
  } else {
339
- log("Mirror done: no changes since last mirror");
340
  }
 
341
  } catch (e) {
342
- log("Mirror failed: " + e.message);
343
- // Loop continues regardless - never let an error stop the mirror loop
344
  }
345
  }
346
 
347
- // –– SIGTERM emergency upload ——————————
348
- // When container is about to die (gateway restart or HF stopping container),
349
- // immediately upload latest state before exit.
350
-
351
  var emergencyDone = false;
352
  function emergencyUpload() {
353
  if (emergencyDone) return;
354
  emergencyDone = true;
355
- log("SIGTERM received - emergency upload before exit…");
356
- try {
357
- runMirror("emergency");
358
- log("Emergency upload complete");
359
- } catch (e) {
360
- log("Emergency upload failed: " + e.message);
361
  }
362
- }
363
-
364
- process.on("SIGTERM", function() {
365
- emergencyUpload();
366
- process.exit(0);
367
- });
368
-
369
- process.on("SIGINT", function() {
370
- process.exit(0);
371
- });
372
 
373
- // –– MAIN –––––––––––––––––––––––––
 
374
 
375
  (async function() {
376
  log("Starting…");
377
  log("OPENCLAW_DIR : " + OPENCLAW_DIR);
378
  log("DATASET : " + (DATASET_ID || "NOT SET"));
379
- log("Mirror every : " + (INTERVAL_MS / 60000) + " min");
380
 
381
  fs.mkdirSync(OPENCLAW_DIR, { recursive: true });
382
  fs.mkdirSync(WORKSPACE, { recursive: true });
@@ -386,39 +299,26 @@ execSync("git config --global user.email openclaw@hf.space", { stdio: "pipe" });
386
  execSync("git config --global user.name OpenClaw-Mirror", { stdio: "pipe" });
387
  execSync("git config --global http.postBuffer 52428800", { stdio: "pipe" });
388
  execSync("git config --global pull.rebase false", { stdio: "pipe" });
389
- } catch (e) { /* non-fatal */ }
390
 
391
- // Step 1: restore from Dataset before gateway starts reading data
392
  bootRestore();
393
 
394
  if (!HF_TOKEN || !DATASET_ID) {
395
- log("Mirror loop disabled: set HF_TOKEN and HF_DATASET_ID in Space Secrets");
396
- log("Ready (no Dataset sync)");
397
  return;
398
  }
399
 
400
- // Step 2: first upload 10 min after boot
401
- // (give OpenClaw time to fully start and write initial data)
402
- log("First mirror upload in 10 min…");
403
  await sleep(10 * 60 * 1000);
404
 
405
- // Step 3: mirror loop - runs forever, catches all errors internally
406
  var cycle = 0;
407
  while (true) {
408
  cycle++;
409
- try {
410
- runMirror("mirror-" + cycle);
411
- } catch (e) {
412
- // This outer catch is a final safety net.
413
- // runMirror already catches errors, but just in case.
414
- log("Unexpected error in mirror cycle " + cycle + ": " + e.message);
415
- }
416
- // Always sleep and continue, even if this cycle failed
417
  await sleep(INTERVAL_MS);
418
  }
419
  })().catch(function(e) {
420
- // This should never fire, but if it does, log and don’t crash
421
- log("Fatal error in main: " + e.message);
422
- log("Attempting emergency upload before exit…");
423
  emergencyUpload();
424
  });
 
3
  //
4
  // Goal: Dataset = OpenClaw runtime data, always in sync.
5
  //
6
+ // Boot: Dataset -> ~/.openclaw/ (full restore, then setup patches env vars)
7
  // Runtime: ~/.openclaw/ -> Dataset every 30 min (full mirror)
8
+ // On die: SIGTERM uploads immediately
9
  //
10
+ // SYNCED (all OpenClaw runtime data):
11
+ // openclaw.json - gateway config (user settings preserved via patch mode)
12
+ // workspace/ - agent brain
13
+ // agents/*/sessions/ - conversation history (bot remembers you)
14
+ // agents/*/agent/ - agent state
15
+ // devices/ - sub-agent pairing info (survives restart)
16
+ // credentials/ - auth
17
+ // skills/ - installed skills
18
+ // cron/ - scheduled tasks
19
+ // memory/ - vector DB
20
  //
21
+ // NOT SYNCED (auto-rebuilt):
22
+ // agents/*/qmd/ - vector index (rebuilt from markdown)
23
+ // canvas/ - UI temp files
 
24
  // *.tmp *.log
25
  // ============================================================
26
 
 
28
  import path from "node:path";
29
  import { execSync } from "node:child_process";
30
 
 
 
31
  var HOME = process.env.OPENCLAW_HOME || process.env.HOME || "/home/user";
32
  var OPENCLAW_DIR = path.join(HOME, ".openclaw");
33
  var WORKSPACE = path.join(OPENCLAW_DIR, "workspace");
34
+ var HF_TOKEN = (process.env.HF_TOKEN || "").trim();
35
+ var DATASET_ID = (process.env.HF_DATASET_ID || "").trim();
36
  var REPO_DIR = "/tmp/oc-dataset";
37
+ var INTERVAL_MS = 30 * 60 * 1000;
38
 
39
+ var EXCLUDE_DIRS = ["qmd", "canvas", ".git"];
 
40
  var EXCLUDE_EXTS = [".tmp", ".log"];
41
 
 
 
42
  function log(msg) {
43
  var ts = new Date().toISOString().substring(11, 19);
44
  console.log("[hf-sync " + ts + "] " + msg);
45
  }
46
+ function sleep(ms) { return new Promise(function(r) { setTimeout(r, ms); }); }
 
 
 
47
 
48
  function repoUrl() {
49
+ return "https://user:" + HF_TOKEN + "@huggingface.co/datasets/" + DATASET_ID;
 
50
  }
51
 
52
+ function git(args, ms) {
53
  return execSync("git " + args, {
54
+ cwd: REPO_DIR, stdio: "pipe", timeout: ms || 120000,
55
+ env: Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
 
 
56
  }).toString().trim();
57
  }
58
 
59
  function shouldSkip(name) {
60
  var i;
61
+ for (i = 0; i < EXCLUDE_DIRS.length; i++) { if (name === EXCLUDE_DIRS[i]) return true; }
 
 
62
  for (i = 0; i < EXCLUDE_EXTS.length; i++) {
63
  if (name.slice(-EXCLUDE_EXTS[i].length) === EXCLUDE_EXTS[i]) return true;
64
  }
65
  return false;
66
  }
67
 
 
 
68
  function ensureRepo() {
69
  if (fs.existsSync(path.join(REPO_DIR, ".git"))) return;
70
  execSync("rm -rf " + REPO_DIR, { stdio: "pipe" });
71
  try {
72
  execSync("git clone --depth 1 " + repoUrl() + " " + REPO_DIR, {
73
+ stdio: "pipe", timeout: 60000,
74
+ env: Object.assign({}, process.env, { GIT_TERMINAL_PROMPT: "0" })
 
75
  });
76
+ log("Cloned Dataset");
77
  } catch (e) {
78
+ log("Clone failed, init: " + e.message);
79
  fs.mkdirSync(REPO_DIR, { recursive: true });
80
  execSync("git init " + REPO_DIR, { stdio: "pipe" });
81
  git("remote add origin " + repoUrl());
 
84
  git("config user.email openclaw@hf.space");
85
  git("config user.name OpenClaw-Mirror");
86
  git("config pull.rebase false");
87
+ } catch (e) { /* ok */ }
88
  }
89
 
90
  function pullLatest() {
91
+ try { git("fetch --quiet origin", 30000); } catch (e) { /* ok */ }
92
  try { git("pull --quiet --no-rebase origin main", 30000); }
93
  catch (e) {
94
  try { git("pull --quiet --no-rebase origin master", 30000); }
95
+ catch (e2) { /* empty repo */ }
96
  }
97
  }
98
 
 
103
  if (!changed) return 0;
104
  var n = changed.split("\n").filter(Boolean).length;
105
  var ts = new Date().toISOString().replace("T", " ").substring(0, 19);
106
+ git("commit -m \"" + label + " " + ts + "\"");
107
  try { git("push --quiet origin HEAD:main", 30000); }
108
  catch (e) { git("push --quiet origin HEAD:master", 30000); }
109
  return n;
110
  } catch (e) {
111
+ log("Push failed: " + e.message);
112
  return 0;
113
  }
114
  }
115
 
 
 
 
116
  function copyToRepo(src, dst) {
117
  if (!fs.existsSync(src)) return 0;
118
+ if (!fs.statSync(src).isDirectory()) {
 
119
  fs.mkdirSync(path.dirname(dst), { recursive: true });
120
  fs.copyFileSync(src, dst);
121
  return 1;
122
  }
123
  fs.mkdirSync(dst, { recursive: true });
124
  var count = 0;
125
+ fs.readdirSync(src).forEach(function(name) {
126
+ if (shouldSkip(name)) return;
127
+ try { count += copyToRepo(path.join(src, name), path.join(dst, name)); }
128
+ catch (e) { log("Copy error [" + name + "]: " + e.message); }
129
+ });
 
 
 
 
 
130
  return count;
131
  }
132
 
 
 
133
  function copyFromRepo(src, dst) {
134
  if (!fs.existsSync(src)) return 0;
135
+ if (!fs.statSync(src).isDirectory()) {
 
136
  fs.mkdirSync(path.dirname(dst), { recursive: true });
137
  fs.copyFileSync(src, dst);
138
  return 1;
139
  }
140
  fs.mkdirSync(dst, { recursive: true });
141
  var count = 0;
142
+ fs.readdirSync(src).forEach(function(name) {
143
+ if (shouldSkip(name)) return;
144
+ try { count += copyFromRepo(path.join(src, name), path.join(dst, name)); }
145
+ catch (e) { log("Restore error [" + name + "]: " + e.message); }
146
+ });
 
 
 
 
 
147
  return count;
148
  }
149
 
 
 
 
 
150
  function seedWorkspace() {
151
  fs.mkdirSync(WORKSPACE, { recursive: true });
152
  fs.mkdirSync(path.join(WORKSPACE, "memory"), { recursive: true });
 
161
  "Always reply in Chinese unless the user writes in another language first.", "",
162
  "## Tone", "",
163
  "- Natural and friendly, not overly formal",
164
+ "- Concise and to the point"
 
165
  ].join("\n") + "\n", "utf-8");
166
+ log("Seeded SOUL.md");
167
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
168
  var mem = path.join(WORKSPACE, "MEMORY.md");
169
  if (!fs.existsSync(mem)) {
170
  fs.writeFileSync(mem, [
171
  "# Long-term Memory", "",
172
+ "<!-- OpenClaw writes important facts here. -->"
 
173
  ].join("\n") + "\n", "utf-8");
174
+ log("Seeded MEMORY.md");
175
  }
176
  }
177
 
 
 
178
  function writeReadme() {
179
+ var ts = new Date().toISOString().replace("T", " ").substring(0, 19);
180
+ var dirs = [];
181
+ try {
182
+ dirs = fs.readdirSync(REPO_DIR).filter(function(n) {
183
+ if (n === ".git" || n === "README.md") return false;
184
+ return fs.statSync(path.join(REPO_DIR, n)).isDirectory();
185
  });
186
+ } catch (e) { /* ok */ }
187
  var lines = [
188
  "# OpenClaw Dataset Mirror",
189
  "",
190
+ "Full mirror of ~/.openclaw/ - auto-managed, do not edit.",
191
  "",
192
  "Last sync: " + ts,
193
  "",
194
+ "## Structure (mirrors ~/.openclaw/ exactly)",
195
  "",
196
  "`",
197
+ "openclaw.json <- gateway config (preserved across restarts)",
198
+ "workspace/ <- agent brain: SOUL.md, MEMORY.md, daily logs",
199
+ "agents/ <- all agent data",
200
+ " main/",
201
+ " sessions/ <- conversation history (bot memory)",
202
+ " agent/ <- agent state",
203
+ "devices/ <- sub-agent pairing (survives restart)",
204
+ "credentials/ <- auth",
205
+ "skills/ <- installed skills",
206
+ "cron/ <- scheduled tasks",
207
+ "memory/ <- vector database",
208
  "`",
209
  "",
210
+ "Found: " + (dirs.length > 0 ? dirs.join(", ") : "(pending first sync)"),
 
 
211
  "",
212
+ "Not synced: agents/*/qmd/ (vector index, rebuilt on boot), canvas/ (UI temp)"
 
 
213
  ];
214
  fs.writeFileSync(path.join(REPO_DIR, "README.md"), lines.join("\n") + "\n", "utf-8");
215
  }
216
 
 
 
 
 
217
  function bootRestore() {
218
  if (!HF_TOKEN || !DATASET_ID) {
219
+ log("No HF_TOKEN/HF_DATASET_ID - skip restore");
220
  seedWorkspace();
221
  return;
222
  }
223
+ log("Boot restore from " + DATASET_ID + "…");
 
 
224
  try {
225
  ensureRepo();
226
  pullLatest();
227
  } catch (e) {
228
+ log("Repo init failed: " + e.message + " - starting fresh");
229
  seedWorkspace();
230
  return;
231
  }
232
+ var n = copyFromRepo(REPO_DIR, OPENCLAW_DIR);
233
+ if (n > 0) {
234
+ log("Restored " + n + " file(s) to " + OPENCLAW_DIR);
235
+ log("Includes: conversations, device pairings, skills, config");
 
236
  } else {
237
  log("No data in Dataset yet - starting fresh");
238
  }
 
 
239
  seedWorkspace();
240
  log("Boot restore complete");
241
  }
242
 
 
 
 
 
 
243
  function runMirror(label) {
244
  if (!HF_TOKEN || !DATASET_ID) return;
 
245
  label = label || "mirror";
246
  var ts = new Date().toISOString().replace("T", " ").substring(0, 19);
247
+ log("Mirror [" + label + "] at " + ts + "…");
 
248
  try {
249
  ensureRepo();
250
  pullLatest();
251
 
252
+ // Clear all content (except .git and README) then re-copy fresh
253
+ fs.readdirSync(REPO_DIR).forEach(function(name) {
 
 
 
254
  if (name === ".git" || name === "README.md") return;
255
+ try { execSync("rm -rf " + path.join(REPO_DIR, name), { stdio: "pipe" }); }
256
+ catch (e) { /* ok */ }
257
  });
258
 
259
  var n = copyToRepo(OPENCLAW_DIR, REPO_DIR);
260
+ log("Copied " + n + " file(s) from local");
261
 
262
  writeReadme();
263
 
264
  var pushed = commitAndPush(label);
265
  if (pushed > 0) {
266
+ log("Pushed " + pushed + " file(s) to Dataset");
267
  } else {
268
+ log("No changes since last mirror");
269
  }
270
+
271
  } catch (e) {
272
+ log("Mirror failed (will retry): " + e.message);
 
273
  }
274
  }
275
 
 
 
 
 
276
  var emergencyDone = false;
277
  function emergencyUpload() {
278
  if (emergencyDone) return;
279
  emergencyDone = true;
280
+ log("SIGTERM - emergency upload…");
281
+ try { runMirror("emergency"); log("Emergency done"); }
282
+ catch (e) { log("Emergency failed: " + e.message); }
 
 
 
283
  }
 
 
 
 
 
 
 
 
 
 
284
 
285
+ process.on("SIGTERM", function() { emergencyUpload(); process.exit(0); });
286
+ process.on("SIGINT", function() { process.exit(0); });
287
 
288
  (async function() {
289
  log("Starting…");
290
  log("OPENCLAW_DIR : " + OPENCLAW_DIR);
291
  log("DATASET : " + (DATASET_ID || "NOT SET"));
292
+ log("Syncing : sessions + devices + skills + config + workspace");
293
 
294
  fs.mkdirSync(OPENCLAW_DIR, { recursive: true });
295
  fs.mkdirSync(WORKSPACE, { recursive: true });
 
299
  execSync("git config --global user.name OpenClaw-Mirror", { stdio: "pipe" });
300
  execSync("git config --global http.postBuffer 52428800", { stdio: "pipe" });
301
  execSync("git config --global pull.rebase false", { stdio: "pipe" });
302
+ } catch (e) { /* ok */ }
303
 
 
304
  bootRestore();
305
 
306
  if (!HF_TOKEN || !DATASET_ID) {
307
+ log("Sync disabled: set HF_TOKEN and HF_DATASET_ID in Secrets");
 
308
  return;
309
  }
310
 
311
+ log("First mirror in 10 min…");
 
 
312
  await sleep(10 * 60 * 1000);
313
 
 
314
  var cycle = 0;
315
  while (true) {
316
  cycle++;
317
+ try { runMirror("mirror-" + cycle); }
318
+ catch (e) { log("Cycle " + cycle + " error: " + e.message); }
 
 
 
 
 
 
319
  await sleep(INTERVAL_MS);
320
  }
321
  })().catch(function(e) {
322
+ log("Fatal: " + e.message);
 
 
323
  emergencyUpload();
324
  });