Upload backup.py with huggingface_hub

backup.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Back up Open WebUI user data to a Hugging Face dataset repo.
+
+Covers everything a user would miss after a volume wipe:
+  * webui.db        - users, logins, settings, chats (consistent SQLite snapshot)
+  * vector_db/      - RAG embeddings for uploaded documents
+  * uploads/        - user-uploaded files
+
+Deliberately EXCLUDES /data/cache (the embedding-model cache, ~1 GB, regenerates
+itself on boot).
+
+Requires the HF_TOKEN env var (Space secret). Never fails the app.
+"""
+import gzip
+import os
+import shutil
+import sqlite3
+import sys
+import tarfile
+import tempfile
+
+DATA_DIR = os.environ.get("OWUI_DATA_DIR", "/data")
+REPO_ID = os.environ.get("OWUI_BACKUP_REPO", "tkgray/open-webui-backups")
+ARCHIVE_IN_REPO = "backups/open-webui-data.tar.gz"
+INCLUDE_DIRS = ("vector_db", "uploads")
+
+DB_PATH = os.path.join(DATA_DIR, "webui.db")
+
+
+def log(msg):
+    print(f"[backup] {msg}", flush=True)
+
+
+def snapshot_sqlite(src, dst):
+    """Write a consistent copy of a live SQLite DB using the backup API.
+
+    Copying the file directly (or tarring it) can capture a torn WAL, so use
+    sqlite3's own online-backup, which is safe while the app is writing.
+    """
+    source = sqlite3.connect(f"file:{src}?mode=ro", uri=True)
+    try:
+        dest = sqlite3.connect(dst)
+        try:
+            with dest:
+                source.backup(dest)
+        finally:
+            dest.close()
+    finally:
+        source.close()
+
+
+def build_archive(payload_dir, archive_path):
+    """Deterministically tar.gz a staged payload.
+
+    Every member's mtime/uid/gid and the gzip header mtime are zeroed and the
+    member order is fixed, so an unchanged payload yields byte-identical output.
+    That lets the dataset host recognise a no-op upload and skip the commit,
+    instead of creating a fresh commit every hour.
+    """
+    entries = []
+    for root, _dirs, files in os.walk(payload_dir):
+        for name in files:
+            full = os.path.join(root, name)
+            entries.append((os.path.relpath(full, payload_dir), full))
+    entries.sort()
+
+    with open(archive_path, "wb") as raw:
+        # filename="" suppresses the gzip FNAME header, which would otherwise
+        # carry the temp archive path and make every run's bytes differ.
+        with gzip.GzipFile(filename="", fileobj=raw, mode="wb", mtime=0) as gz:
+            with tarfile.open(fileobj=gz, mode="w") as tf:
+                for rel, full in entries:
+                    info = tf.gettarinfo(full, arcname=rel)
+                    info.mtime = 0
+                    info.uid = 0
+                    info.gid = 0
+                    info.uname = ""
+                    info.gname = ""
+                    with open(full, "rb") as fh:
+                        tf.addfile(info, fh)
+    return archive_path
+
+
+def main():
+    if not os.path.isfile(DB_PATH):
+        log(f"{DB_PATH} does not exist yet — skipping.")
+        return 0
+
+    token = os.environ.get("HF_TOKEN")
+    if not token:
+        log("HF_TOKEN not set — skipping.")
+        return 0
+
+    stage = tempfile.mkdtemp()
+    payload = os.path.join(stage, "data")
+    os.makedirs(payload, exist_ok=True)
+    try:
+        snapshot_sqlite(DB_PATH, os.path.join(payload, "webui.db"))
+        for name in INCLUDE_DIRS:
+            src = os.path.join(DATA_DIR, name)
+            if os.path.isdir(src):
+                shutil.copytree(src, os.path.join(payload, name), symlinks=True)
+
+        archive = os.path.join(stage, "open-webui-data.tar.gz")
+        build_archive(payload, archive)
+        log(f"staged {os.path.getsize(archive)} bytes; uploading to {REPO_ID}:{ARCHIVE_IN_REPO} ...")
+
+        from huggingface_hub import HfApi
+
+        HfApi(token=token).upload_file(
+            path_or_fileobj=archive,
+            path_in_repo=ARCHIVE_IN_REPO,
+            repo_id=REPO_ID,
+            repo_type="dataset",
+            token=token,
+        )
+        log("Upload OK")
+        return 0
+    except Exception as exc:  # noqa: BLE001 - never fail the app over a backup
+        log(f"Upload FAILED: {exc}")
+        return 1
+    finally:
+        shutil.rmtree(stage, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    sys.exit(main())