"""Publish source-only commits to the private GitHub mirror. GitHub holds the code; it deliberately does NOT hold `data/`. The two repositories therefore have unrelated histories: GitHub was seeded from a `git-filter-repo` export, so the same work carries different SHAs on each side (`chore: rename the project to Readthrough` is 4d22778 here and 7b56291 there). A plain `git push` is rejected as unrelated, and forcing one would destroy the mirror's history. So this script REPLAYS each local commit that touches something outside `data/` on top of the mirror's own history, preserving the message, the author and the date. The result is a fast-forward: nothing on GitHub is overwritten, and no force push is ever needed. It works entirely through a temporary index and `git commit-tree`. **The working tree is never touched** -- you can run it while the app is running and while `data/` holds gigabytes that must not go anywhere near GitHub. State lives in two refs: refs/heads/github-sync the mirror-side branch being built refs/sync/github-last the last local commit already replayed Usage:: python -m scripts.sync_github --dry-run python -m scripts.sync_github See CLAUDE.md, "Publishing", for how this relates to the Hugging Face Space (which is the opposite case: it NEEDS `data/`). """ from __future__ import annotations import argparse import os import subprocess import sys import tempfile REMOTE = "origin" MIRROR_BRANCH = "github-sync" STATE_REF = "refs/sync/github-last" # Everything under this path stays out of the mirror. It is the whole reason # the mirror exists as a separate history. EXCLUDE = ":(exclude)data" def _git(args: list[str], **kw) -> bytes: result = subprocess.run(["git"] + args, capture_output=True, **kw) if result.returncode != 0: sys.exit( f"[sync] git {' '.join(args)} failed:\n" + result.stderr.decode("utf-8", "replace") ) return result.stdout def _rev(ref: str) -> str | None: result = subprocess.run( ["git", "rev-parse", "--verify", "--quiet", ref + "^{commit}"], capture_output=True, ) return result.stdout.decode().strip() or None def replay(*, dry_run: bool = False) -> int: mirror = _rev(MIRROR_BRANCH) if mirror is None: sys.exit( f"[sync] no {MIRROR_BRANCH} branch. It must point at the mirror's " f"history — create it from {REMOTE}/main after fetching." ) last = _rev(STATE_REF) if last is None: sys.exit( f"[sync] {STATE_REF} is unset, so the script cannot tell which local " "commits are already on the mirror. Set it to the last replayed " "commit with: git update-ref " + STATE_REF + " " ) head = _rev("HEAD") shas = _git(["rev-list", "--reverse", f"{last}..HEAD"]).decode().split() if not shas: print("[sync] mirror is already current.") return 0 index = os.path.join(tempfile.gettempdir(), "sync_github_index") if os.path.exists(index): os.remove(index) env = dict(os.environ, GIT_INDEX_FILE=index) subprocess.run(["git", "read-tree", mirror], env=env, check=True) parent, replayed, skipped = mirror, 0, 0 for sha in shas: diff = _git(["diff", "--binary", f"{sha}^", sha, "--", ".", EXCLUDE]) subject = _git(["log", "-1", "--format=%s", sha]).decode().strip() short = sha[:7] if not diff.strip(): print(f"[sync] skip {short} (data only) {subject}") skipped += 1 continue applied = subprocess.run( ["git", "apply", "--cached", "--whitespace=nowarn"], input=diff, env=env, capture_output=True, ) if applied.returncode != 0: sys.exit( f"[sync] cannot replay {short} onto the mirror:\n" + applied.stderr.decode("utf-8", "replace") + "\n[sync] nothing was pushed and no ref was moved." ) tree = subprocess.run( ["git", "write-tree"], env=env, capture_output=True, check=True ).stdout.decode().strip() message = _git(["log", "-1", "--format=%B", sha]) who = _git( ["log", "-1", "--format=%an%n%ae%n%aI%n%cn%n%ce%n%cI", sha] ).decode().splitlines() commit_env = dict( os.environ, GIT_AUTHOR_NAME=who[0], GIT_AUTHOR_EMAIL=who[1], GIT_AUTHOR_DATE=who[2], GIT_COMMITTER_NAME=who[3], GIT_COMMITTER_EMAIL=who[4], GIT_COMMITTER_DATE=who[5], ) parent = subprocess.run( ["git", "commit-tree", tree, "-p", parent], input=message, env=commit_env, capture_output=True, check=True, ).stdout.decode().strip() print(f"[sync] replay {short} {subject}") replayed += 1 print(f"\n[sync] {replayed} replayed, {skipped} skipped as data-only") if replayed == 0: print("[sync] nothing to publish.") return 0 # A last guard before anything leaves the machine: the mirror must carry # no data/ file, and the push must be a fast-forward. tracked_data = _git(["ls-tree", "-r", "--name-only", parent]).decode().splitlines() leaked = [f for f in tracked_data if f.startswith("data/")] if leaked: sys.exit(f"[sync] REFUSING: {len(leaked)} data/ file(s) reached the mirror tree.") if subprocess.run( ["git", "merge-base", "--is-ancestor", mirror, parent] ).returncode != 0: sys.exit("[sync] REFUSING: the result is not a fast-forward of the mirror.") if dry_run: print(f"[sync] dry run — would push {parent[:7]} and move {STATE_REF} to {head[:7]}") return 0 _git(["branch", "-f", MIRROR_BRANCH, parent]) _git(["push", REMOTE, f"{MIRROR_BRANCH}:main"]) _git(["update-ref", STATE_REF, head]) print(f"[sync] pushed {parent[:7]} to {REMOTE}/main") return 0 def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--dry-run", action="store_true", help="build the commits and check the guards, push nothing") args = parser.parse_args() sys.exit(replay(dry_run=args.dry_run)) if __name__ == "__main__": main()