Configure single-repository Xet asset storage
Browse files- .gitattributes +4 -0
- .githooks/pre-commit +3 -0
- .githooks/pre-push +8 -0
- .gitignore +8 -0
- assets/README.md +3 -0
- docs/README.md +2 -1
- docs/STORAGE.md +76 -0
- scripts/setup-repository.py +39 -0
- scripts/storage_policy.py +166 -0
- scripts/test_storage_policy.py +202 -0
.gitattributes
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Source, scripts, modern shaders and documentation stay ordinary Git text.
|
| 2 |
+
* text=auto eol=lf filter=urt-code
|
| 3 |
+
# Every asset, including original textual shaders/configs, is an LFS pointer.
|
| 4 |
+
assets/** filter=lfs diff=lfs merge=lfs -text !eol
|
.githooks/pre-commit
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/sh
|
| 2 |
+
set -eu
|
| 3 |
+
exec python3 scripts/storage_policy.py check --require-setup
|
.githooks/pre-push
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/sh
|
| 2 |
+
set -eu
|
| 3 |
+
updates=$(mktemp)
|
| 4 |
+
trap 'rm -f "$updates"' EXIT HUP INT TERM
|
| 5 |
+
cat > "$updates"
|
| 6 |
+
python3 scripts/storage_policy.py check --require-setup
|
| 7 |
+
python3 scripts/storage_policy.py push --require-setup < "$updates"
|
| 8 |
+
git lfs pre-push "$@" < "$updates"
|
.gitignore
CHANGED
|
@@ -1 +1,9 @@
|
|
| 1 |
/.cache/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
/.cache/
|
| 2 |
+
/target/
|
| 3 |
+
/.work/
|
| 4 |
+
/.direnv/
|
| 5 |
+
__pycache__/
|
| 6 |
+
*.pyc
|
| 7 |
+
.DS_Store
|
| 8 |
+
.env
|
| 9 |
+
.env.*
|
assets/README.md
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2f2d302ee6bed775d99b82459dbe4e9558c6d86621f1031ce4d8f901fcc2f460
|
| 3 |
+
size 544
|
docs/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
|
| 3 |
Planning snapshot: **2026-10-10**. Reference installation: `.cache/UrbanTerror43/`.
|
| 4 |
|
| 5 |
-
The objective is to reverse engineer and document the supplied game, then rewrite its client and dedicated server in Rust with Bevy and Solari, running on macOS and Linux.
|
| 6 |
|
| 7 |
## Reading order
|
| 8 |
|
|
@@ -15,6 +15,7 @@ The objective is to reverse engineer and document the supplied game, then rewrit
|
|
| 15 |
7. [Research ledger](RESEARCH_LEDGER.md): evidence conventions and templates for documenting subsequent discoveries.
|
| 16 |
8. [Inventory evidence](inventory/README.md): machine-readable manifests, hashes, formats, maps, overrides, and case collisions.
|
| 17 |
9. [Every installation file checklist](FILE_COVERAGE_TODO.md): explicit coverage of all 39 files, including every archive-member variant beneath the PK3 entries.
|
|
|
|
| 18 |
|
| 19 |
## How to use the checklist
|
| 20 |
|
|
|
|
| 2 |
|
| 3 |
Planning snapshot: **2026-10-10**. Reference installation: `.cache/UrbanTerror43/`.
|
| 4 |
|
| 5 |
+
The objective is to reverse engineer and document the supplied game, then rewrite its client and dedicated server in Rust with Bevy and Solari, running on macOS and Linux. The planning pass produced documentation and static inventory evidence. Repository storage setup now also includes Git/LFS/Xet filters and commit/push guards, described in [STORAGE.md](STORAGE.md). The game, converters, and research tools remain TODO; the original game binaries have not been launched.
|
| 6 |
|
| 7 |
## Reading order
|
| 8 |
|
|
|
|
| 15 |
7. [Research ledger](RESEARCH_LEDGER.md): evidence conventions and templates for documenting subsequent discoveries.
|
| 16 |
8. [Inventory evidence](inventory/README.md): machine-readable manifests, hashes, formats, maps, overrides, and case collisions.
|
| 17 |
9. [Every installation file checklist](FILE_COVERAGE_TODO.md): explicit coverage of all 39 files, including every archive-member variant beneath the PK3 entries.
|
| 18 |
+
10. [Git/Xet storage setup](STORAGE.md): one Hugging Face repository, ordinary Git code, Xet assets, and pointer/history validation.
|
| 19 |
|
| 20 |
## How to use the checklist
|
| 21 |
|
docs/STORAGE.md
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Ordinary Git code and Xet assets in one repository
|
| 2 |
+
|
| 3 |
+
The project uses one Hugging Face repository: [nmarshall/urt-rs](https://huggingface.co/nmarshall/urt-rs), SSH remote `hf` at `git@hf.co:nmarshall/urt-rs`. Source and documentation are ordinary Git text; every file beneath `assets/` is represented by a canonical Git LFS pointer, with payload transfers handled by Git Xet. A single commit identifies both code and assets. There is no separate dataset, nested asset repository, submodule, or asset revision lock.
|
| 4 |
+
|
| 5 |
+
## Layout
|
| 6 |
+
|
| 7 |
+
```text
|
| 8 |
+
urban-terror-rs/
|
| 9 |
+
.gitattributes # Ordinary text by default; assets/** uses LFS
|
| 10 |
+
.githooks/ # Index/history guards and LFS upload hook
|
| 11 |
+
scripts/ # Setup and storage-policy code, ordinary Git
|
| 12 |
+
docs/ # Findings, plans and text inventories, ordinary Git
|
| 13 |
+
assets/
|
| 14 |
+
README.md # Also an LFS pointer in Git
|
| 15 |
+
original/UrbanTerror43/ # All 39 byte-identical installation files
|
| 16 |
+
extracted/ # Future per-archive content and variants
|
| 17 |
+
converted/ # Future Bevy/Solari runtime content
|
| 18 |
+
captures/ # Future media, traces and reference fixtures
|
| 19 |
+
.cache/ # Ignored originals, caches and scratch work
|
| 20 |
+
```
|
| 21 |
+
|
| 22 |
+
The initial installation copies are verified against `docs/inventory/snapshot.json`. The original `.cache/UrbanTerror43/` remains intact, preserving the evidence paths used in the research documents. Empty future output directories become tracked when the pipeline writes content. The retired, unpublished nested checkout's Git metadata is preserved under ignored `.cache/retired-assets-checkout/`; it is no longer an active repository or dependency.
|
| 23 |
+
|
| 24 |
+
## Routing and enforcement
|
| 25 |
+
|
| 26 |
+
The committed attributes are:
|
| 27 |
+
|
| 28 |
+
```gitattributes
|
| 29 |
+
* text=auto eol=lf filter=urt-code
|
| 30 |
+
assets/** filter=lfs diff=lfs merge=lfs -text !eol
|
| 31 |
+
```
|
| 32 |
+
|
| 33 |
+
`urt-code` is a required validation/pass-through clean filter. Source, scripts, modern WGSL shader source, documentation and readable inventory manifests remain literal UTF-8 Git blobs. The filter rejects known game/media/archive/binary extensions, asset/build/cache directories, NUL bytes, non-UTF-8 content, LFS pointers outside `assets/`, and ordinary Git files larger than 2 MiB. Original shaders/configs, archives, maps, models, textures, sounds, executables and libraries belong under `assets/`, including small textual assets. Content classification uses paths, known formats and encoding; newly discovered asset formats must extend the policy before ingestion.
|
| 34 |
+
|
| 35 |
+
**Every asset file uses LFS, with no metadata exceptions.** Git stores only the pointer's SHA-256 and size. The working tree retains the actual bytes. `filter=lfs` is the supported Git filter; `git-xet` is the locally registered custom transfer agent. Hugging Face negotiates Xet during upload/download, as described in its [Git/Xet integration documentation](https://huggingface.co/docs/hub/xet/using-xet-storage).
|
| 36 |
+
|
| 37 |
+
The pre-commit guard validates staged blobs and staged attributes, rejecting raw asset bytes, incorrect routing, unresolved entries, symlinks and nested Git repositories. It also requires local LFS/Xet/filter configuration. The pre-push hook first checks the index and every newly reachable commit, including payloads deleted by later commits, then invokes `git lfs pre-push` with the original ref updates. Payloads are uploaded before Git references are updated. Existing inherited documentation commits are checked for content; current staged attributes are enforced for future commits.
|
| 38 |
+
|
| 39 |
+
Hooks and local drivers are installed by setup; Git clones do not install these automatically. Anyone deliberately disabling hooks/configuration can bypass local protections. The committed filters and tests provide the normal workflow's enforcement; run the same checks in future CI/server protections when available. No global Git settings are modified.
|
| 40 |
+
|
| 41 |
+
## Clone and configure
|
| 42 |
+
|
| 43 |
+
Requirements: Git, Python 3, Git LFS, `git-xet`, and Hugging Face SSH access for remote operations.
|
| 44 |
+
|
| 45 |
+
```sh
|
| 46 |
+
GIT_LFS_SKIP_SMUDGE=1 git clone git@hf.co:nmarshall/urt-rs
|
| 47 |
+
cd urt-rs
|
| 48 |
+
python3 scripts/setup-repository.py
|
| 49 |
+
git lfs pull
|
| 50 |
+
git lfs fsck
|
| 51 |
+
python3 scripts/storage_policy.py check --require-setup
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
A clone calls the remote `origin`; this existing checkout calls it `hf`. Substitute the configured remote in push commands. Setup is repeatable and installs repository-local filters, Xet transfer configuration and the committed hooks. It refuses to replace an unrelated existing hooks directory.
|
| 55 |
+
|
| 56 |
+
## Change, verify and publish
|
| 57 |
+
|
| 58 |
+
```sh
|
| 59 |
+
python3 scripts/setup-repository.py
|
| 60 |
+
python3 scripts/test_storage_policy.py
|
| 61 |
+
# Write source under src/scripts/docs, and game content under assets/.
|
| 62 |
+
git add .
|
| 63 |
+
python3 scripts/storage_policy.py check --require-setup
|
| 64 |
+
git lfs ls-files
|
| 65 |
+
git commit -m "Describe the change"
|
| 66 |
+
git lfs fsck
|
| 67 |
+
git push hf main
|
| 68 |
+
```
|
| 69 |
+
|
| 70 |
+
The normal push hook uploads through LFS/Xet. `git show HEAD:assets/path/to/file` must show a small `version https://git-lfs.github.com/spec/v1` pointer, while opening that file in the working tree shows its actual content. Source files must show their source text directly in Git. Neither `.cache/` nor raw asset payloads belong in ordinary Git history.
|
| 71 |
+
|
| 72 |
+
If an asset was staged without LFS, restore setup/attributes, run `git add --renormalize assets`, and validate again before committing. If raw payloads have already been committed in unpublished history, repair those commits before pushing; merely removing the file in a later commit will still fail the history guard. Preserve the published history when correcting an existing remote.
|
| 73 |
+
|
| 74 |
+
## Planning scope
|
| 75 |
+
|
| 76 |
+
This storage setup does not implement the game or conversion pipeline. The exhaustive research, unpacking, recoding, conversion, rendering and Rust/Bevy/Solari rewrite tasks remain in the planning documents. Future evidence media and converted assets use this same `assets/` policy; future code and textual findings remain ordinary Git.
|
scripts/setup-repository.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Install the committed storage policy in this checkout's local Git config."""
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
import shlex
|
| 5 |
+
import subprocess
|
| 6 |
+
import sys
|
| 7 |
+
|
| 8 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def main():
|
| 12 |
+
if not (ROOT / ".git").exists():
|
| 13 |
+
raise SystemExit("Run setup in a Git checkout, not a Nix store source copy.")
|
| 14 |
+
def git(*args):
|
| 15 |
+
return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip()
|
| 16 |
+
existing = subprocess.run(["git", "-C", str(ROOT), "config", "--local", "--get", "core.hooksPath"],
|
| 17 |
+
text=True, capture_output=True).stdout.strip()
|
| 18 |
+
if existing and existing != ".githooks":
|
| 19 |
+
raise SystemExit(f"Existing hooksPath {existing!r}: integrate it before replacing hooks.")
|
| 20 |
+
git("lfs", "version")
|
| 21 |
+
git("xet", "--version")
|
| 22 |
+
git("lfs", "install", "--local", "--skip-repo")
|
| 23 |
+
git("xet", "install", "--local")
|
| 24 |
+
git("config", "--local", "lfs.basictransfersonly", "false")
|
| 25 |
+
python = shlex.quote(sys.executable)
|
| 26 |
+
script = shlex.quote(str(ROOT / "scripts/storage_policy.py"))
|
| 27 |
+
# Git shell-quotes its %f substitution. Keep it as a distinct argument.
|
| 28 |
+
git("config", "--local", "filter.urt-code.clean", f"{python} {script} clean -- %f")
|
| 29 |
+
git("config", "--local", "filter.urt-code.smudge", "cat")
|
| 30 |
+
git("config", "--local", "filter.urt-code.required", "true")
|
| 31 |
+
git("config", "--local", "core.hooksPath", ".githooks")
|
| 32 |
+
for name in ("pre-commit", "pre-push"):
|
| 33 |
+
path = ROOT / ".githooks" / name
|
| 34 |
+
path.chmod(path.stat().st_mode | 0o111)
|
| 35 |
+
print("Installed local code/LFS filters, Xet transfers and commit/push guards.")
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
if __name__ == "__main__":
|
| 39 |
+
main()
|
scripts/storage_policy.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Fail-closed checks for code Git objects and asset LFS pointers."""
|
| 3 |
+
import argparse
|
| 4 |
+
import os
|
| 5 |
+
from pathlib import PurePosixPath
|
| 6 |
+
import re
|
| 7 |
+
import subprocess
|
| 8 |
+
import sys
|
| 9 |
+
|
| 10 |
+
MAX_CODE_BYTES = 2 * 1024 * 1024
|
| 11 |
+
PAYLOAD_SUFFIXES = frozenset(
|
| 12 |
+
".png .jpg .jpeg .webp .gif .bmp .tga .tif .tiff .exr .hdr .svg .ico "
|
| 13 |
+
".dds .ktx .ktx2 .texture .glb .gltf .obj .fbx .blend .mesh "
|
| 14 |
+
".wav .ogg .mp3 .flac .bank .mp4 .webm .mov .ttf .otf .woff .woff2 "
|
| 15 |
+
".exe .dll .pdb .so .dylib .a .lib .bin .spv .snapshot .cube "
|
| 16 |
+
".zip .7z .rar .tar .gz .xz .zst .pdf .doc .docx .pptx .xlsx "
|
| 17 |
+
".pk3 .bsp .aas .md3 .md4 .mdc .mds .md5mesh .md5anim .iqm .ase "
|
| 18 |
+
".qvm .roq .icns .dat .map .prt .srf .rad .skin .shader .arena .bot .voice .crosshair".split()
|
| 19 |
+
)
|
| 20 |
+
PAYLOAD_DIRS = frozenset({"assets", "assets-src", "compiled-assets", "compiled-shaders",
|
| 21 |
+
".cache", ".work", "target", "captures"})
|
| 22 |
+
POINTER = re.compile(rb"version https://git-lfs.github.com/spec/v1\noid sha256:[0-9a-f]{64}\nsize [0-9]+\n")
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def git(*args, input=None):
|
| 26 |
+
return subprocess.check_output(["git", *args], input=input)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def check_code(path, data):
|
| 30 |
+
name = PurePosixPath(path)
|
| 31 |
+
if any(part.lower() in PAYLOAD_DIRS for part in name.parts):
|
| 32 |
+
raise ValueError(f"{path}: asset/build/cache directory is not allowed in code Git")
|
| 33 |
+
if name.suffix.lower() in PAYLOAD_SUFFIXES or ".so." in name.name.lower():
|
| 34 |
+
raise ValueError(f"{path}: asset or binary extension belongs under assets/")
|
| 35 |
+
if len(data) > MAX_CODE_BYTES:
|
| 36 |
+
raise ValueError(f"{path}: exceeds the {MAX_CODE_BYTES}-byte code limit")
|
| 37 |
+
if b"\0" in data:
|
| 38 |
+
raise ValueError(f"{path}: binary data belongs under assets/")
|
| 39 |
+
try:
|
| 40 |
+
data.decode("utf-8")
|
| 41 |
+
except UnicodeDecodeError as exc:
|
| 42 |
+
raise ValueError(f"{path}: code/documentation must be UTF-8 text") from exc
|
| 43 |
+
if data.startswith(b"version https://git-lfs.github.com/spec/v1\n"):
|
| 44 |
+
raise ValueError(f"{path}: LFS pointers belong under assets/")
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def check_asset(path, data):
|
| 48 |
+
if not POINTER.fullmatch(data):
|
| 49 |
+
raise ValueError(f"{path}: asset Git object is not a canonical LFS pointer; run setup and git add --renormalize")
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def is_asset(path):
|
| 53 |
+
return PurePosixPath(path).parts[0] == "assets"
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def validate_blob(path, oid):
|
| 57 |
+
size = int(git("cat-file", "-s", oid))
|
| 58 |
+
limit = 256 if is_asset(path) else MAX_CODE_BYTES
|
| 59 |
+
if size > limit:
|
| 60 |
+
raise ValueError(f"{path}: Git blob is {size} bytes; limit is {limit}")
|
| 61 |
+
data = git("cat-file", "blob", oid)
|
| 62 |
+
(check_asset if is_asset(path) else check_code)(path, data)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def check_index():
|
| 66 |
+
paths = []
|
| 67 |
+
for entry in git("ls-files", "--stage", "-z").split(b"\0"):
|
| 68 |
+
if not entry:
|
| 69 |
+
continue
|
| 70 |
+
header, path_bytes = entry.split(b"\t", 1)
|
| 71 |
+
permissions, oid, stage = header.decode().split()
|
| 72 |
+
path = os.fsdecode(path_bytes)
|
| 73 |
+
if stage != "0" or permissions not in {"100644", "100755"}:
|
| 74 |
+
raise ValueError(f"{path}: unresolved, symlink or nested repository entry is not allowed")
|
| 75 |
+
validate_blob(path, oid)
|
| 76 |
+
paths.append(path)
|
| 77 |
+
if paths:
|
| 78 |
+
raw = git("check-attr", "--cached", "-z", "--stdin", "filter",
|
| 79 |
+
input=b"\0".join(os.fsencode(p) for p in paths) + b"\0").split(b"\0")
|
| 80 |
+
for i in range(0, len(raw) - 1, 3):
|
| 81 |
+
path, _, value = raw[i:i + 3]
|
| 82 |
+
path = os.fsdecode(path)
|
| 83 |
+
expected = b"lfs" if is_asset(path) else b"urt-code"
|
| 84 |
+
if value != expected:
|
| 85 |
+
raise ValueError(f"{path}: expected filter={expected.decode()}, found {value.decode()}")
|
| 86 |
+
asset_paths = [p for p in paths if is_asset(p)]
|
| 87 |
+
if asset_paths:
|
| 88 |
+
raw = git("check-attr", "--cached", "-z", "--stdin", "text", "diff", "merge",
|
| 89 |
+
input=b"\0".join(os.fsencode(p) for p in asset_paths) + b"\0").split(b"\0")
|
| 90 |
+
for i in range(0, len(raw) - 1, 3):
|
| 91 |
+
path, attr, value = raw[i:i + 3]
|
| 92 |
+
expected = b"unset" if attr == b"text" else b"lfs"
|
| 93 |
+
if value != expected:
|
| 94 |
+
raise ValueError(f"{os.fsdecode(path)}: incorrect asset {attr.decode()} attribute")
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def check_push():
|
| 98 |
+
# Validate all newly reachable commits, including files deleted later.
|
| 99 |
+
# A new remote branch checks its entire reachable history.
|
| 100 |
+
seen = set()
|
| 101 |
+
for line in sys.stdin:
|
| 102 |
+
_, local_oid, _, remote_oid = line.split()
|
| 103 |
+
if set(local_oid) == {"0"}:
|
| 104 |
+
continue
|
| 105 |
+
revisions = [local_oid]
|
| 106 |
+
if set(remote_oid) != {"0"}:
|
| 107 |
+
exists = subprocess.run(["git", "cat-file", "-e", remote_oid],
|
| 108 |
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
| 109 |
+
if exists.returncode == 0:
|
| 110 |
+
revisions.append("^" + remote_oid)
|
| 111 |
+
for commit in git("rev-list", *revisions).decode().splitlines():
|
| 112 |
+
for entry in git("ls-tree", "-rz", commit).split(b"\0"):
|
| 113 |
+
if not entry:
|
| 114 |
+
continue
|
| 115 |
+
header, path = entry.split(b"\t", 1)
|
| 116 |
+
permissions, kind, oid = header.decode().split()
|
| 117 |
+
if permissions not in {"100644", "100755"} or kind != "blob":
|
| 118 |
+
raise ValueError(f"{os.fsdecode(path)}: unsupported Git tree entry")
|
| 119 |
+
key = (os.fsdecode(path), oid)
|
| 120 |
+
if key not in seen:
|
| 121 |
+
validate_blob(*key)
|
| 122 |
+
seen.add(key)
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def check_setup():
|
| 126 |
+
required = {
|
| 127 |
+
"filter.urt-code.required": "true",
|
| 128 |
+
"filter.lfs.required": "true",
|
| 129 |
+
"lfs.customtransfer.xet.path": "git-xet",
|
| 130 |
+
"lfs.customtransfer.xet.args": "transfer",
|
| 131 |
+
"lfs.basictransfersonly": "false",
|
| 132 |
+
"core.hooksPath": ".githooks",
|
| 133 |
+
}
|
| 134 |
+
for key, value in required.items():
|
| 135 |
+
if git("config", "--local", "--get", key).decode().strip() != value:
|
| 136 |
+
raise ValueError(f"{key}: rerun scripts/setup-repository.py")
|
| 137 |
+
for key in ("filter.urt-code.clean", "filter.lfs.clean", "filter.lfs.process"):
|
| 138 |
+
if not git("config", "--local", "--get", key).strip():
|
| 139 |
+
raise ValueError(f"{key}: missing filter driver")
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def main():
|
| 143 |
+
parser = argparse.ArgumentParser()
|
| 144 |
+
parser.add_argument("command", choices=["clean", "check", "push"])
|
| 145 |
+
parser.add_argument("--require-setup", action="store_true")
|
| 146 |
+
parser.add_argument("path", nargs="?")
|
| 147 |
+
args = parser.parse_args()
|
| 148 |
+
try:
|
| 149 |
+
if args.require_setup:
|
| 150 |
+
check_setup()
|
| 151 |
+
if args.command == "clean":
|
| 152 |
+
data = sys.stdin.buffer.read(MAX_CODE_BYTES + 1)
|
| 153 |
+
check_code(args.path or "<stdin>", data)
|
| 154 |
+
sys.stdout.buffer.write(data)
|
| 155 |
+
elif args.command == "push":
|
| 156 |
+
check_push()
|
| 157 |
+
else:
|
| 158 |
+
check_index()
|
| 159 |
+
except (ValueError, subprocess.CalledProcessError) as exc:
|
| 160 |
+
print(f"Storage policy: {exc}", file=sys.stderr)
|
| 161 |
+
return 1
|
| 162 |
+
return 0
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
if __name__ == "__main__":
|
| 166 |
+
sys.exit(main())
|
scripts/test_storage_policy.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Storage-routing tests using disposable repositories, without network access."""
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
import os
|
| 5 |
+
import shlex
|
| 6 |
+
import subprocess
|
| 7 |
+
import sys
|
| 8 |
+
import tempfile
|
| 9 |
+
import unittest
|
| 10 |
+
|
| 11 |
+
import storage_policy as policy
|
| 12 |
+
|
| 13 |
+
POLICY = Path(policy.__file__).resolve()
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class ContentPolicyTests(unittest.TestCase):
|
| 17 |
+
def test_code_and_readable_shader_source(self):
|
| 18 |
+
for name in ["src/main.rs", "shaders/wall.wgsl", "docs/evidence/files.csv", "config.ron"]:
|
| 19 |
+
policy.check_code(name, "// source and metadata ✓\n".encode())
|
| 20 |
+
|
| 21 |
+
def test_payload_paths_even_for_text_assets(self):
|
| 22 |
+
for name in ["assets/mesh.json", "crates/app/assets/settings.ron", ".cache/foo.txt", "captures/frame.txt"]:
|
| 23 |
+
with self.assertRaises(ValueError, msg=name):
|
| 24 |
+
policy.check_code(name, b"{}")
|
| 25 |
+
|
| 26 |
+
def test_payload_extensions_outside_asset_directory(self):
|
| 27 |
+
for name in ["foo.PNG", "src/model.glb", "texture.texture", "docs/example.svg", "plugin.so.1",
|
| 28 |
+
"game.pk3", "cgame.qvm", "map.bsp", "map.aas", "player.MD3",
|
| 29 |
+
"original.shader", "library.dylib", "app.icns", "font.dat"]:
|
| 30 |
+
with self.assertRaises(ValueError, msg=name):
|
| 31 |
+
policy.check_code(name, b"innocent looking text")
|
| 32 |
+
|
| 33 |
+
def test_binary_invalid_encoding_and_oversize(self):
|
| 34 |
+
for data in [b"a\0b", b"\xff", b"x" * (policy.MAX_CODE_BYTES + 1)]:
|
| 35 |
+
with self.assertRaises(ValueError):
|
| 36 |
+
policy.check_code("src/data.rs", data)
|
| 37 |
+
|
| 38 |
+
def test_asset_pointer_is_required(self):
|
| 39 |
+
pointer = b"version https://git-lfs.github.com/spec/v1\noid sha256:" + b"a" * 64 + b"\nsize 10\n"
|
| 40 |
+
policy.check_asset("mesh.json", pointer)
|
| 41 |
+
policy.check_asset("assets/README.md", pointer)
|
| 42 |
+
for data in [b"{}", pointer + b"hidden payload", b""]:
|
| 43 |
+
with self.assertRaises(ValueError):
|
| 44 |
+
policy.check_asset("mesh.json", data)
|
| 45 |
+
with self.assertRaises(ValueError):
|
| 46 |
+
policy.check_code("data.txt", pointer)
|
| 47 |
+
|
| 48 |
+
def test_nested_asset_readme_is_not_control_metadata(self):
|
| 49 |
+
with self.assertRaises(ValueError):
|
| 50 |
+
policy.check_asset("original/README.md", b"original game text\n")
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class GitIntegrationTests(unittest.TestCase):
|
| 54 |
+
def setUp(self):
|
| 55 |
+
self.temp = tempfile.TemporaryDirectory()
|
| 56 |
+
self.root = Path(self.temp.name)
|
| 57 |
+
self.env = {**os.environ, "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull,
|
| 58 |
+
"GIT_AUTHOR_NAME": "Test", "GIT_AUTHOR_EMAIL": "test@example.invalid",
|
| 59 |
+
"GIT_COMMITTER_NAME": "Test", "GIT_COMMITTER_EMAIL": "test@example.invalid"}
|
| 60 |
+
# Do not inherit a parent hook's repository/index environment.
|
| 61 |
+
for key in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR"):
|
| 62 |
+
self.env.pop(key, None)
|
| 63 |
+
self.git("init", "-q")
|
| 64 |
+
|
| 65 |
+
def tearDown(self):
|
| 66 |
+
self.temp.cleanup()
|
| 67 |
+
|
| 68 |
+
def run_command(self, *args, input=None, success=True):
|
| 69 |
+
result = subprocess.run(args, cwd=self.root, env=self.env, input=input,
|
| 70 |
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
| 71 |
+
if success:
|
| 72 |
+
self.assertEqual(result.returncode, 0, result.stderr.decode(errors="replace"))
|
| 73 |
+
else:
|
| 74 |
+
self.assertNotEqual(result.returncode, 0)
|
| 75 |
+
return result
|
| 76 |
+
|
| 77 |
+
def git(self, *args, **kwargs):
|
| 78 |
+
return self.run_command("git", *args, **kwargs)
|
| 79 |
+
|
| 80 |
+
def install_code_filter(self):
|
| 81 |
+
(self.root / ".gitattributes").write_text((POLICY.parent.parent / ".gitattributes").read_text())
|
| 82 |
+
command = f"{shlex.quote(sys.executable)} {shlex.quote(str(POLICY))} clean -- %f"
|
| 83 |
+
self.git("config", "filter.urt-code.clean", command)
|
| 84 |
+
self.git("config", "filter.urt-code.smudge", "cat")
|
| 85 |
+
self.git("config", "filter.urt-code.required", "true")
|
| 86 |
+
self.git("add", ".gitattributes")
|
| 87 |
+
|
| 88 |
+
def test_git_add_preserves_source_and_quoted_filename(self):
|
| 89 |
+
self.install_code_filter()
|
| 90 |
+
name = "space ' and dollar $.rs"
|
| 91 |
+
data = b"fn main() {}\n"
|
| 92 |
+
(self.root / name).write_bytes(data)
|
| 93 |
+
self.git("add", "--", name)
|
| 94 |
+
self.assertEqual(self.git("show", ":" + name).stdout, data)
|
| 95 |
+
self.run_command(sys.executable, str(POLICY), "check")
|
| 96 |
+
|
| 97 |
+
def test_clean_filter_rejects_asset_and_missing_driver(self):
|
| 98 |
+
self.install_code_filter()
|
| 99 |
+
(self.root / "image.png").write_bytes(b"small")
|
| 100 |
+
self.git("add", "image.png", success=False)
|
| 101 |
+
(self.root / "main.rs").write_text("fn main() {}\n")
|
| 102 |
+
self.git("config", "filter.urt-code.clean", "missing-urt-filter-command")
|
| 103 |
+
self.git("add", "main.rs", success=False)
|
| 104 |
+
|
| 105 |
+
def test_index_guard_catches_bypassed_filter_and_attribute_override(self):
|
| 106 |
+
self.install_code_filter()
|
| 107 |
+
(self.root / "data.txt").write_bytes(b"binary\0data")
|
| 108 |
+
self.git("-c", "filter.urt-code.clean=cat", "add", "data.txt")
|
| 109 |
+
self.run_command(sys.executable, str(POLICY), "check", success=False)
|
| 110 |
+
self.git("-c", "filter.urt-code.clean=cat", "rm", "--cached", "data.txt")
|
| 111 |
+
(self.root / ".gitattributes").write_text("* -filter\n")
|
| 112 |
+
self.git("add", ".gitattributes")
|
| 113 |
+
self.run_command(sys.executable, str(POLICY), "check", success=False)
|
| 114 |
+
|
| 115 |
+
def test_push_guard_catches_payload_deleted_in_later_commit(self):
|
| 116 |
+
self.install_code_filter()
|
| 117 |
+
(self.root / "data.txt").write_bytes(b"binary\0data")
|
| 118 |
+
self.git("-c", "filter.urt-code.clean=cat", "add", "data.txt")
|
| 119 |
+
self.git("-c", "filter.urt-code.clean=cat", "commit", "-qm", "payload")
|
| 120 |
+
self.git("-c", "filter.urt-code.clean=cat", "rm", "data.txt")
|
| 121 |
+
self.git("commit", "-qm", "remove payload")
|
| 122 |
+
self.run_command(sys.executable, str(POLICY), "check")
|
| 123 |
+
oid = self.git("rev-parse", "HEAD").stdout.decode().strip()
|
| 124 |
+
update = f"refs/heads/main {oid} refs/heads/main {'0' * 40}\n".encode()
|
| 125 |
+
self.run_command(sys.executable, str(POLICY), "push", input=update, success=False)
|
| 126 |
+
|
| 127 |
+
def install_mixed_filters(self):
|
| 128 |
+
self.install_code_filter()
|
| 129 |
+
self.git("lfs", "install", "--local", "--skip-repo")
|
| 130 |
+
(self.root / "assets").mkdir()
|
| 131 |
+
|
| 132 |
+
def bypass_lfs(self):
|
| 133 |
+
self.git("config", "--unset", "filter.lfs.process")
|
| 134 |
+
self.git("config", "filter.lfs.clean", "cat")
|
| 135 |
+
|
| 136 |
+
def test_lfs_stores_text_asset_as_pointer_and_keeps_working_bytes(self):
|
| 137 |
+
self.install_mixed_filters()
|
| 138 |
+
data = b'{"positions": [1, 2, 3]}\n'
|
| 139 |
+
(self.root / "assets/mesh.json").write_bytes(data)
|
| 140 |
+
self.git("add", "assets/mesh.json")
|
| 141 |
+
pointer = self.git("show", ":assets/mesh.json").stdout
|
| 142 |
+
self.assertTrue(policy.POINTER.fullmatch(pointer))
|
| 143 |
+
self.assertEqual((self.root / "assets/mesh.json").read_bytes(), data)
|
| 144 |
+
self.run_command(sys.executable, str(POLICY), "check")
|
| 145 |
+
self.bypass_lfs()
|
| 146 |
+
self.git("add", "--renormalize", "assets/mesh.json")
|
| 147 |
+
self.run_command(sys.executable, str(POLICY), "check", success=False)
|
| 148 |
+
|
| 149 |
+
def test_actual_attributes_route_all_assets_and_keep_code_literal(self):
|
| 150 |
+
self.install_mixed_filters()
|
| 151 |
+
(self.root / "README.md").write_text("# Code repository\n")
|
| 152 |
+
(self.root / "assets/original").mkdir()
|
| 153 |
+
payloads = {
|
| 154 |
+
"assets/README.md": b"asset documentation\n",
|
| 155 |
+
"assets/original/README.md": b"original attribution\n",
|
| 156 |
+
"assets/original/default.cfg": b"seta snaps 20\n",
|
| 157 |
+
"assets/original/wall.shader": b"textures/wall { { map wall.tga } }\n",
|
| 158 |
+
"assets/original/game.exe": b"MZ\0game payload",
|
| 159 |
+
}
|
| 160 |
+
for name, data in payloads.items():
|
| 161 |
+
(self.root / name).write_bytes(data)
|
| 162 |
+
self.git("add", ".")
|
| 163 |
+
self.assertEqual(self.git("show", ":README.md").stdout, b"# Code repository\n")
|
| 164 |
+
for name, data in payloads.items():
|
| 165 |
+
pointer = self.git("show", ":" + name).stdout
|
| 166 |
+
self.assertTrue(policy.POINTER.fullmatch(pointer), name)
|
| 167 |
+
self.assertEqual((self.root / name).read_bytes(), data)
|
| 168 |
+
self.run_command(sys.executable, str(POLICY), "check")
|
| 169 |
+
|
| 170 |
+
def test_asset_history_rejects_payload_even_after_deletion(self):
|
| 171 |
+
self.install_mixed_filters()
|
| 172 |
+
self.bypass_lfs()
|
| 173 |
+
(self.root / "assets/config.txt").write_text("original game asset\n")
|
| 174 |
+
self.git("add", ".")
|
| 175 |
+
self.git("commit", "-qm", "raw asset")
|
| 176 |
+
self.git("rm", "assets/config.txt")
|
| 177 |
+
self.git("commit", "-qm", "remove asset")
|
| 178 |
+
self.run_command(sys.executable, str(POLICY), "check")
|
| 179 |
+
oid = self.git("rev-parse", "HEAD").stdout.decode().strip()
|
| 180 |
+
update = f"refs/heads/main {oid} refs/heads/main {'0' * 40}\n".encode()
|
| 181 |
+
self.run_command(sys.executable, str(POLICY), "push", input=update, success=False)
|
| 182 |
+
|
| 183 |
+
def test_nested_attributes_cannot_exempt_asset_payload(self):
|
| 184 |
+
self.install_mixed_filters()
|
| 185 |
+
(self.root / "assets/.gitattributes").write_text("* -filter\n")
|
| 186 |
+
(self.root / "assets/data.txt").write_text("game data\n")
|
| 187 |
+
self.git("add", ".")
|
| 188 |
+
self.run_command(sys.executable, str(POLICY), "check", success=False)
|
| 189 |
+
|
| 190 |
+
def test_symlink_is_rejected(self):
|
| 191 |
+
self.install_mixed_filters()
|
| 192 |
+
(self.root / "link").symlink_to("README.md")
|
| 193 |
+
self.git("add", "link")
|
| 194 |
+
self.run_command(sys.executable, str(POLICY), "check", success=False)
|
| 195 |
+
|
| 196 |
+
def test_missing_local_setup_is_rejected(self):
|
| 197 |
+
self.install_mixed_filters()
|
| 198 |
+
self.run_command(sys.executable, str(POLICY), "check", "--require-setup", success=False)
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
if __name__ == "__main__":
|
| 202 |
+
unittest.main()
|