Download app.py from misukisu/minecraft: direct link, hf CLI and curl.
- Browser
- Download file 18.6 kB
-
https://huggingface.co/spaces/misukisu/minecraft/resolve/main/app.py
- Command line
-
hf download hf://spaces/misukisu/minecraft/app.py
-
curl -L -o app.py https://huggingface.co/spaces/misukisu/minecraft/resolve/main/app.py
18.6 kB
| import asyncio, os, shutil, signal, subprocess, tarfile, time, datetime, json, logging, zipfile | |
| from pathlib import Path | |
| import aiohttp | |
| from aiohttp import web, WSMsgType | |
| from huggingface_hub import HfApi, CommitOperationDelete | |
| logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") | |
| log = logging.getLogger("sup") | |
| CRAFTY = Path("/crafty") | |
| APP = Path(__file__).parent | |
| DATA = Path(os.environ.get("DATA", "/data")) | |
| TMP = DATA / "tmp" | |
| REPO = os.environ.get("BACKUP_REPO", "misukisu/minecraft-backups") | |
| api = HfApi(token=os.environ.get("HF_TOKEN")) | |
| ADMIN_KEY = os.environ.get("RELAY_KEY", "") | |
| CRAFTY_USER = "admin" | |
| CRAFTY_PASSWORD = os.environ.get("CRAFTY_PASSWORD", "") | |
| PLAYIT_SECRET = os.environ.get("PLAYIT_SECRET", "").strip() | |
| SPACE_URL = "https://" + os.environ.get("SPACE_HOST", "misukisu-minecraft.hf.space") | |
| INTERVAL = int(os.environ.get("BACKUP_INTERVAL", "3600")) | |
| KEEP = int(os.environ.get("BACKUP_KEEP", "48")) | |
| UPSTREAM = "https://127.0.0.1:8443" | |
| SKIP = {"logs", "playitd.sock", "bedrock_server"} | |
| state = {"last_backup": None, "err": None, "restored": None, "tunnels": []} | |
| backup_lock = asyncio.Lock() | |
| http: aiohttp.ClientSession | |
| def list_backups(prefix): | |
| try: | |
| return sorted(f for f in api.list_repo_files(REPO, repo_type="dataset") | |
| if f.startswith("backups/" + prefix) and f.endswith(".tar.zst")) | |
| except Exception as e: | |
| log.warning("list failed: %s", e) | |
| return [] | |
| def extract(remote, dest): | |
| p = api.hf_hub_download(REPO, remote, repo_type="dataset", local_dir=str(TMP)) | |
| dest.mkdir(parents=True, exist_ok=True) | |
| subprocess.run(f"zstd -dc '{p}' | tar -xf - -C '{dest}'", shell=True, check=True) | |
| os.remove(p) | |
| def restore(): | |
| """Returns path of a legacy world dir to import, or None.""" | |
| TMP.mkdir(parents=True, exist_ok=True) | |
| api.create_repo(REPO, repo_type="dataset", private=True, exist_ok=True) | |
| c = list_backups("bedrock-") | |
| if c: | |
| log.info("restoring %s", c[-1]) | |
| extract(c[-1], CRAFTY) | |
| state["restored"] = c[-1] | |
| return None | |
| dest = DATA / "import-server" | |
| shutil.rmtree(dest, ignore_errors=True) | |
| dest.mkdir(parents=True, exist_ok=True) | |
| return dest | |
| BDS_CMD = "./bedrock_server" | |
| NETHERNET = {} | |
| PROPS = {"server-name": "Kreatix Minecraft", "server-port": "19132", "server-portv6": "19133", | |
| "level-name": "world", "online-mode": "true", "allow-list": "false", "server-ip": "0.0.0.0"} | |
| def playit_tunnels(): | |
| import urllib.request | |
| req = urllib.request.Request("https://api.playit.gg/agents/rundata", data=b"{}", method="POST", | |
| headers={"Authorization": f"Agent-Key {PLAYIT_SECRET}", "content-type": "application/json"}) | |
| with urllib.request.urlopen(req, timeout=15) as r: | |
| return (json.load(r).get("data") or {}).get("tunnels", []) | |
| def nethernet_props(): | |
| """Advertise the playit UDP tunnel as NetherNet's public gameplay endpoint.""" | |
| if os.environ.get("BDS_TRANSPORT", "nethernet") == "raknet": | |
| return {"transport": "raknet"} | |
| try: | |
| import socket | |
| for t in playit_tunnels(): | |
| if t.get("proto") == "udp" and t.get("local_port") == 19132: | |
| ip = socket.gethostbyname(t["assigned_domain"]) | |
| return {"transport": "nethernet", "server-udp-ports": f"{ip}:{t['port']['from']}:{t['local_port']}"} | |
| except Exception as e: | |
| log.warning("playit lookup failed: %s", e) | |
| return {"transport": "nethernet"} | |
| def set_props(srv: Path): | |
| PROPS.update(NETHERNET) | |
| props = srv / "server.properties" | |
| lines, seen = [], set() | |
| for l in (props.read_text().splitlines() if props.exists() else []): | |
| k = l.split("=", 1)[0].strip() | |
| if k in PROPS: | |
| l = f"{k}={PROPS[k]}" | |
| seen.add(k) | |
| lines.append(l) | |
| lines += [f"{k}={v}" for k, v in PROPS.items() if k not in seen] | |
| props.write_text("\n".join(lines) + "\n") | |
| def build_import_zip(src: Path) -> str: | |
| tpl = APP / "template" / "bedrock" | |
| for f in tpl.rglob("*"): | |
| rel = f.relative_to(tpl) | |
| if f.is_file() and not (src / rel).exists(): | |
| (src / rel).parent.mkdir(parents=True, exist_ok=True) | |
| shutil.copy2(f, src / rel) | |
| set_props(src) | |
| out = CRAFTY / "import" / "upload" / "server.zip" | |
| with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as z: | |
| for f in src.rglob("*"): | |
| if f.is_file(): | |
| z.write(f, f.relative_to(src)) | |
| return out.name | |
| def install_world(srv: Path, mcworld: Path): | |
| tmp = TMP / "world-in" | |
| shutil.rmtree(tmp, ignore_errors=True) | |
| with zipfile.ZipFile(mcworld) as z: | |
| z.extractall(tmp) | |
| root = next((d.parent for d in tmp.rglob("level.dat")), None) | |
| if root is None: | |
| raise ValueError("no level.dat in upload") | |
| dest = srv / "worlds" / "world" | |
| shutil.rmtree(dest, ignore_errors=True) | |
| dest.parent.mkdir(parents=True, exist_ok=True) | |
| shutil.move(str(root), dest) | |
| shutil.rmtree(tmp, ignore_errors=True) | |
| class Crafty: | |
| token = None | |
| async def req(self, method, path, retry=True, **kw): | |
| h = {"Authorization": f"Bearer {self.token}"} if self.token else {} | |
| async with http.request(method, UPSTREAM + path, headers=h, ssl=False, **kw) as r: | |
| body = await r.text() | |
| if r.status in (401, 403) and retry and CRAFTY_PASSWORD: | |
| await self.login(CRAFTY_PASSWORD) | |
| return await self.req(method, path, retry=False, **kw) | |
| try: | |
| return json.loads(body) | |
| except ValueError: | |
| return {"status": r.status, "raw": body[:300]} | |
| async def login(self, password): | |
| self.token = None | |
| r = await self.req("POST", "/api/v2/auth/login", retry=False, json={"username": CRAFTY_USER, "password": password}) | |
| self.token = (r.get("data") or {}).get("token") | |
| return r | |
| async def server_id(self): | |
| r = await self.req("GET", "/api/v2/servers") | |
| s = r.get("data") or [] | |
| return str(s[0]["server_id"]) if s else None | |
| async def cmd(self, c): | |
| sid = await self.server_id() | |
| if sid: | |
| await self.req("POST", f"/api/v2/servers/{sid}/stdin", data=c.encode()) | |
| crafty = Crafty() | |
| async def wait_crafty(): | |
| for _ in range(300): | |
| try: | |
| async with http.get(UPSTREAM + "/login", ssl=False) as r: | |
| if r.status < 500: | |
| return | |
| except Exception: | |
| pass | |
| await asyncio.sleep(2) | |
| async def bootstrap(import_dir): | |
| creds = CRAFTY / "app" / "config" / "default-creds.txt" | |
| pw = json.loads(creds.read_text())["password"] if creds.exists() else CRAFTY_PASSWORD | |
| r = await crafty.login(pw) | |
| uid = (r.get("data") or {}).get("user_id") | |
| if CRAFTY_PASSWORD and pw != CRAFTY_PASSWORD: | |
| log.info("set admin password: %s", await crafty.req("PATCH", f"/api/v2/users/{uid}", json={"password": CRAFTY_PASSWORD})) | |
| await crafty.login(CRAFTY_PASSWORD) | |
| creds.unlink(missing_ok=True) | |
| name = await asyncio.to_thread(build_import_zip, import_dir) | |
| r = await crafty.req("POST", "/api/v2/servers", json={ | |
| "name": "Kreatix", "monitoring_type": "minecraft_bedrock", "create_type": "minecraft_bedrock", | |
| "stop_command": "stop", | |
| "minecraft_bedrock_monitoring_data": {"host": "127.0.0.1", "port": 19132}, | |
| "minecraft_bedrock_create_data": {"create_type": "import_server", "import_server_create_data": { | |
| "archive_name": name, "archive_internal_path": "", "executable": "bedrock_server", | |
| "command": BDS_CMD}}}) | |
| log.info("create server: %s", r) | |
| sid = (r.get("data") or {}).get("new_server_id") | |
| archive = CRAFTY / "import" / "upload" / name | |
| for _ in range(300): | |
| await asyncio.sleep(3) | |
| if not archive.exists() and (CRAFTY / "servers" / sid / "bedrock_server").exists(): | |
| break | |
| await asyncio.sleep(10) | |
| srv = CRAFTY / "servers" / sid | |
| (srv / "bedrock_server").chmod(0o755) | |
| set_props(srv) | |
| cmdline = BDS_CMD | |
| log.info("patch: %s", await crafty.req("PATCH", f"/api/v2/servers/{sid}", json={"execution_command": cmdline, "auto_start": True, "crash_detection": True, "auto_start_delay": 3})) | |
| log.info("start: %s", await crafty.req("POST", f"/api/v2/servers/{sid}/action/start_server")) | |
| await backup("initial") | |
| def make_archive(path: Path): | |
| with open(path, "wb") as raw: | |
| z = subprocess.Popen(["zstd", "-q", "-T0", "-6", "-c"], stdin=subprocess.PIPE, stdout=raw) | |
| with tarfile.open(fileobj=z.stdin, mode="w|") as tar: | |
| def filt(ti): | |
| return None if set(Path(ti.name).parts) & SKIP else ti | |
| for d in ("app/config", "servers"): | |
| if (CRAFTY / d).exists(): | |
| tar.add(CRAFTY / d, arcname=d, filter=filt) | |
| z.stdin.close() | |
| z.wait() | |
| async def backup(reason="hourly"): | |
| async with backup_lock: | |
| name = "bedrock-" + datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S") | |
| path = TMP / f"{name}.tar.zst" | |
| TMP.mkdir(parents=True, exist_ok=True) | |
| try: | |
| await crafty.cmd("save hold") | |
| sid = await crafty.server_id() | |
| for _ in range(30): | |
| await asyncio.sleep(2) | |
| await crafty.cmd("save query") | |
| await asyncio.sleep(1) | |
| logs = (await crafty.req("GET", f"/api/v2/servers/{sid}/logs?raw=true")).get("data") or [] | |
| if any("ready to be copied" in l for l in logs[-5:]): | |
| break | |
| await asyncio.to_thread(make_archive, path) | |
| finally: | |
| await crafty.cmd("save resume") | |
| try: | |
| await asyncio.to_thread(api.upload_file, path_or_fileobj=str(path), path_in_repo=f"backups/{name}.tar.zst", | |
| repo_id=REPO, repo_type="dataset", commit_message=f"{reason} backup") | |
| state["last_backup"] = {"name": name, "at": time.time(), "mb": round(path.stat().st_size / 1e6, 1)} | |
| state["err"] = None | |
| log.info("backup %s ok", name) | |
| old = (await asyncio.to_thread(list_backups, ""))[:-KEEP] | |
| if old: | |
| await asyncio.to_thread(api.create_commit, repo_id=REPO, repo_type="dataset", | |
| operations=[CommitOperationDelete(f) for f in old], commit_message="prune") | |
| except Exception as e: | |
| state["err"] = str(e) | |
| log.exception("backup failed") | |
| finally: | |
| path.unlink(missing_ok=True) | |
| async def save_identity(): | |
| sid = await crafty.server_id() | |
| if sid and not (CRAFTY / "servers" / sid / "keys" / "server_identity_key.pem").exists(): | |
| await asyncio.sleep(60) | |
| await crafty.cmd("serveridentity save") | |
| async def backup_loop(): | |
| while True: | |
| await asyncio.sleep(INTERVAL) | |
| await backup() | |
| async def run_playit(): | |
| if not PLAYIT_SECRET: | |
| return | |
| while True: | |
| p = await asyncio.create_subprocess_exec("playitd", "--secret", PLAYIT_SECRET, "--platform-docker", | |
| "--socket-path", str(DATA / "playitd.sock"), | |
| stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT) | |
| async for line in p.stdout: | |
| print("[playit]", line.decode(errors="replace").rstrip(), flush=True) | |
| await p.wait() | |
| await asyncio.sleep(5) | |
| async def tunnels_loop(): | |
| while True: | |
| try: | |
| async with http.post("https://api.playit.gg/agents/rundata", json={}, | |
| headers={"Authorization": f"Agent-Key {PLAYIT_SECRET}"}) as r: | |
| d = (await r.json()).get("data") or {} | |
| state["tunnels"] = [{"type": t.get("tunnel_type"), "addr": f"{t.get('assigned_domain')}:{t['port']['from']}"} | |
| for t in d.get("tunnels", [])] | |
| except Exception: | |
| pass | |
| try: | |
| async with http.get(SPACE_URL + "/health") as r: | |
| await r.read() | |
| except Exception: | |
| pass | |
| await asyncio.sleep(300) | |
| HOP = {"connection", "keep-alive", "transfer-encoding", "upgrade", "content-length", "content-encoding", "te", "trailer", "proxy-authorization"} | |
| async def proxy(request: web.Request): | |
| url = UPSTREAM + request.path_qs | |
| headers = {k: v for k, v in request.headers.items() if k.lower() not in HOP} | |
| if request.headers.get("upgrade", "").lower() == "websocket": | |
| ws = web.WebSocketResponse(protocols=request.headers.get("Sec-WebSocket-Protocol", "").split(", ") if request.headers.get("Sec-WebSocket-Protocol") else ()) | |
| await ws.prepare(request) | |
| h = {k: v for k, v in headers.items() if not k.lower().startswith("sec-websocket")} | |
| async with http.ws_connect(url, headers=h, ssl=False, max_msg_size=0) as up: | |
| async def a(): | |
| async for m in ws: | |
| if m.type == WSMsgType.TEXT: | |
| await up.send_str(m.data) | |
| elif m.type == WSMsgType.BINARY: | |
| await up.send_bytes(m.data) | |
| await up.close() | |
| async def b(): | |
| async for m in up: | |
| if m.type == WSMsgType.TEXT: | |
| await ws.send_str(m.data) | |
| elif m.type == WSMsgType.BINARY: | |
| await ws.send_bytes(m.data) | |
| await ws.close() | |
| await asyncio.gather(a(), b(), return_exceptions=True) | |
| return ws | |
| body = await request.read() | |
| async with http.request(request.method, url, headers=headers, data=body or None, ssl=False, allow_redirects=False) as r: | |
| resp = web.StreamResponse(status=r.status) | |
| for k, v in r.headers.items(): | |
| if k.lower() in HOP: | |
| continue | |
| if k.lower() == "location": | |
| v = v.replace(UPSTREAM, "").replace("https://127.0.0.1:8443", "") | |
| resp.headers.add(k, v) | |
| await resp.prepare(request) | |
| async for chunk in r.content.iter_chunked(65536): | |
| await resp.write(chunk) | |
| await resp.write_eof() | |
| return resp | |
| async def status(request): | |
| return web.json_response(state) | |
| def admin(request): | |
| if not ADMIN_KEY or request.headers.get("authorization") != f"Bearer {ADMIN_KEY}": | |
| raise web.HTTPForbidden() | |
| async def do_backup(request): | |
| admin(request) | |
| await backup("manual") | |
| return web.json_response(state) | |
| async def upload_world(request): | |
| admin(request) | |
| sid = await crafty.server_id() | |
| srv = CRAFTY / "servers" / sid | |
| f = TMP / "upload.mcworld" | |
| TMP.mkdir(parents=True, exist_ok=True) | |
| with open(f, "wb") as out: | |
| async for chunk in request.content.iter_chunked(1 << 20): | |
| out.write(chunk) | |
| await crafty.req("POST", f"/api/v2/servers/{sid}/action/stop_server") | |
| for _ in range(60): | |
| await asyncio.sleep(2) | |
| if not ((await crafty.req("GET", f"/api/v2/servers/{sid}/stats")).get("data") or {}).get("running"): | |
| break | |
| try: | |
| await asyncio.to_thread(install_world, srv, f) | |
| finally: | |
| f.unlink(missing_ok=True) | |
| await crafty.req("POST", f"/api/v2/servers/{sid}/action/start_server") | |
| await asyncio.sleep(15) | |
| await backup("world-upload") | |
| return web.json_response(state) | |
| async def tcp_bridge(request): | |
| admin(request) | |
| ws = web.WebSocketResponse(max_msg_size=0) | |
| await ws.prepare(request) | |
| try: | |
| r, w = await asyncio.open_connection("127.0.0.1", 19132) | |
| except OSError: | |
| await ws.close() | |
| return ws | |
| async def up(): | |
| async for m in ws: | |
| if m.type == WSMsgType.BINARY: | |
| w.write(m.data) | |
| await w.drain() | |
| w.close() | |
| async def down(): | |
| while data := await r.read(65536): | |
| await ws.send_bytes(data) | |
| await ws.close() | |
| await asyncio.gather(up(), down(), return_exceptions=True) | |
| return ws | |
| async def main(): | |
| global http | |
| http = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=None, sock_connect=10), auto_decompress=False) | |
| import_dir = await asyncio.to_thread(restore) | |
| app = web.Application(client_max_size=1024 ** 3) | |
| app.add_routes([web.get("/health", lambda r: web.json_response({"ok": True})), | |
| web.get("/_mc/status", status), web.post("/_mc/backup", do_backup), web.post("/_mc/world", upload_world), web.get("/_mc/tcp", tcp_bridge), | |
| web.route("*", "/{tail:.*}", proxy)]) | |
| runner = web.AppRunner(app) | |
| await runner.setup() | |
| await web.TCPSite(runner, "0.0.0.0", 7860).start() | |
| (CRAFTY / "app" / "config").mkdir(parents=True, exist_ok=True) | |
| if not any(p.name != ".gitkeep" for p in (CRAFTY / "app" / "config").iterdir()): | |
| subprocess.run(f"cp -r {CRAFTY}/app/config_original/* {CRAFTY}/app/config/", shell=True) | |
| shutil.copy(CRAFTY / "app" / "config_original" / "version.json", CRAFTY / "app" / "config" / "version.json") | |
| NETHERNET.update(await asyncio.to_thread(nethernet_props)) | |
| log.info("nethernet: %s", NETHERNET) | |
| for d in (CRAFTY / "servers").iterdir(): | |
| if (d / "server.properties").exists(): | |
| shutil.copy2(APP / "template" / "bedrock" / "bedrock_server", d / "bedrock_server") | |
| (d / "bedrock_server").chmod(0o755) | |
| set_props(d) | |
| cp = await asyncio.create_subprocess_exec(str(CRAFTY / ".venv/bin/python"), "main.py", "-d", "-i", cwd=str(CRAFTY)) | |
| asyncio.create_task(run_playit()) | |
| asyncio.create_task(tunnels_loop()) | |
| await wait_crafty() | |
| log.info("crafty up") | |
| if import_dir is not None: | |
| await bootstrap(import_dir) | |
| else: | |
| await crafty.login(CRAFTY_PASSWORD) | |
| await asyncio.sleep(10) | |
| sid = await crafty.server_id() | |
| st = await crafty.req("GET", f"/api/v2/servers/{sid}/stats") | |
| if sid and not (st.get("data") or {}).get("running"): | |
| log.info("start: %s", await crafty.req("POST", f"/api/v2/servers/{sid}/action/start_server")) | |
| asyncio.create_task(save_identity()) | |
| asyncio.create_task(backup_loop()) | |
| stop = asyncio.Event() | |
| loop = asyncio.get_running_loop() | |
| for s in (signal.SIGTERM, signal.SIGINT): | |
| loop.add_signal_handler(s, stop.set) | |
| await stop.wait() | |
| await backup("shutdown") | |
| cp.terminate() | |
| asyncio.run(main()) | |