#!/usr/bin/env python3 """Ora GPU worker: runs Ora's video and image jobs on a free online GPU (Kaggle, Colab) for the person who owns it. Kaggle (Ora starts this itself): python3 ora_worker.py --batch job.json --out /kaggle/working/out Colab (paste in one cell): !curl -fsSL https://devseis-ora.static.hf.space/worker/ora_worker.py -o w.py && python3 w.py --serve It downloads Ora's GPU engine (stable-diffusion.cpp built for CUDA) and the model files a job needs, at the pinned revisions in Ora's models.json, runs the engine with the arguments Ora built, and returns MP4s (made with ffmpeg) or PNGs. In --serve mode it starts a local HTTP server behind a temporary Cloudflare tunnel and prints a connection code; only requests carrying that code's secret are accepted. Standard library only. """ import argparse, base64, hashlib, json, os, re, secrets, shutil, subprocess, sys, tarfile, threading, time, urllib.request from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer VERSION = '0.4.3' SITE = 'https://devseis-ora.static.hf.space' RELEASES = 'https://huggingface.co/datasets/Devseis/ora-releases/resolve/main' ENGINE_FILE = 'ora-engine-linux-cuda.tar.gz' HOME = os.path.expanduser(os.environ.get('ORA_HOME', '~/.ora')) MODELS = os.path.join(HOME, 'models') TOKEN = re.compile(r'\{(model|out|init|ref):?([A-Za-z0-9_]*)\}') def log(*a): print('[ora]', *a, flush=True) def fetch(url, dest, size=None, tries=6): """Download url to dest. Big model files sometimes stop early when the connection drops, so keep the .part file and continue from where it stopped (HTTP Range), up to `tries` times, before giving up.""" if os.path.exists(dest) and (size is None or os.path.getsize(dest) == size): return dest last = None for attempt in range(tries): try: return fetch_once(url, dest, size) except Exception as e: last = e part = dest + '.part' have = os.path.getsize(part) if os.path.exists(part) else 0 log(f'download of {os.path.basename(dest)} stopped at {have / 1e9:.2f} GB ({e}); continuing (try {attempt + 2} of {tries})') time.sleep(min(30, 3 * (attempt + 1))) raise RuntimeError(f'{os.path.basename(dest)}: download kept failing: {last}') def fetch_once(url, dest, size=None): os.makedirs(os.path.dirname(dest), exist_ok=True) part = dest + '.part' have = os.path.getsize(part) if os.path.exists(part) else 0 if size is not None and have > size: os.remove(part); have = 0 req = urllib.request.Request(url, headers={'User-Agent': f'ora-worker/{VERSION}', **({'Range': f'bytes={have}-'} if have else {})}) with urllib.request.urlopen(req, timeout=60) as r, open(part, 'ab' if have and r.status == 206 else 'wb') as f: if r.status != 206: have = 0 while True: chunk = r.read(8 << 20) if not chunk: break f.write(chunk) if size is not None and os.path.getsize(part) != size: raise RuntimeError(f'{os.path.basename(dest)}: got {os.path.getsize(part)} bytes, expected {size}') os.replace(part, dest) return dest def get_models(files): """Download the model files for these keys in parallel (Kaggle and Colab have fast links to Hugging Face).""" errors, threads = [], [] def one(k): f = files[k] url = f.get('url') or f"https://huggingface.co/{f['repo']}/resolve/{f['rev']}/{f['path']}" try: t = time.time() fetch(url, os.path.join(MODELS, f['name']), f['size']) log(f"model ready: {f['name']} ({f['size'] / 1e9:.1f} GB, {time.time() - t:.0f} s)") except Exception as e: errors.append(f"{f['name']}: {e}") for k in sorted(files): th = threading.Thread(target=one, args=(k,)) th.start(); threads.append(th) for th in threads: th.join() if errors: raise RuntimeError('; '.join(errors)) def get_engine(engine_arg): if engine_arg: return engine_arg exe = os.path.join(HOME, 'engine', 'ora-engine', 'sd-cli') if not os.path.exists(exe): tgz = fetch(f'{RELEASES}/{ENGINE_FILE}', os.path.join(HOME, ENGINE_FILE)) with tarfile.open(tgz) as t: t.extractall(os.path.join(HOME, 'engine')) os.chmod(exe, 0o755) return exe # NVIDIA libraries the CUDA engine links to: cuBLAS and the CUDA runtime (in /usr/local/cuda on Kaggle and Colab) # and NCCL. Look in the system CUDA folders and in the nvidia-* pip packages that come with PyTorch; pip-install # whatever is still missing. libcuda.so.1 comes with the GPU driver. PIP_FOR = {'libcublas.so.12': 'nvidia-cublas-cu12', 'libcublasLt.so.12': 'nvidia-cublas-cu12', 'libcudart.so.12': 'nvidia-cuda-runtime-cu12', 'libnccl.so.2': 'nvidia-nccl-cu12'} def lib_dirs(): dirs = ['/usr/local/cuda/lib64', '/usr/local/nvidia/lib64', '/usr/lib64-nvidia'] for sp in sys.path: nv = os.path.join(sp, 'nvidia') if os.path.isdir(nv): dirs += [os.path.join(nv, d, 'lib') for d in os.listdir(nv) if os.path.isdir(os.path.join(nv, d, 'lib'))] return [d for d in dirs if os.path.isdir(d)] def engine_env(exe): env = dict(os.environ) for attempt in range(2): env['LD_LIBRARY_PATH'] = ':'.join(lib_dirs() + [env.get('LD_LIBRARY_PATH', '')]).strip(':') try: out = subprocess.run(['ldd', exe], capture_output=True, text=True, env=env, timeout=30).stdout except Exception: return env missing = sorted({PIP_FOR[l.split()[0]] for l in out.splitlines() if 'not found' in l and l.split()[0] in PIP_FOR}) if not missing: return env if attempt == 0: log(f'installing NVIDIA libraries: {", ".join(missing)}') subprocess.run([sys.executable, '-m', 'pip', 'install', '-q', *missing]) log('warning: some NVIDIA libraries are still missing; the engine may not start') return env def gpu_name(): try: return subprocess.run(['nvidia-smi', '--query-gpu=name,memory.total', '--format=csv,noheader'], capture_output=True, text=True, timeout=10).stdout.strip() except Exception: return 'no NVIDIA GPU found' def fill(args, out_dir, init_path, man_files, ref_paths=()): """Replace Ora's placeholders: {model:key} -> model file, {out} -> output folder, {init} -> start image, {ref:N} -> reference picture N (Combine: the product, the scene, the person).""" def rep(m): kind, key = m.group(1), m.group(2) if kind == 'model': return os.path.join(MODELS, man_files[key]['name']) if kind == 'out': return out_dir if kind == 'ref': i = int(key or 0) return ref_paths[i] if i < len(ref_paths) else '' return init_path or '' return [TOKEN.sub(rep, str(a)) for a in args] def run_job(job, engine, base_out, on_line=None): """Run one job. Returns a result dict with the files made.""" jid = re.sub(r'[^\w-]', '', job['id'])[:80] or 'job' out_dir = os.path.join(base_out, jid) os.makedirs(out_dir, exist_ok=True) init_path = None if job.get('initPng'): init_path = os.path.join(out_dir, 'start.png') with open(init_path, 'wb') as f: f.write(base64.b64decode(job['initPng'])) ref_paths = [] for i, b64 in enumerate((job.get('refPngs') or [])[:3]): ref_paths.append(os.path.join(out_dir, 'ref_%d.png' % i)) with open(ref_paths[-1], 'wb') as f: f.write(base64.b64decode(b64)) # Ora sends the exact pinned model files the job needs: {key: {name, repo, rev, path, size}} files = {k: v for k, v in job['files'].items() if re.match(r'^[\w.-]+$', v.get('name', ''))} get_models(files) args = fill(job['args'], out_dir, init_path, files, ref_paths) t0 = time.time() with open(os.path.join(out_dir, 'engine.log'), 'w') as lf: env = STATE.setdefault('env', {}).get(engine) or engine_env(engine) STATE['env'][engine] = env p = subprocess.Popen([engine, *args], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=out_dir, env=env) STATE['proc'] = p buf = b'' while True: chunk = p.stdout.read1(4096) if hasattr(p.stdout, 'read1') else p.stdout.read(4096) if not chunk: break lf.write(chunk.decode('utf8', 'replace')) buf += chunk *lines, buf = re.split(rb'[\r\n]', buf) for ln in lines: if ln.strip() and on_line: on_line(ln.decode('utf8', 'replace')) code = p.wait() STATE['proc'] = None result = {'id': job['id'], 'code': code, 'seconds': round(time.time() - t0), 'files': []} if code != 0: return result if job['kind'] == 'video' and os.path.exists(os.path.join(out_dir, 'video.webm')): # models with sound (LTX, MiniMax-H3) write one WebM: make an MP4 with H.264 + AAC, and a poster frame webm, mp4 = os.path.join(out_dir, 'video.webm'), os.path.join(out_dir, 'video.mp4') subprocess.run(['ffmpeg', '-y', '-loglevel', 'error', '-i', webm, '-frames:v', '1', os.path.join(out_dir, 'poster.png')]) if shutil.which('ffmpeg') and subprocess.run(['ffmpeg', '-y', '-loglevel', 'error', '-i', webm, '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-crf', '16', '-c:a', 'aac', '-b:a', '192k', '-movflags', '+faststart', mp4]).returncode == 0: result['files'] = ['video.mp4'] + (['poster.png'] if os.path.exists(os.path.join(out_dir, 'poster.png')) else []) else: result['files'] = ['video.webm'] return result if job['kind'] == 'video': frames = sorted(f for f in os.listdir(out_dir) if re.match(r'frame_\d+\.png$', f)) if frames: shutil.copy(os.path.join(out_dir, frames[0]), os.path.join(out_dir, 'poster.png')) mp4 = os.path.join(out_dir, 'video.mp4') if shutil.which('ffmpeg') and subprocess.run(['ffmpeg', '-y', '-loglevel', 'error', '-framerate', str(job.get('fps', 16)), '-i', os.path.join(out_dir, 'frame_%03d.png'), '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-crf', '16', '-movflags', '+faststart', mp4]).returncode == 0: for f in frames: os.remove(os.path.join(out_dir, f)) result['files'] = ['video.mp4', 'poster.png'] else: result['files'] = frames + ['poster.png'] # Ora encodes the MP4 itself else: result['files'] = sorted(f for f in os.listdir(out_dir) if re.match(r'img_\d+\.png$', f)) return result STATE = {'proc': None} # --- Kaggle: run a batch of jobs, leave the results in the notebook output ------------------------------------------ def batch(job_file, out, engine_arg): with open(job_file) as f: spec = json.load(f) log(f'GPU: {gpu_name()}') engine = get_engine(engine_arg) results = [] for job in spec['jobs']: log(f"job {job['id']} ({job['kind']})") r = run_job(job, engine, out, on_line=lambda ln: print(ln, flush=True)) results.append(r) log(f"job {job['id']} finished with code {r['code']} in {r['seconds']} s") with open(os.path.join(out, 'results.json'), 'w') as f: json.dump({'version': VERSION, 'gpu': gpu_name(), 'results': results}, f) # --- Colab: serve jobs over a tunnel, with live progress ------------------------------------------------------------ class Jobs: def __init__(self): self.lock = threading.Lock() self.items = {} # id -> {state, lines, result} def add(self, job): with self.lock: self.items[job['id']] = {'state': 'queued', 'lines': [], 'result': None} threading.Thread(target=self._run, args=(job,), daemon=True).start() def _run(self, job): it = self.items[job['id']] with SERVE['busy']: # one job at a time: one GPU if it['state'] == 'cancelled': return it['state'] = 'running' try: it['result'] = run_job(job, SERVE['engine'], SERVE['out'], on_line=lambda ln: it['lines'].append(ln[-400:])) it['state'] = 'done' if it['result']['code'] == 0 else 'failed' except Exception as e: it['lines'].append(f'[ora] error: {e}') it['state'] = 'failed' SERVE = {'secret': None, 'jobs': Jobs(), 'busy': threading.Lock(), 'engine': None, 'out': None} class Handler(BaseHTTPRequestHandler): def log_message(self, *a): pass def _auth(self): if not secrets.compare_digest(self.headers.get('X-Ora-Secret', ''), SERVE['secret']): self._send(403, {'error': 'wrong connection code'}) return False return True def _send(self, code, obj=None, body=None, ctype='application/json'): data = body if body is not None else json.dumps(obj).encode() self.send_response(code) self.send_header('Content-Type', ctype) self.send_header('Content-Length', str(len(data))) self.end_headers() self.wfile.write(data) def do_GET(self): if not self._auth(): return parts = [p for p in self.path.split('?')[0].split('/') if p] if parts == ['hello']: return self._send(200, {'ok': True, 'version': VERSION, 'gpu': gpu_name(), 'busy': SERVE['busy'].locked()}) if len(parts) == 2 and parts[0] == 'status': it = SERVE['jobs'].items.get(parts[1]) if not it: return self._send(404, {'error': 'no such job'}) since = int((re.search(r'since=(\d+)', self.path) or [0, 0])[1]) return self._send(200, {'state': it['state'], 'lines': it['lines'][since:], 'next': len(it['lines']), 'result': it['result']}) if len(parts) == 3 and parts[0] == 'file': jid, name = re.sub(r'[^\w-]', '', parts[1]), os.path.basename(parts[2]) p = os.path.join(SERVE['out'], jid, name) if not re.match(r'^(video\.mp4|video\.webm|poster\.png|frame_\d+\.png|img_\d+\.png)$', name) or not os.path.exists(p): return self._send(404, {'error': 'no such file'}) with open(p, 'rb') as f: return self._send(200, body=f.read(), ctype='video/mp4' if name.endswith('.mp4') else 'image/png') self._send(404, {'error': 'not found'}) def do_POST(self): if not self._auth(): return parts = [p for p in self.path.split('/') if p] n = int(self.headers.get('Content-Length', 0)) if n > 32 << 20: return self._send(413, {'error': 'too large'}) body = json.loads(self.rfile.read(n) or b'{}') if parts == ['job']: if not isinstance(body.get('args'), list) or body.get('kind') not in ('video', 'image'): return self._send(400, {'error': 'bad job'}) SERVE['jobs'].add(body) return self._send(200, {'ok': True}) if len(parts) == 2 and parts[0] == 'cancel': it = SERVE['jobs'].items.get(parts[1]) if it: it['state'] = 'cancelled' if it['state'] == 'queued' else it['state'] p = STATE['proc'] if p and it['state'] == 'running': p.terminate() return self._send(200, {'ok': True}) self._send(404, {'error': 'not found'}) def start_tunnel(port): """A free Cloudflare quick tunnel (no account): prints https://.trycloudflare.com.""" exe = os.path.join(HOME, 'cloudflared') if not os.path.exists(exe): fetch('https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64', exe) os.chmod(exe, 0o755) p = subprocess.Popen([exe, 'tunnel', '--no-autoupdate', '--url', f'http://127.0.0.1:{port}'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) for line in p.stdout: m = re.search(r'https://[a-z0-9-]+\.trycloudflare\.com', line) if m: threading.Thread(target=lambda: [None for _ in p.stdout], daemon=True).start() return m.group(0) raise RuntimeError('the tunnel did not start') def serve(port, tunnel, engine_arg, out): SERVE.update(secret=secrets.token_urlsafe(24), engine=get_engine(engine_arg), out=out) os.makedirs(out, exist_ok=True) httpd = ThreadingHTTPServer(('127.0.0.1', port), Handler) threading.Thread(target=httpd.serve_forever, daemon=True).start() url = start_tunnel(port) if tunnel else f'http://127.0.0.1:{port}' code = 'ora1.' + base64.urlsafe_b64encode(json.dumps({'url': url, 'secret': SERVE['secret']}).encode()).decode().rstrip('=') log(f'GPU: {gpu_name()}') print('\n' + '=' * 70 + '\nCopy this connection code into Ora (Connections > Colab > Paste code):\n\n' + code + '\n\nKeep this cell running while you use Ora. Stop it to disconnect.\n' + '=' * 70, flush=True) while True: time.sleep(3600) if __name__ == '__main__': ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--batch', help='job file from Ora (Kaggle)') ap.add_argument('--serve', action='store_true', help='serve jobs from Ora over a tunnel (Colab)') ap.add_argument('--out', default=os.path.join(HOME, 'out')) ap.add_argument('--port', type=int, default=8765) ap.add_argument('--no-tunnel', action='store_true', help='testing: serve on localhost only') ap.add_argument('--engine', help='testing: path to an sd-cli to use instead of the CUDA build') a = ap.parse_args() if a.batch: batch(a.batch, a.out, a.engine) elif a.serve: serve(a.port, not a.no_tunnel, a.engine, a.out) else: ap.print_help()