Y / app.py
MuXue1230's picture
Offer all compatible local chat checkpoints
c4d8d17 verified
Raw History Blame Contribute Delete
4.68 kB
import spaces # ZeroGPU must intercept CUDA before any torch import.
import atexit
import hashlib
import json
import os
from pathlib import Path
import threading
import time
import torch
import gradio as gr
from fastapi import HTTPException, Request
from fastapi.responses import HTMLResponse
from huggingface_hub import hf_hub_download
from transformers import AutoTokenizer, Qwen3_5ForConditionalGeneration
from peft import LoraConfig,TaskType,get_peft_model
from safetensors.torch import load_file
from bridge import Registry,serve,rpc,run_turn
from training.d3_sequence import generate_with
ROOT=Path(__file__).resolve().parent
release=json.loads((ROOT/'release.json').read_text())
releases={item['checkpoint']['id']:item for item in release.get('checkpoints',[release])}
tokenizer=AutoTokenizer.from_pretrained(release['base'],revision=release['base_revision'])
core,loading=Qwen3_5ForConditionalGeneration.from_pretrained(release['base'],revision=release['base_revision'],
dtype=torch.bfloat16,attn_implementation='eager',output_loading_info=True)
if any(loading.get(k) for k in ('missing_keys','unexpected_keys','mismatched_keys','error_msgs')):
raise RuntimeError('Incomplete base load')
core=get_peft_model(core,LoraConfig(task_type=TaskType.CAUSAL_LM,r=8,lora_alpha=16,lora_dropout=0.,
target_modules=release['targets'],bias='none'))
params={n:p for n,p in core.named_parameters() if p.requires_grad}
adapter_paths={}
for checkpoint,item in releases.items():
if any(item[key]!=release[key] for key in ('base','base_revision','targets')):
raise RuntimeError('Incompatible checkpoint base or adapter architecture')
adapter_paths[checkpoint]={}
for route in ('external','memory'):
entry=item[route]
path=hf_hub_download(item['model_repo'],entry['file'],revision=item['model_revision'])
if hashlib.sha256(Path(path).read_bytes()).hexdigest()!=entry['sha256']:
raise RuntimeError('Adapter digest mismatch')
adapter_paths[checkpoint][route]=path
core.eval().to('cuda')
decode=generate_with(core,tokenizer)
executable=ROOT/'host/Y.Host'
executable.chmod(0o755)
registry=Registry([str(executable)],json.loads((ROOT/'chat-context.json').read_text()),[item['checkpoint'] for item in releases.values()])
server,rpc_url,rpc_secret=serve(registry)
atexit.register(registry.close)
def reap():
while True:
time.sleep(60);registry.expire()
threading.Thread(target=reap,daemon=True).start()
@spaces.GPU(duration=60)
def gpu_chat(text: str, token: str, job: str, memory_id: str='') -> dict:
"""Generate and submit one text turn using this visitor's current Harness views."""
torch.set_num_threads(4)
call=lambda op,**kw:rpc(rpc_url,rpc_secret,token,op,**kw)
prepared=call('selection',job=job)
if prepared.get('status')!='ready':
return dict(responseStatus='canceled',memoryStatus='canceled')
paths=adapter_paths[prepared['checkpoint']]
adapters={path:load_file(path) for path in set(paths.values())}
for weights in adapters.values():
if weights.keys()!=params.keys() or any(weights[n].shape!=p.shape for n,p in params.items()):
raise RuntimeError('Adapter architecture mismatch')
def generate(channel,visible):
route='external' if channel=='external' else 'memory'
with torch.no_grad():
for name,param in params.items():param.copy_(adapters[paths[route]][name])
return decode(channel,visible)
started=time.monotonic()
result=run_turn(call,generate,text,job,memory_id or None)
result['seconds']=round(time.monotonic()-started,3)
return result
demo=gr.Server(title='和 Y 聊聊')
@demo.api(name='chat',concurrency_limit=1,time_limit=90)
def chat(text: str, token: str, job: str, memory_id: str='') -> dict:
"""Talk with Y. Use a session token and a prepared job from the same browser."""
try:
return gpu_chat(text,token,job,memory_id)
finally:
registry.apply(token,'finish',job=job)
@demo.get('/',response_class=HTMLResponse)
def homepage():return (ROOT/'index.html').read_text(encoding='utf-8')
@demo.post('/y/session')
def create_session():return dict(token=registry.create())
@demo.post('/y/control')
async def control(request:Request):
body=await request.json()
if body.get('op') not in ('state','prepare','interrupt','forget','restart'):raise HTTPException(400,'Unknown operation')
try:return registry.apply(**body)
except ValueError as e:raise HTTPException(400,str(e))
if __name__=='__main__':demo.launch(ssr_mode=False)