"""Real-model loaders; one isolated process per upstream module namespace. Nothing here fabricates outputs. Hardware/model failures are propagated. """ import os import sys import math from pathlib import Path from types import SimpleNamespace import numpy as np import torch from PIL import Image from .design import ROOT from .control import gate, target_mask, masked_decode def activate(model): path=ROOT/'vendor'/model sys.path.insert(0,str(path)); os.chdir(path) return path def pixels_image(path, resolution): image=Image.open(path).convert('RGB') if image.size!=(resolution,resolution): raise ValueError('Reference resolution mismatch; convert offline explicitly') return torch.from_numpy(np.asarray(image).copy()).permute(2,0,1)[None].float().cuda()/255 def sdpa_flash_fallback(): """Replace Wan's flash_attention explicitly, including length masking. This is an alternate attention backend, not an unmodified upstream claim. Reject unsupported windows instead of silently ignoring them. """ from wan.modules import attention as mod def fallback(q,k,v,q_lens=None,k_lens=None,dropout_p=0.,softmax_scale=None, q_scale=None,causal=False,window_size=(-1,-1),deterministic=False, dtype=torch.bfloat16,version=None): if window_size!=(-1,-1): raise ValueError('SDPA fallback does not implement sliding attention windows') b,lq,_,_=q.shape; lk=k.shape[1] q=q.transpose(1,2); k=k.transpose(1,2); v=v.transpose(1,2) if q_scale is not None: q=q*q_scale allow=None if k_lens is not None: allow=torch.arange(lk,device=k.device)[None,None,None,:]','<|eoi|>','<|sov|>','<|eov|>','<|t2i|>','<|mmu|>','<|t2v|>','<|v2v|>','<|lvg|>'),ignore_id=-100,cond_dropout_prob=.1) self.attention_fn=create_attention_mask_predict_next self.offset=self.config.model.showo.llm_vocab_size+self.config.model.showo.num_new_special_tokens self.n=self.config.model.showo.num_vq_tokens def inputs(self,prompt,ids): v=torch.where(ids==self.model.config.mask_token_id,ids,ids+self.offset) cond,_=self.prompting(([prompt],v),'t2i_gen'); uncond,_=self.prompting(([''],v),'t2i_gen') return cond,uncond def predict(self,prompt,ids): cond,uncond=self.inputs(prompt,ids); both=torch.cat([cond,uncond]) d=self.prompting.sptids_dict mask=self.attention_fn(both,pad_id=int(d['<|pad|>']),soi_id=int(d['<|soi|>']),eoi_id=int(d['<|eoi|>']),rm_pad_in_image=True) c,u=self.model(both,attention_mask=mask).chunk(2) logits=(1+self.cfg['guidance'])*c-self.cfg['guidance']*u return logits[:,-(self.n+1):-1,self.offset:-1] def decode(self,tokens): image=self.vq.decode_code(tokens.reshape(1,-1)) return ((image+1)/2).clamp(0,1) @torch.no_grad() def native(self,prompt,seed): from models import get_mask_chedule torch.manual_seed(seed); ids=torch.full((1,self.n),self.model.config.mask_token_id,device='cuda',dtype=torch.long) cond,uncond=self.inputs(prompt,ids); d=self.prompting.sptids_dict attention=self.attention_fn(torch.cat([cond,uncond]),pad_id=int(d['<|pad|>']),soi_id=int(d['<|soi|>']),eoi_id=int(d['<|eoi|>']),rm_pad_in_image=True) tokens=self.model.t2i_generate(input_ids=cond,uncond_input_ids=uncond,attention_mask=attention, guidance_scale=self.cfg['guidance'],timesteps=self.cfg['steps'],noise_schedule=get_mask_chedule('cosine'), generator=torch.Generator(device='cuda').manual_seed(seed),config=self.config) return self.decode(tokens),tokens def reference(self,task): if task.get('reference_path'): image=pixels_image(task['reference_path'],self.cfg['resolution']) return image,self.vq.get_code(image*2-1) return self.native(task['reference_prompt'],task.get('reference_seed',42)) @torch.no_grad() def generate(self,job,ref,trace): h=int(math.sqrt(self.n)); mask=target_mask((h,h),job['task']) tokens,commits=masked_decode(lambda ids:self.predict(job['task']['prompt'],ids),ref,mask, self.model.config.mask_token_id,job.get('steps',self.cfg['steps']),job,trace) return self.decode(tokens),tokens,commits,mask class Momask: def __init__(self,cfg,weights): activate('momask'); self.cfg=cfg from .momask_loading import loading_functions load_vq_model,load_trans_model,load_res_model=loading_functions() from utils.get_opt import get_opt opt=SimpleNamespace(checkpoints_dir=str(Path(weights)/'momask/checkpoints'),dataset_name='t2m', name=cfg['name'],res_name=cfg['res_name'],device=torch.device('cuda')) model_opt=get_opt(str(Path(opt.checkpoints_dir)/'t2m'/opt.name/'opt.txt'),device=opt.device,checkpoints_dir=opt.checkpoints_dir) vq_opt=get_opt(str(Path(opt.checkpoints_dir)/'t2m'/model_opt.vq_name/'opt.txt'),device=opt.device,checkpoints_dir=opt.checkpoints_dir) vq_opt.dim_pose=263 self.vq,vq_opt=load_vq_model(vq_opt) model_opt.num_tokens=vq_opt.nb_code; model_opt.num_quantizers=vq_opt.num_quantizers; model_opt.code_dim=vq_opt.code_dim res_opt=get_opt(str(Path(opt.checkpoints_dir)/'t2m'/opt.res_name/'opt.txt'),device=opt.device,checkpoints_dir=opt.checkpoints_dir) self.res=load_res_model(res_opt,vq_opt,opt) self.model=load_trans_model(model_opt,opt,'latest.tar') for module in [self.vq,self.res,self.model]: module.cuda().eval() meta=Path(opt.checkpoints_dir)/'t2m'/model_opt.vq_name/'meta' self.mean=torch.tensor(np.load(meta/'mean.npy'),device='cuda',dtype=torch.float32) self.std=torch.tensor(np.load(meta/'std.npy'),device='cuda',dtype=torch.float32) self.lens=torch.tensor([cfg['frames']//4],device='cuda',dtype=torch.long) def decode(self,tokens,prompt,seed): from utils.motion_process import recover_from_ric torch.manual_seed(seed+999) residual=self.res.generate(tokens,[prompt],self.lens,temperature=1,cond_scale=5) normalized=self.vq.forward_decoder(residual) features=normalized*self.std+self.mean joints=recover_from_ric(features.float(),22) return joints,features @torch.no_grad() def native(self,prompt,seed): torch.manual_seed(seed) tokens=self.model.generate([prompt],self.lens,timesteps=self.cfg['steps'],cond_scale=self.cfg['guidance']) joints,features=self.decode(tokens,prompt,seed) return joints,tokens def reference(self,task): if task.get('reference_path'): # Explicit format: denormalized HumanML3D 263D features, NOT XYZ joints. data=np.load(task['reference_path'],allow_pickle=False) if data.shape!=(self.cfg['frames'],263): raise ValueError('MoMask reference must be [196,263] HumanML3D features') f=torch.tensor(data,device='cuda',dtype=torch.float32)[None] tokens,_=self.vq.encode((f-self.mean)/self.std) from utils.motion_process import recover_from_ric return recover_from_ric(f,22),tokens[:,:,0] return self.native(task['reference_prompt'],task.get('reference_seed',42)) @torch.no_grad() def generate(self,job,ref,trace): vector=self.model.encode_text([job['task']['prompt']]); mask=target_mask((ref.shape[-1],),job['task']) padding=torch.zeros_like(ref,dtype=torch.bool) def predict(ids): logits=self.model.forward_with_cond_scale(ids,cond_vector=vector,padding_mask=padding,cond_scale=self.cfg['guidance']) return logits.permute(0,2,1) tokens,commits=masked_decode(predict,ref,mask,self.model.mask_id,job.get('steps',self.cfg['steps']),job,trace) joints,features=self.decode(tokens,job['task']['prompt'],job['seed']) return joints,features,commits,mask class SD35: def __init__(self,cfg,weights): from diffusers import StableDiffusion3Pipeline self.cfg=cfg self.pipe=StableDiffusion3Pipeline.from_pretrained(cfg['repo'],torch_dtype=torch.bfloat16).to('cuda') self.pipe.set_progress_bar_config(disable=True) def decode(self,latents): config=self.pipe.vae.config return ((self.pipe.vae.decode(latents/config.scaling_factor+config.shift_factor,return_dict=False)[0]+1)/2).clamp(0,1) @torch.no_grad() def native(self,prompt,seed): lat=self.pipe(prompt,height=self.cfg['resolution'],width=self.cfg['resolution'], num_inference_steps=self.cfg['steps'],guidance_scale=self.cfg['guidance'],output_type='latent', generator=torch.Generator(device='cuda').manual_seed(seed)).images return self.decode(lat),lat def reference(self,task): if task.get('reference_path'): image=pixels_image(task['reference_path'],self.cfg['resolution']) # mode, not sample: reference encoding has no additional RNG draw. lat=self.pipe.vae.encode((image*2-1).to(self.pipe.vae.dtype)).latent_dist.mode() config=self.pipe.vae.config return image,(lat-config.shift_factor)*config.scaling_factor return self.native(task['reference_prompt'],task.get('reference_seed',42)) @torch.no_grad() def generate(self,job,ref,trace): mask=target_mask(ref.shape[-2:],job['task'])[None,None] prior=torch.randn(ref.shape,device='cuda',dtype=ref.dtype,generator=torch.Generator(device='cuda').manual_seed(job.get('prior_seed',1234))) counter=[0]; state={} def before(module,args,kw): x=kw['hidden_states']; t=kw['timestep'][0] scheduler=self.pipe.scheduler sigma=float(scheduler.sigmas[counter[0]]); p=1-sigma g=gate(mask*job['alpha'],p,1/job.get('steps',self.cfg['steps']),job.get('soft_release',True)).to(x.dtype) if job['alpha']>0 and job['arm']!='guide': x=x*(1-g)+(ref*(1-sigma)+prior*sigma)*g kw['hidden_states']=x; state.update(x=x,g=g,sigma=sigma) trace.append(dict(step=counter[0],progress=p,gate_mean=float(g.mean()),committed_fraction=0.,sigma=sigma)) counter[0]+=1 return args,kw def after(module,args,kw,out): if job['arm']!='guide' or job['alpha']==0: return out v=out[0] if isinstance(out,tuple) else out.sample sigma=state['sigma'] if sigma<=1e-8: return out x=state['x']; r=ref.expand_as(x) # x0=x−σv; guide clean prediction and convert back to velocity. g=state['g']*job.get('guide_amplitude',.1) pred=x-sigma*v; pred=pred+g*(r-pred); changed=(x-pred)/sigma if isinstance(out,tuple): return (changed,*out[1:]) out.sample=changed; return out handles=[self.pipe.transformer.register_forward_pre_hook(before,with_kwargs=True), self.pipe.transformer.register_forward_hook(after,with_kwargs=True)] original_step=self.pipe.scheduler.step def controlled_step(model_output,timestep,sample,*args,**kwargs): # Scheduler must integrate from the same pinned state seen by the model. if job['alpha']>0:sample=state['x'][-sample.shape[0]:] return original_step(model_output,timestep,sample,*args,**kwargs) self.pipe.scheduler.step=controlled_step try: lat=self.pipe(job['task']['prompt'],height=self.cfg['resolution'],width=self.cfg['resolution'], num_inference_steps=job.get('steps',self.cfg['steps']),guidance_scale=self.cfg['guidance'],output_type='latent', generator=torch.Generator(device='cuda').manual_seed(job['seed'])).images finally: for h in handles:h.remove() self.pipe.scheduler.step=original_step return self.decode(lat),lat,None,mask class SelfForcing: def __init__(self,cfg,weights): root=activate('self_forcing'); self.cfg=cfg src=Path(weights)/'self_forcing/wan_models/Wan2.1-T2V-1.3B' dest=root/'wan_models/Wan2.1-T2V-1.3B' dest.parent.mkdir(parents=True,exist_ok=True) if not dest.exists(): dest.symlink_to(src,target_is_directory=True) sdpa_flash_fallback() from omegaconf import OmegaConf from pipeline.causal_inference import CausalInferencePipeline config=OmegaConf.merge(OmegaConf.load(root/'configs/default_config.yaml'),OmegaConf.load(root/'configs/self_forcing_dmd.yaml')) self.pipe=CausalInferencePipeline(config,device=torch.device('cuda')).to(dtype=torch.bfloat16) ckpt=Path(weights)/'self_forcing'/cfg['student_file'] state=torch.load(ckpt,map_location='cpu',weights_only=False) self.pipe.generator.load_state_dict(state['generator_ema']) for model in [self.pipe.text_encoder,self.pipe.generator,self.pipe.vae]: model.cuda().eval() @torch.no_grad() def native(self,prompt,seed): torch.manual_seed(seed) noise=torch.randn(1,self.cfg['latent_frames'],16,60,104,device='cuda',dtype=torch.bfloat16) return self.pipe.inference(noise,[prompt],return_latents=True) def reference(self,task): if task.get('reference_path'): import imageio.v3 as iio meta=iio.immeta(task['reference_path']) if 'fps' not in meta or abs(float(meta['fps'])-self.cfg['fps'])>.01: raise ValueError('Reference video must explicitly have native 16 fps') frames=iio.imread(task['reference_path']) expected=1+4*(self.cfg['latent_frames']-1) if frames.shape!=(expected,480,832,3): raise ValueError(f'Reference video must be [{expected},480,832,3], native frame rate; no silent resampling') p=torch.tensor(frames.copy(),device='cuda').permute(3,0,1,2)[None].to(torch.bfloat16)/127.5-1 lat=self.pipe.vae.encode_to_latent(p).to(torch.bfloat16) return ((p+1)/2).permute(0,2,1,3,4),lat return self.native(task['reference_prompt'],task.get('reference_seed',42)) @torch.no_grad() def generate(self,job,ref,trace): mask=target_mask((ref.shape[1],),job['task'])[None,:,None,None,None] prior=torch.randn(ref.shape,device=ref.device,dtype=ref.dtype,generator=torch.Generator(device='cuda').manual_seed(job.get('prior_seed',1234))) state={}; steps=len(self.pipe.denoising_step_list) first=int((mask.reshape(-1)>0).nonzero()[0]); lead=torch.zeros_like(mask);lead[:,:first]=1 def before(module,args,kw): t=kw['timestep']; start=int(kw['current_start']//self.pipe.frame_seq_length) x=kw['noisy_image_or_video']; n=x.shape[1] # Never inject the separate clean-cache refresh evaluation. cache=float(t.flatten()[0])==float(self.pipe.args.context_noise) sched=self.pipe.scheduler ti=(sched.timesteps.to(t.device)-t.flatten()[0]).abs().argmin() sigma=float(sched.sigmas[ti]);p=1-sigma local_mask=mask[:,start:start+n] if job['arm']=='full_prefix': local_mask=torch.maximum(local_mask,lead[:,start:start+n]) g=gate(local_mask*job['alpha'],p,1/steps,job.get('soft_release',True)).to(x.dtype) r=ref[:,start:start+n] if not cache and job['alpha']>0: x=x*(1-g)+(r*(1-sigma)+prior[:,start:start+n]*sigma)*g kw['noisy_image_or_video']=x state.update(cache=cache,start=start,r=r,p=p,n=n) trace.append(dict(step=len(trace),progress=p,gate_mean=0. if cache else float(g.mean()),sigma=sigma, committed_fraction=start/ref.shape[1],block_start=start,block_frames=n, evaluation_kind='cache_refresh' if cache else 'denoise')) return args,kw def after(module,args,kw,out): if state['cache'] or job['arm']!='guide' or job['alpha']==0:return out flow,pred=out guide=gate(lead[:,state['start']:state['start']+state['n']]*job.get('guide_duration',.05)*job['alpha'],state['p'],1/steps) pred=pred+guide.to(pred.dtype)*job.get('guide_amplitude',.1)*(state['r']-pred) return flow,pred handles=[self.pipe.generator.register_forward_pre_hook(before,with_kwargs=True), self.pipe.generator.register_forward_hook(after,with_kwargs=True)] original_block=self.pipe.num_frame_per_block original_model_block=self.pipe.generator.model.num_frame_per_block try: block=job.get('block_frames',original_block) if ref.shape[1]%block:raise ValueError('Block stress size must divide native latent length') self.pipe.num_frame_per_block=block;self.pipe.generator.model.num_frame_per_block=block output,lat=self.native(job['task']['prompt'],job['seed']) if job.get('rollback_blocks'): # Reopen a suffix before the target; the official inference entry # resets all KV/cross-attention caches and rebuilds the clean prefix. boundary=(first//block)*block restart=max(0,boundary-job['rollback_blocks']*block) noise=torch.randn(ref.shape,device=ref.device,dtype=ref.dtype, generator=torch.Generator(device='cuda').manual_seed(job['seed']+100000)) output,lat=self.pipe.inference(noise[:,restart:],[job['task']['prompt']], initial_latent=lat[:,:restart] if restart else None,return_latents=True) for item in trace:item['rollback_restart']=restart finally: for h in handles:h.remove() self.pipe.num_frame_per_block=original_block self.pipe.generator.model.num_frame_per_block=original_model_block return output,lat,None,mask LOADERS={'meissonic':Meissonic,'momask':Momask,'sd35':SD35,'self_forcing':SelfForcing,'showo':Showo}