Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified Download method_v2/evidence/code/suite/adapters.py from CharlesZhang-USTC/snf_experiments: direct link, hf CLI and curl.
- Browser
- Download file 23.3 kB
-
https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/adapters.py
- Command line
-
hf download hf://CharlesZhang-USTC/snf_experiments/method_v2/evidence/code/suite/adapters.py
-
curl -L -o adapters.py https://huggingface.co/CharlesZhang-USTC/snf_experiments/resolve/main/method_v2/evidence/code/suite/adapters.py
23.3 kB
| """Real-model loaders; one isolated process per upstream module namespace. | |
| Nothing here fabricates outputs. Hardware/model failures are propagated. | |
| """ | |
| import os | |
| import sys | |
| import math | |
| from pathlib import Path | |
| from types import SimpleNamespace | |
| import numpy as np | |
| import torch | |
| from PIL import Image | |
| from .design import ROOT | |
| from .control import gate, target_mask, masked_decode | |
| def activate(model): | |
| path=ROOT/'vendor'/model | |
| sys.path.insert(0,str(path)); os.chdir(path) | |
| return path | |
| def pixels_image(path, resolution): | |
| image=Image.open(path).convert('RGB') | |
| if image.size!=(resolution,resolution): | |
| raise ValueError('Reference resolution mismatch; convert offline explicitly') | |
| return torch.from_numpy(np.asarray(image).copy()).permute(2,0,1)[None].float().cuda()/255 | |
| def sdpa_flash_fallback(): | |
| """Replace Wan's flash_attention explicitly, including length masking. | |
| This is an alternate attention backend, not an unmodified upstream claim. | |
| Reject unsupported windows instead of silently ignoring them. | |
| """ | |
| from wan.modules import attention as mod | |
| def fallback(q,k,v,q_lens=None,k_lens=None,dropout_p=0.,softmax_scale=None, | |
| q_scale=None,causal=False,window_size=(-1,-1),deterministic=False, | |
| dtype=torch.bfloat16,version=None): | |
| if window_size!=(-1,-1): | |
| raise ValueError('SDPA fallback does not implement sliding attention windows') | |
| b,lq,_,_=q.shape; lk=k.shape[1] | |
| q=q.transpose(1,2); k=k.transpose(1,2); v=v.transpose(1,2) | |
| if q_scale is not None: q=q*q_scale | |
| allow=None | |
| if k_lens is not None: | |
| allow=torch.arange(lk,device=k.device)[None,None,None,:]<k_lens[:,None,None,None] | |
| if causal: | |
| # FlashAttention uses bottom-right alignment for unequal lengths. | |
| tri=torch.arange(lk,device=q.device)[None,:] <= (torch.arange(lq,device=q.device)[:,None]+lk-lq) | |
| allow=tri[None,None] if allow is None else allow & tri[None,None] | |
| out=torch.nn.functional.scaled_dot_product_attention(q,k,v,attn_mask=allow, | |
| dropout_p=dropout_p,scale=softmax_scale).transpose(1,2) | |
| if q_lens is not None: | |
| valid=torch.arange(lq,device=out.device)[None,:]<q_lens[:,None] | |
| out=out*valid[:,:,None,None] | |
| return out | |
| mod.flash_attention=fallback | |
| def attention(*args,fa_version=None,**kwargs):return fallback(*args,version=fa_version,**kwargs) | |
| mod.attention=attention | |
| # wan.__init__ eagerly imports modules before this replacement. | |
| for name,module in list(sys.modules.items()): | |
| if name.startswith('wan.modules.') and name != mod.__name__ and hasattr(module,'attention'): | |
| module.attention=attention | |
| class Meissonic: | |
| def __init__(self,cfg,weights): | |
| activate('meissonic') | |
| from src.transformer import Transformer2DModel | |
| from src.pipeline import Pipeline | |
| from src.scheduler import Scheduler | |
| from transformers import CLIPTextModelWithProjection,CLIPTokenizer | |
| from diffusers import VQModel | |
| self.cfg=cfg | |
| model=Transformer2DModel.from_pretrained(cfg['repo'],subfolder='transformer',torch_dtype=torch.float16) | |
| vq=VQModel.from_pretrained(cfg['repo'],subfolder='vqvae') | |
| text=CLIPTextModelWithProjection.from_pretrained(cfg['text_repo'],torch_dtype=torch.float16) | |
| self.pipe=Pipeline(vq,tokenizer=CLIPTokenizer.from_pretrained(cfg['repo'],subfolder='tokenizer'), | |
| text_encoder=text,transformer=model,scheduler=Scheduler.from_pretrained(cfg['repo'],subfolder='scheduler')).to('cuda') | |
| self.pipe.set_progress_bar_config(disable=True) | |
| def decode(self,tokens): | |
| h=w=self.cfg['resolution']//self.pipe.vae_scale_factor | |
| return self.pipe.vqvae.decode(tokens.reshape(1,h,w),force_not_quantize=True, | |
| shape=(1,h,w,self.pipe.vqvae.config.latent_channels)).sample.clamp(0,1) | |
| def native(self,prompt,seed): | |
| tokens=self.pipe(prompt=prompt,height=self.cfg['resolution'],width=self.cfg['resolution'], | |
| guidance_scale=self.cfg['guidance'],num_inference_steps=self.cfg['steps'],output_type='latent', | |
| generator=torch.Generator(device='cuda').manual_seed(seed)).images | |
| return self.decode(tokens),tokens | |
| def reference(self,task): | |
| if task.get('reference_path'): | |
| p=pixels_image(task['reference_path'],self.cfg['resolution']) | |
| lat=self.pipe.vqvae.encode(p).latents | |
| tokens=self.pipe.vqvae.quantize(lat)[2][2].reshape(1,*lat.shape[-2:]) | |
| return p,tokens | |
| return self.native(task['reference_prompt'],task.get('reference_seed',42)) | |
| def generate(self,job,ref,trace): | |
| h,w=ref.shape[-2:]; mask=target_mask((h,w),job['task']) | |
| holder={} | |
| # Capture the upstream first forward's static conditioning; abort before sampling. | |
| class Captured(Exception): pass | |
| def capture(module,args,kw): | |
| holder.update(kw); raise Captured() | |
| handle=self.pipe.transformer.register_forward_pre_hook(capture,with_kwargs=True) | |
| try: | |
| self.pipe(prompt=job['task']['prompt'],height=self.cfg['resolution'],width=self.cfg['resolution'], | |
| num_inference_steps=job.get('steps',self.cfg['steps']),guidance_scale=self.cfg['guidance']) | |
| except Captured: | |
| pass | |
| finally: handle.remove() | |
| steps=job.get('steps',self.cfg['steps']); calls=[0] | |
| def predict(ids): | |
| kw=dict(holder); x=ids.reshape(1,h,w) | |
| kw['hidden_states']=torch.cat([x,x]) | |
| kw['timestep']=torch.tensor([steps-1-calls[0]],device=x.device,dtype=torch.long) | |
| calls[0]+=1 | |
| output=self.pipe.transformer(**kw) | |
| u,c=output.chunk(2) | |
| logits=u+self.cfg['guidance']*(c-u) | |
| return logits.reshape(1,logits.shape[1],-1).permute(0,2,1) | |
| tokens,commits=masked_decode(predict,ref,mask,self.pipe.scheduler.config.mask_token_id,steps,job,trace) | |
| return self.decode(tokens),tokens.reshape(1,h,w),commits,mask | |
| class Showo: | |
| def __init__(self,cfg,weights): | |
| root=activate('showo'); self.cfg=cfg | |
| from models import Showo as Net,MAGVITv2 | |
| from transformers import AutoTokenizer | |
| from training.prompting_utils import UniversalPrompting,create_attention_mask_predict_next | |
| from omegaconf import OmegaConf | |
| self.config=OmegaConf.load(root/'configs/showo_demo.yaml') | |
| self.model=Net.from_pretrained(cfg['repo'],llm_model_path=cfg['text_repo']).eval().cuda() | |
| self.vq=MAGVITv2.from_pretrained(cfg['vq_repo']).eval().cuda() | |
| tokenizer=AutoTokenizer.from_pretrained(cfg['text_repo'],padding_side='left') | |
| self.prompting=UniversalPrompting(tokenizer,max_text_len=128, | |
| special_tokens=('<|soi|>','<|eoi|>','<|sov|>','<|eov|>','<|t2i|>','<|mmu|>','<|t2v|>','<|v2v|>','<|lvg|>'),ignore_id=-100,cond_dropout_prob=.1) | |
| self.attention_fn=create_attention_mask_predict_next | |
| self.offset=self.config.model.showo.llm_vocab_size+self.config.model.showo.num_new_special_tokens | |
| self.n=self.config.model.showo.num_vq_tokens | |
| def inputs(self,prompt,ids): | |
| v=torch.where(ids==self.model.config.mask_token_id,ids,ids+self.offset) | |
| cond,_=self.prompting(([prompt],v),'t2i_gen'); uncond,_=self.prompting(([''],v),'t2i_gen') | |
| return cond,uncond | |
| def predict(self,prompt,ids): | |
| cond,uncond=self.inputs(prompt,ids); both=torch.cat([cond,uncond]) | |
| d=self.prompting.sptids_dict | |
| mask=self.attention_fn(both,pad_id=int(d['<|pad|>']),soi_id=int(d['<|soi|>']),eoi_id=int(d['<|eoi|>']),rm_pad_in_image=True) | |
| c,u=self.model(both,attention_mask=mask).chunk(2) | |
| logits=(1+self.cfg['guidance'])*c-self.cfg['guidance']*u | |
| return logits[:,-(self.n+1):-1,self.offset:-1] | |
| def decode(self,tokens): | |
| image=self.vq.decode_code(tokens.reshape(1,-1)) | |
| return ((image+1)/2).clamp(0,1) | |
| def native(self,prompt,seed): | |
| from models import get_mask_chedule | |
| torch.manual_seed(seed); ids=torch.full((1,self.n),self.model.config.mask_token_id,device='cuda',dtype=torch.long) | |
| cond,uncond=self.inputs(prompt,ids); d=self.prompting.sptids_dict | |
| attention=self.attention_fn(torch.cat([cond,uncond]),pad_id=int(d['<|pad|>']),soi_id=int(d['<|soi|>']),eoi_id=int(d['<|eoi|>']),rm_pad_in_image=True) | |
| tokens=self.model.t2i_generate(input_ids=cond,uncond_input_ids=uncond,attention_mask=attention, | |
| guidance_scale=self.cfg['guidance'],timesteps=self.cfg['steps'],noise_schedule=get_mask_chedule('cosine'), | |
| generator=torch.Generator(device='cuda').manual_seed(seed),config=self.config) | |
| return self.decode(tokens),tokens | |
| def reference(self,task): | |
| if task.get('reference_path'): | |
| image=pixels_image(task['reference_path'],self.cfg['resolution']) | |
| return image,self.vq.get_code(image*2-1) | |
| return self.native(task['reference_prompt'],task.get('reference_seed',42)) | |
| def generate(self,job,ref,trace): | |
| h=int(math.sqrt(self.n)); mask=target_mask((h,h),job['task']) | |
| tokens,commits=masked_decode(lambda ids:self.predict(job['task']['prompt'],ids),ref,mask, | |
| self.model.config.mask_token_id,job.get('steps',self.cfg['steps']),job,trace) | |
| return self.decode(tokens),tokens,commits,mask | |
| class Momask: | |
| def __init__(self,cfg,weights): | |
| activate('momask'); self.cfg=cfg | |
| from .momask_loading import loading_functions | |
| load_vq_model,load_trans_model,load_res_model=loading_functions() | |
| from utils.get_opt import get_opt | |
| opt=SimpleNamespace(checkpoints_dir=str(Path(weights)/'momask/checkpoints'),dataset_name='t2m', | |
| name=cfg['name'],res_name=cfg['res_name'],device=torch.device('cuda')) | |
| model_opt=get_opt(str(Path(opt.checkpoints_dir)/'t2m'/opt.name/'opt.txt'),device=opt.device,checkpoints_dir=opt.checkpoints_dir) | |
| vq_opt=get_opt(str(Path(opt.checkpoints_dir)/'t2m'/model_opt.vq_name/'opt.txt'),device=opt.device,checkpoints_dir=opt.checkpoints_dir) | |
| vq_opt.dim_pose=263 | |
| self.vq,vq_opt=load_vq_model(vq_opt) | |
| model_opt.num_tokens=vq_opt.nb_code; model_opt.num_quantizers=vq_opt.num_quantizers; model_opt.code_dim=vq_opt.code_dim | |
| res_opt=get_opt(str(Path(opt.checkpoints_dir)/'t2m'/opt.res_name/'opt.txt'),device=opt.device,checkpoints_dir=opt.checkpoints_dir) | |
| self.res=load_res_model(res_opt,vq_opt,opt) | |
| self.model=load_trans_model(model_opt,opt,'latest.tar') | |
| for module in [self.vq,self.res,self.model]: module.cuda().eval() | |
| meta=Path(opt.checkpoints_dir)/'t2m'/model_opt.vq_name/'meta' | |
| self.mean=torch.tensor(np.load(meta/'mean.npy'),device='cuda',dtype=torch.float32) | |
| self.std=torch.tensor(np.load(meta/'std.npy'),device='cuda',dtype=torch.float32) | |
| self.lens=torch.tensor([cfg['frames']//4],device='cuda',dtype=torch.long) | |
| def decode(self,tokens,prompt,seed): | |
| from utils.motion_process import recover_from_ric | |
| torch.manual_seed(seed+999) | |
| residual=self.res.generate(tokens,[prompt],self.lens,temperature=1,cond_scale=5) | |
| normalized=self.vq.forward_decoder(residual) | |
| features=normalized*self.std+self.mean | |
| joints=recover_from_ric(features.float(),22) | |
| return joints,features | |
| def native(self,prompt,seed): | |
| torch.manual_seed(seed) | |
| tokens=self.model.generate([prompt],self.lens,timesteps=self.cfg['steps'],cond_scale=self.cfg['guidance']) | |
| joints,features=self.decode(tokens,prompt,seed) | |
| return joints,tokens | |
| def reference(self,task): | |
| if task.get('reference_path'): | |
| # Explicit format: denormalized HumanML3D 263D features, NOT XYZ joints. | |
| data=np.load(task['reference_path'],allow_pickle=False) | |
| if data.shape!=(self.cfg['frames'],263): raise ValueError('MoMask reference must be [196,263] HumanML3D features') | |
| f=torch.tensor(data,device='cuda',dtype=torch.float32)[None] | |
| tokens,_=self.vq.encode((f-self.mean)/self.std) | |
| from utils.motion_process import recover_from_ric | |
| return recover_from_ric(f,22),tokens[:,:,0] | |
| return self.native(task['reference_prompt'],task.get('reference_seed',42)) | |
| def generate(self,job,ref,trace): | |
| vector=self.model.encode_text([job['task']['prompt']]); mask=target_mask((ref.shape[-1],),job['task']) | |
| padding=torch.zeros_like(ref,dtype=torch.bool) | |
| def predict(ids): | |
| logits=self.model.forward_with_cond_scale(ids,cond_vector=vector,padding_mask=padding,cond_scale=self.cfg['guidance']) | |
| return logits.permute(0,2,1) | |
| tokens,commits=masked_decode(predict,ref,mask,self.model.mask_id,job.get('steps',self.cfg['steps']),job,trace) | |
| joints,features=self.decode(tokens,job['task']['prompt'],job['seed']) | |
| return joints,features,commits,mask | |
| class SD35: | |
| def __init__(self,cfg,weights): | |
| from diffusers import StableDiffusion3Pipeline | |
| self.cfg=cfg | |
| self.pipe=StableDiffusion3Pipeline.from_pretrained(cfg['repo'],torch_dtype=torch.bfloat16).to('cuda') | |
| self.pipe.set_progress_bar_config(disable=True) | |
| def decode(self,latents): | |
| config=self.pipe.vae.config | |
| return ((self.pipe.vae.decode(latents/config.scaling_factor+config.shift_factor,return_dict=False)[0]+1)/2).clamp(0,1) | |
| def native(self,prompt,seed): | |
| lat=self.pipe(prompt,height=self.cfg['resolution'],width=self.cfg['resolution'], | |
| num_inference_steps=self.cfg['steps'],guidance_scale=self.cfg['guidance'],output_type='latent', | |
| generator=torch.Generator(device='cuda').manual_seed(seed)).images | |
| return self.decode(lat),lat | |
| def reference(self,task): | |
| if task.get('reference_path'): | |
| image=pixels_image(task['reference_path'],self.cfg['resolution']) | |
| # mode, not sample: reference encoding has no additional RNG draw. | |
| lat=self.pipe.vae.encode((image*2-1).to(self.pipe.vae.dtype)).latent_dist.mode() | |
| config=self.pipe.vae.config | |
| return image,(lat-config.shift_factor)*config.scaling_factor | |
| return self.native(task['reference_prompt'],task.get('reference_seed',42)) | |
| def generate(self,job,ref,trace): | |
| mask=target_mask(ref.shape[-2:],job['task'])[None,None] | |
| prior=torch.randn(ref.shape,device='cuda',dtype=ref.dtype,generator=torch.Generator(device='cuda').manual_seed(job.get('prior_seed',1234))) | |
| counter=[0]; state={} | |
| def before(module,args,kw): | |
| x=kw['hidden_states']; t=kw['timestep'][0] | |
| scheduler=self.pipe.scheduler | |
| sigma=float(scheduler.sigmas[counter[0]]); p=1-sigma | |
| g=gate(mask*job['alpha'],p,1/job.get('steps',self.cfg['steps']),job.get('soft_release',True)).to(x.dtype) | |
| if job['alpha']>0 and job['arm']!='guide': x=x*(1-g)+(ref*(1-sigma)+prior*sigma)*g | |
| kw['hidden_states']=x; state.update(x=x,g=g,sigma=sigma) | |
| trace.append(dict(step=counter[0],progress=p,gate_mean=float(g.mean()),committed_fraction=0.,sigma=sigma)) | |
| counter[0]+=1 | |
| return args,kw | |
| def after(module,args,kw,out): | |
| if job['arm']!='guide' or job['alpha']==0: return out | |
| v=out[0] if isinstance(out,tuple) else out.sample | |
| sigma=state['sigma'] | |
| if sigma<=1e-8: return out | |
| x=state['x']; r=ref.expand_as(x) | |
| # x0=x−σv; guide clean prediction and convert back to velocity. | |
| g=state['g']*job.get('guide_amplitude',.1) | |
| pred=x-sigma*v; pred=pred+g*(r-pred); changed=(x-pred)/sigma | |
| if isinstance(out,tuple): return (changed,*out[1:]) | |
| out.sample=changed; return out | |
| handles=[self.pipe.transformer.register_forward_pre_hook(before,with_kwargs=True), | |
| self.pipe.transformer.register_forward_hook(after,with_kwargs=True)] | |
| original_step=self.pipe.scheduler.step | |
| def controlled_step(model_output,timestep,sample,*args,**kwargs): | |
| # Scheduler must integrate from the same pinned state seen by the model. | |
| if job['alpha']>0:sample=state['x'][-sample.shape[0]:] | |
| return original_step(model_output,timestep,sample,*args,**kwargs) | |
| self.pipe.scheduler.step=controlled_step | |
| try: | |
| lat=self.pipe(job['task']['prompt'],height=self.cfg['resolution'],width=self.cfg['resolution'], | |
| num_inference_steps=job.get('steps',self.cfg['steps']),guidance_scale=self.cfg['guidance'],output_type='latent', | |
| generator=torch.Generator(device='cuda').manual_seed(job['seed'])).images | |
| finally: | |
| for h in handles:h.remove() | |
| self.pipe.scheduler.step=original_step | |
| return self.decode(lat),lat,None,mask | |
| class SelfForcing: | |
| def __init__(self,cfg,weights): | |
| root=activate('self_forcing'); self.cfg=cfg | |
| src=Path(weights)/'self_forcing/wan_models/Wan2.1-T2V-1.3B' | |
| dest=root/'wan_models/Wan2.1-T2V-1.3B' | |
| dest.parent.mkdir(parents=True,exist_ok=True) | |
| if not dest.exists(): dest.symlink_to(src,target_is_directory=True) | |
| sdpa_flash_fallback() | |
| from omegaconf import OmegaConf | |
| from pipeline.causal_inference import CausalInferencePipeline | |
| config=OmegaConf.merge(OmegaConf.load(root/'configs/default_config.yaml'),OmegaConf.load(root/'configs/self_forcing_dmd.yaml')) | |
| self.pipe=CausalInferencePipeline(config,device=torch.device('cuda')).to(dtype=torch.bfloat16) | |
| ckpt=Path(weights)/'self_forcing'/cfg['student_file'] | |
| state=torch.load(ckpt,map_location='cpu',weights_only=False) | |
| self.pipe.generator.load_state_dict(state['generator_ema']) | |
| for model in [self.pipe.text_encoder,self.pipe.generator,self.pipe.vae]: model.cuda().eval() | |
| def native(self,prompt,seed): | |
| torch.manual_seed(seed) | |
| noise=torch.randn(1,self.cfg['latent_frames'],16,60,104,device='cuda',dtype=torch.bfloat16) | |
| return self.pipe.inference(noise,[prompt],return_latents=True) | |
| def reference(self,task): | |
| if task.get('reference_path'): | |
| import imageio.v3 as iio | |
| meta=iio.immeta(task['reference_path']) | |
| if 'fps' not in meta or abs(float(meta['fps'])-self.cfg['fps'])>.01: | |
| raise ValueError('Reference video must explicitly have native 16 fps') | |
| frames=iio.imread(task['reference_path']) | |
| expected=1+4*(self.cfg['latent_frames']-1) | |
| if frames.shape!=(expected,480,832,3): | |
| raise ValueError(f'Reference video must be [{expected},480,832,3], native frame rate; no silent resampling') | |
| p=torch.tensor(frames.copy(),device='cuda').permute(3,0,1,2)[None].to(torch.bfloat16)/127.5-1 | |
| lat=self.pipe.vae.encode_to_latent(p).to(torch.bfloat16) | |
| return ((p+1)/2).permute(0,2,1,3,4),lat | |
| return self.native(task['reference_prompt'],task.get('reference_seed',42)) | |
| def generate(self,job,ref,trace): | |
| mask=target_mask((ref.shape[1],),job['task'])[None,:,None,None,None] | |
| prior=torch.randn(ref.shape,device=ref.device,dtype=ref.dtype,generator=torch.Generator(device='cuda').manual_seed(job.get('prior_seed',1234))) | |
| state={}; steps=len(self.pipe.denoising_step_list) | |
| first=int((mask.reshape(-1)>0).nonzero()[0]); lead=torch.zeros_like(mask);lead[:,:first]=1 | |
| def before(module,args,kw): | |
| t=kw['timestep']; start=int(kw['current_start']//self.pipe.frame_seq_length) | |
| x=kw['noisy_image_or_video']; n=x.shape[1] | |
| # Never inject the separate clean-cache refresh evaluation. | |
| cache=float(t.flatten()[0])==float(self.pipe.args.context_noise) | |
| sched=self.pipe.scheduler | |
| ti=(sched.timesteps.to(t.device)-t.flatten()[0]).abs().argmin() | |
| sigma=float(sched.sigmas[ti]);p=1-sigma | |
| local_mask=mask[:,start:start+n] | |
| if job['arm']=='full_prefix': local_mask=torch.maximum(local_mask,lead[:,start:start+n]) | |
| g=gate(local_mask*job['alpha'],p,1/steps,job.get('soft_release',True)).to(x.dtype) | |
| r=ref[:,start:start+n] | |
| if not cache and job['alpha']>0: | |
| x=x*(1-g)+(r*(1-sigma)+prior[:,start:start+n]*sigma)*g | |
| kw['noisy_image_or_video']=x | |
| state.update(cache=cache,start=start,r=r,p=p,n=n) | |
| trace.append(dict(step=len(trace),progress=p,gate_mean=0. if cache else float(g.mean()),sigma=sigma, | |
| committed_fraction=start/ref.shape[1],block_start=start,block_frames=n, | |
| evaluation_kind='cache_refresh' if cache else 'denoise')) | |
| return args,kw | |
| def after(module,args,kw,out): | |
| if state['cache'] or job['arm']!='guide' or job['alpha']==0:return out | |
| flow,pred=out | |
| guide=gate(lead[:,state['start']:state['start']+state['n']]*job.get('guide_duration',.05)*job['alpha'],state['p'],1/steps) | |
| pred=pred+guide.to(pred.dtype)*job.get('guide_amplitude',.1)*(state['r']-pred) | |
| return flow,pred | |
| handles=[self.pipe.generator.register_forward_pre_hook(before,with_kwargs=True), | |
| self.pipe.generator.register_forward_hook(after,with_kwargs=True)] | |
| original_block=self.pipe.num_frame_per_block | |
| original_model_block=self.pipe.generator.model.num_frame_per_block | |
| try: | |
| block=job.get('block_frames',original_block) | |
| if ref.shape[1]%block:raise ValueError('Block stress size must divide native latent length') | |
| self.pipe.num_frame_per_block=block;self.pipe.generator.model.num_frame_per_block=block | |
| output,lat=self.native(job['task']['prompt'],job['seed']) | |
| if job.get('rollback_blocks'): | |
| # Reopen a suffix before the target; the official inference entry | |
| # resets all KV/cross-attention caches and rebuilds the clean prefix. | |
| boundary=(first//block)*block | |
| restart=max(0,boundary-job['rollback_blocks']*block) | |
| noise=torch.randn(ref.shape,device=ref.device,dtype=ref.dtype, | |
| generator=torch.Generator(device='cuda').manual_seed(job['seed']+100000)) | |
| output,lat=self.pipe.inference(noise[:,restart:],[job['task']['prompt']], | |
| initial_latent=lat[:,:restart] if restart else None,return_latents=True) | |
| for item in trace:item['rollback_restart']=restart | |
| finally: | |
| for h in handles:h.remove() | |
| self.pipe.num_frame_per_block=original_block | |
| self.pipe.generator.model.num_frame_per_block=original_model_block | |
| return output,lat,None,mask | |
| LOADERS={'meissonic':Meissonic,'momask':Momask,'sd35':SD35,'self_forcing':SelfForcing,'showo':Showo} | |