CharlesZhang-USTC's picture
Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified
Raw History Blame Contribute Delete
23.3 kB
"""Real-model loaders; one isolated process per upstream module namespace.
Nothing here fabricates outputs. Hardware/model failures are propagated.
"""
import os
import sys
import math
from pathlib import Path
from types import SimpleNamespace
import numpy as np
import torch
from PIL import Image
from .design import ROOT
from .control import gate, target_mask, masked_decode
def activate(model):
path=ROOT/'vendor'/model
sys.path.insert(0,str(path)); os.chdir(path)
return path
def pixels_image(path, resolution):
image=Image.open(path).convert('RGB')
if image.size!=(resolution,resolution):
raise ValueError('Reference resolution mismatch; convert offline explicitly')
return torch.from_numpy(np.asarray(image).copy()).permute(2,0,1)[None].float().cuda()/255
def sdpa_flash_fallback():
"""Replace Wan's flash_attention explicitly, including length masking.
This is an alternate attention backend, not an unmodified upstream claim.
Reject unsupported windows instead of silently ignoring them.
"""
from wan.modules import attention as mod
def fallback(q,k,v,q_lens=None,k_lens=None,dropout_p=0.,softmax_scale=None,
q_scale=None,causal=False,window_size=(-1,-1),deterministic=False,
dtype=torch.bfloat16,version=None):
if window_size!=(-1,-1):
raise ValueError('SDPA fallback does not implement sliding attention windows')
b,lq,_,_=q.shape; lk=k.shape[1]
q=q.transpose(1,2); k=k.transpose(1,2); v=v.transpose(1,2)
if q_scale is not None: q=q*q_scale
allow=None
if k_lens is not None:
allow=torch.arange(lk,device=k.device)[None,None,None,:]<k_lens[:,None,None,None]
if causal:
# FlashAttention uses bottom-right alignment for unequal lengths.
tri=torch.arange(lk,device=q.device)[None,:] <= (torch.arange(lq,device=q.device)[:,None]+lk-lq)
allow=tri[None,None] if allow is None else allow & tri[None,None]
out=torch.nn.functional.scaled_dot_product_attention(q,k,v,attn_mask=allow,
dropout_p=dropout_p,scale=softmax_scale).transpose(1,2)
if q_lens is not None:
valid=torch.arange(lq,device=out.device)[None,:]<q_lens[:,None]
out=out*valid[:,:,None,None]
return out
mod.flash_attention=fallback
def attention(*args,fa_version=None,**kwargs):return fallback(*args,version=fa_version,**kwargs)
mod.attention=attention
# wan.__init__ eagerly imports modules before this replacement.
for name,module in list(sys.modules.items()):
if name.startswith('wan.modules.') and name != mod.__name__ and hasattr(module,'attention'):
module.attention=attention
class Meissonic:
def __init__(self,cfg,weights):
activate('meissonic')
from src.transformer import Transformer2DModel
from src.pipeline import Pipeline
from src.scheduler import Scheduler
from transformers import CLIPTextModelWithProjection,CLIPTokenizer
from diffusers import VQModel
self.cfg=cfg
model=Transformer2DModel.from_pretrained(cfg['repo'],subfolder='transformer',torch_dtype=torch.float16)
vq=VQModel.from_pretrained(cfg['repo'],subfolder='vqvae')
text=CLIPTextModelWithProjection.from_pretrained(cfg['text_repo'],torch_dtype=torch.float16)
self.pipe=Pipeline(vq,tokenizer=CLIPTokenizer.from_pretrained(cfg['repo'],subfolder='tokenizer'),
text_encoder=text,transformer=model,scheduler=Scheduler.from_pretrained(cfg['repo'],subfolder='scheduler')).to('cuda')
self.pipe.set_progress_bar_config(disable=True)
def decode(self,tokens):
h=w=self.cfg['resolution']//self.pipe.vae_scale_factor
return self.pipe.vqvae.decode(tokens.reshape(1,h,w),force_not_quantize=True,
shape=(1,h,w,self.pipe.vqvae.config.latent_channels)).sample.clamp(0,1)
@torch.no_grad()
def native(self,prompt,seed):
tokens=self.pipe(prompt=prompt,height=self.cfg['resolution'],width=self.cfg['resolution'],
guidance_scale=self.cfg['guidance'],num_inference_steps=self.cfg['steps'],output_type='latent',
generator=torch.Generator(device='cuda').manual_seed(seed)).images
return self.decode(tokens),tokens
def reference(self,task):
if task.get('reference_path'):
p=pixels_image(task['reference_path'],self.cfg['resolution'])
lat=self.pipe.vqvae.encode(p).latents
tokens=self.pipe.vqvae.quantize(lat)[2][2].reshape(1,*lat.shape[-2:])
return p,tokens
return self.native(task['reference_prompt'],task.get('reference_seed',42))
@torch.no_grad()
def generate(self,job,ref,trace):
h,w=ref.shape[-2:]; mask=target_mask((h,w),job['task'])
holder={}
# Capture the upstream first forward's static conditioning; abort before sampling.
class Captured(Exception): pass
def capture(module,args,kw):
holder.update(kw); raise Captured()
handle=self.pipe.transformer.register_forward_pre_hook(capture,with_kwargs=True)
try:
self.pipe(prompt=job['task']['prompt'],height=self.cfg['resolution'],width=self.cfg['resolution'],
num_inference_steps=job.get('steps',self.cfg['steps']),guidance_scale=self.cfg['guidance'])
except Captured:
pass
finally: handle.remove()
steps=job.get('steps',self.cfg['steps']); calls=[0]
def predict(ids):
kw=dict(holder); x=ids.reshape(1,h,w)
kw['hidden_states']=torch.cat([x,x])
kw['timestep']=torch.tensor([steps-1-calls[0]],device=x.device,dtype=torch.long)
calls[0]+=1
output=self.pipe.transformer(**kw)
u,c=output.chunk(2)
logits=u+self.cfg['guidance']*(c-u)
return logits.reshape(1,logits.shape[1],-1).permute(0,2,1)
tokens,commits=masked_decode(predict,ref,mask,self.pipe.scheduler.config.mask_token_id,steps,job,trace)
return self.decode(tokens),tokens.reshape(1,h,w),commits,mask
class Showo:
def __init__(self,cfg,weights):
root=activate('showo'); self.cfg=cfg
from models import Showo as Net,MAGVITv2
from transformers import AutoTokenizer
from training.prompting_utils import UniversalPrompting,create_attention_mask_predict_next
from omegaconf import OmegaConf
self.config=OmegaConf.load(root/'configs/showo_demo.yaml')
self.model=Net.from_pretrained(cfg['repo'],llm_model_path=cfg['text_repo']).eval().cuda()
self.vq=MAGVITv2.from_pretrained(cfg['vq_repo']).eval().cuda()
tokenizer=AutoTokenizer.from_pretrained(cfg['text_repo'],padding_side='left')
self.prompting=UniversalPrompting(tokenizer,max_text_len=128,
special_tokens=('<|soi|>','<|eoi|>','<|sov|>','<|eov|>','<|t2i|>','<|mmu|>','<|t2v|>','<|v2v|>','<|lvg|>'),ignore_id=-100,cond_dropout_prob=.1)
self.attention_fn=create_attention_mask_predict_next
self.offset=self.config.model.showo.llm_vocab_size+self.config.model.showo.num_new_special_tokens
self.n=self.config.model.showo.num_vq_tokens
def inputs(self,prompt,ids):
v=torch.where(ids==self.model.config.mask_token_id,ids,ids+self.offset)
cond,_=self.prompting(([prompt],v),'t2i_gen'); uncond,_=self.prompting(([''],v),'t2i_gen')
return cond,uncond
def predict(self,prompt,ids):
cond,uncond=self.inputs(prompt,ids); both=torch.cat([cond,uncond])
d=self.prompting.sptids_dict
mask=self.attention_fn(both,pad_id=int(d['<|pad|>']),soi_id=int(d['<|soi|>']),eoi_id=int(d['<|eoi|>']),rm_pad_in_image=True)
c,u=self.model(both,attention_mask=mask).chunk(2)
logits=(1+self.cfg['guidance'])*c-self.cfg['guidance']*u
return logits[:,-(self.n+1):-1,self.offset:-1]
def decode(self,tokens):
image=self.vq.decode_code(tokens.reshape(1,-1))
return ((image+1)/2).clamp(0,1)
@torch.no_grad()
def native(self,prompt,seed):
from models import get_mask_chedule
torch.manual_seed(seed); ids=torch.full((1,self.n),self.model.config.mask_token_id,device='cuda',dtype=torch.long)
cond,uncond=self.inputs(prompt,ids); d=self.prompting.sptids_dict
attention=self.attention_fn(torch.cat([cond,uncond]),pad_id=int(d['<|pad|>']),soi_id=int(d['<|soi|>']),eoi_id=int(d['<|eoi|>']),rm_pad_in_image=True)
tokens=self.model.t2i_generate(input_ids=cond,uncond_input_ids=uncond,attention_mask=attention,
guidance_scale=self.cfg['guidance'],timesteps=self.cfg['steps'],noise_schedule=get_mask_chedule('cosine'),
generator=torch.Generator(device='cuda').manual_seed(seed),config=self.config)
return self.decode(tokens),tokens
def reference(self,task):
if task.get('reference_path'):
image=pixels_image(task['reference_path'],self.cfg['resolution'])
return image,self.vq.get_code(image*2-1)
return self.native(task['reference_prompt'],task.get('reference_seed',42))
@torch.no_grad()
def generate(self,job,ref,trace):
h=int(math.sqrt(self.n)); mask=target_mask((h,h),job['task'])
tokens,commits=masked_decode(lambda ids:self.predict(job['task']['prompt'],ids),ref,mask,
self.model.config.mask_token_id,job.get('steps',self.cfg['steps']),job,trace)
return self.decode(tokens),tokens,commits,mask
class Momask:
def __init__(self,cfg,weights):
activate('momask'); self.cfg=cfg
from .momask_loading import loading_functions
load_vq_model,load_trans_model,load_res_model=loading_functions()
from utils.get_opt import get_opt
opt=SimpleNamespace(checkpoints_dir=str(Path(weights)/'momask/checkpoints'),dataset_name='t2m',
name=cfg['name'],res_name=cfg['res_name'],device=torch.device('cuda'))
model_opt=get_opt(str(Path(opt.checkpoints_dir)/'t2m'/opt.name/'opt.txt'),device=opt.device,checkpoints_dir=opt.checkpoints_dir)
vq_opt=get_opt(str(Path(opt.checkpoints_dir)/'t2m'/model_opt.vq_name/'opt.txt'),device=opt.device,checkpoints_dir=opt.checkpoints_dir)
vq_opt.dim_pose=263
self.vq,vq_opt=load_vq_model(vq_opt)
model_opt.num_tokens=vq_opt.nb_code; model_opt.num_quantizers=vq_opt.num_quantizers; model_opt.code_dim=vq_opt.code_dim
res_opt=get_opt(str(Path(opt.checkpoints_dir)/'t2m'/opt.res_name/'opt.txt'),device=opt.device,checkpoints_dir=opt.checkpoints_dir)
self.res=load_res_model(res_opt,vq_opt,opt)
self.model=load_trans_model(model_opt,opt,'latest.tar')
for module in [self.vq,self.res,self.model]: module.cuda().eval()
meta=Path(opt.checkpoints_dir)/'t2m'/model_opt.vq_name/'meta'
self.mean=torch.tensor(np.load(meta/'mean.npy'),device='cuda',dtype=torch.float32)
self.std=torch.tensor(np.load(meta/'std.npy'),device='cuda',dtype=torch.float32)
self.lens=torch.tensor([cfg['frames']//4],device='cuda',dtype=torch.long)
def decode(self,tokens,prompt,seed):
from utils.motion_process import recover_from_ric
torch.manual_seed(seed+999)
residual=self.res.generate(tokens,[prompt],self.lens,temperature=1,cond_scale=5)
normalized=self.vq.forward_decoder(residual)
features=normalized*self.std+self.mean
joints=recover_from_ric(features.float(),22)
return joints,features
@torch.no_grad()
def native(self,prompt,seed):
torch.manual_seed(seed)
tokens=self.model.generate([prompt],self.lens,timesteps=self.cfg['steps'],cond_scale=self.cfg['guidance'])
joints,features=self.decode(tokens,prompt,seed)
return joints,tokens
def reference(self,task):
if task.get('reference_path'):
# Explicit format: denormalized HumanML3D 263D features, NOT XYZ joints.
data=np.load(task['reference_path'],allow_pickle=False)
if data.shape!=(self.cfg['frames'],263): raise ValueError('MoMask reference must be [196,263] HumanML3D features')
f=torch.tensor(data,device='cuda',dtype=torch.float32)[None]
tokens,_=self.vq.encode((f-self.mean)/self.std)
from utils.motion_process import recover_from_ric
return recover_from_ric(f,22),tokens[:,:,0]
return self.native(task['reference_prompt'],task.get('reference_seed',42))
@torch.no_grad()
def generate(self,job,ref,trace):
vector=self.model.encode_text([job['task']['prompt']]); mask=target_mask((ref.shape[-1],),job['task'])
padding=torch.zeros_like(ref,dtype=torch.bool)
def predict(ids):
logits=self.model.forward_with_cond_scale(ids,cond_vector=vector,padding_mask=padding,cond_scale=self.cfg['guidance'])
return logits.permute(0,2,1)
tokens,commits=masked_decode(predict,ref,mask,self.model.mask_id,job.get('steps',self.cfg['steps']),job,trace)
joints,features=self.decode(tokens,job['task']['prompt'],job['seed'])
return joints,features,commits,mask
class SD35:
def __init__(self,cfg,weights):
from diffusers import StableDiffusion3Pipeline
self.cfg=cfg
self.pipe=StableDiffusion3Pipeline.from_pretrained(cfg['repo'],torch_dtype=torch.bfloat16).to('cuda')
self.pipe.set_progress_bar_config(disable=True)
def decode(self,latents):
config=self.pipe.vae.config
return ((self.pipe.vae.decode(latents/config.scaling_factor+config.shift_factor,return_dict=False)[0]+1)/2).clamp(0,1)
@torch.no_grad()
def native(self,prompt,seed):
lat=self.pipe(prompt,height=self.cfg['resolution'],width=self.cfg['resolution'],
num_inference_steps=self.cfg['steps'],guidance_scale=self.cfg['guidance'],output_type='latent',
generator=torch.Generator(device='cuda').manual_seed(seed)).images
return self.decode(lat),lat
def reference(self,task):
if task.get('reference_path'):
image=pixels_image(task['reference_path'],self.cfg['resolution'])
# mode, not sample: reference encoding has no additional RNG draw.
lat=self.pipe.vae.encode((image*2-1).to(self.pipe.vae.dtype)).latent_dist.mode()
config=self.pipe.vae.config
return image,(lat-config.shift_factor)*config.scaling_factor
return self.native(task['reference_prompt'],task.get('reference_seed',42))
@torch.no_grad()
def generate(self,job,ref,trace):
mask=target_mask(ref.shape[-2:],job['task'])[None,None]
prior=torch.randn(ref.shape,device='cuda',dtype=ref.dtype,generator=torch.Generator(device='cuda').manual_seed(job.get('prior_seed',1234)))
counter=[0]; state={}
def before(module,args,kw):
x=kw['hidden_states']; t=kw['timestep'][0]
scheduler=self.pipe.scheduler
sigma=float(scheduler.sigmas[counter[0]]); p=1-sigma
g=gate(mask*job['alpha'],p,1/job.get('steps',self.cfg['steps']),job.get('soft_release',True)).to(x.dtype)
if job['alpha']>0 and job['arm']!='guide': x=x*(1-g)+(ref*(1-sigma)+prior*sigma)*g
kw['hidden_states']=x; state.update(x=x,g=g,sigma=sigma)
trace.append(dict(step=counter[0],progress=p,gate_mean=float(g.mean()),committed_fraction=0.,sigma=sigma))
counter[0]+=1
return args,kw
def after(module,args,kw,out):
if job['arm']!='guide' or job['alpha']==0: return out
v=out[0] if isinstance(out,tuple) else out.sample
sigma=state['sigma']
if sigma<=1e-8: return out
x=state['x']; r=ref.expand_as(x)
# x0=x−σv; guide clean prediction and convert back to velocity.
g=state['g']*job.get('guide_amplitude',.1)
pred=x-sigma*v; pred=pred+g*(r-pred); changed=(x-pred)/sigma
if isinstance(out,tuple): return (changed,*out[1:])
out.sample=changed; return out
handles=[self.pipe.transformer.register_forward_pre_hook(before,with_kwargs=True),
self.pipe.transformer.register_forward_hook(after,with_kwargs=True)]
original_step=self.pipe.scheduler.step
def controlled_step(model_output,timestep,sample,*args,**kwargs):
# Scheduler must integrate from the same pinned state seen by the model.
if job['alpha']>0:sample=state['x'][-sample.shape[0]:]
return original_step(model_output,timestep,sample,*args,**kwargs)
self.pipe.scheduler.step=controlled_step
try:
lat=self.pipe(job['task']['prompt'],height=self.cfg['resolution'],width=self.cfg['resolution'],
num_inference_steps=job.get('steps',self.cfg['steps']),guidance_scale=self.cfg['guidance'],output_type='latent',
generator=torch.Generator(device='cuda').manual_seed(job['seed'])).images
finally:
for h in handles:h.remove()
self.pipe.scheduler.step=original_step
return self.decode(lat),lat,None,mask
class SelfForcing:
def __init__(self,cfg,weights):
root=activate('self_forcing'); self.cfg=cfg
src=Path(weights)/'self_forcing/wan_models/Wan2.1-T2V-1.3B'
dest=root/'wan_models/Wan2.1-T2V-1.3B'
dest.parent.mkdir(parents=True,exist_ok=True)
if not dest.exists(): dest.symlink_to(src,target_is_directory=True)
sdpa_flash_fallback()
from omegaconf import OmegaConf
from pipeline.causal_inference import CausalInferencePipeline
config=OmegaConf.merge(OmegaConf.load(root/'configs/default_config.yaml'),OmegaConf.load(root/'configs/self_forcing_dmd.yaml'))
self.pipe=CausalInferencePipeline(config,device=torch.device('cuda')).to(dtype=torch.bfloat16)
ckpt=Path(weights)/'self_forcing'/cfg['student_file']
state=torch.load(ckpt,map_location='cpu',weights_only=False)
self.pipe.generator.load_state_dict(state['generator_ema'])
for model in [self.pipe.text_encoder,self.pipe.generator,self.pipe.vae]: model.cuda().eval()
@torch.no_grad()
def native(self,prompt,seed):
torch.manual_seed(seed)
noise=torch.randn(1,self.cfg['latent_frames'],16,60,104,device='cuda',dtype=torch.bfloat16)
return self.pipe.inference(noise,[prompt],return_latents=True)
def reference(self,task):
if task.get('reference_path'):
import imageio.v3 as iio
meta=iio.immeta(task['reference_path'])
if 'fps' not in meta or abs(float(meta['fps'])-self.cfg['fps'])>.01:
raise ValueError('Reference video must explicitly have native 16 fps')
frames=iio.imread(task['reference_path'])
expected=1+4*(self.cfg['latent_frames']-1)
if frames.shape!=(expected,480,832,3):
raise ValueError(f'Reference video must be [{expected},480,832,3], native frame rate; no silent resampling')
p=torch.tensor(frames.copy(),device='cuda').permute(3,0,1,2)[None].to(torch.bfloat16)/127.5-1
lat=self.pipe.vae.encode_to_latent(p).to(torch.bfloat16)
return ((p+1)/2).permute(0,2,1,3,4),lat
return self.native(task['reference_prompt'],task.get('reference_seed',42))
@torch.no_grad()
def generate(self,job,ref,trace):
mask=target_mask((ref.shape[1],),job['task'])[None,:,None,None,None]
prior=torch.randn(ref.shape,device=ref.device,dtype=ref.dtype,generator=torch.Generator(device='cuda').manual_seed(job.get('prior_seed',1234)))
state={}; steps=len(self.pipe.denoising_step_list)
first=int((mask.reshape(-1)>0).nonzero()[0]); lead=torch.zeros_like(mask);lead[:,:first]=1
def before(module,args,kw):
t=kw['timestep']; start=int(kw['current_start']//self.pipe.frame_seq_length)
x=kw['noisy_image_or_video']; n=x.shape[1]
# Never inject the separate clean-cache refresh evaluation.
cache=float(t.flatten()[0])==float(self.pipe.args.context_noise)
sched=self.pipe.scheduler
ti=(sched.timesteps.to(t.device)-t.flatten()[0]).abs().argmin()
sigma=float(sched.sigmas[ti]);p=1-sigma
local_mask=mask[:,start:start+n]
if job['arm']=='full_prefix': local_mask=torch.maximum(local_mask,lead[:,start:start+n])
g=gate(local_mask*job['alpha'],p,1/steps,job.get('soft_release',True)).to(x.dtype)
r=ref[:,start:start+n]
if not cache and job['alpha']>0:
x=x*(1-g)+(r*(1-sigma)+prior[:,start:start+n]*sigma)*g
kw['noisy_image_or_video']=x
state.update(cache=cache,start=start,r=r,p=p,n=n)
trace.append(dict(step=len(trace),progress=p,gate_mean=0. if cache else float(g.mean()),sigma=sigma,
committed_fraction=start/ref.shape[1],block_start=start,block_frames=n,
evaluation_kind='cache_refresh' if cache else 'denoise'))
return args,kw
def after(module,args,kw,out):
if state['cache'] or job['arm']!='guide' or job['alpha']==0:return out
flow,pred=out
guide=gate(lead[:,state['start']:state['start']+state['n']]*job.get('guide_duration',.05)*job['alpha'],state['p'],1/steps)
pred=pred+guide.to(pred.dtype)*job.get('guide_amplitude',.1)*(state['r']-pred)
return flow,pred
handles=[self.pipe.generator.register_forward_pre_hook(before,with_kwargs=True),
self.pipe.generator.register_forward_hook(after,with_kwargs=True)]
original_block=self.pipe.num_frame_per_block
original_model_block=self.pipe.generator.model.num_frame_per_block
try:
block=job.get('block_frames',original_block)
if ref.shape[1]%block:raise ValueError('Block stress size must divide native latent length')
self.pipe.num_frame_per_block=block;self.pipe.generator.model.num_frame_per_block=block
output,lat=self.native(job['task']['prompt'],job['seed'])
if job.get('rollback_blocks'):
# Reopen a suffix before the target; the official inference entry
# resets all KV/cross-attention caches and rebuilds the clean prefix.
boundary=(first//block)*block
restart=max(0,boundary-job['rollback_blocks']*block)
noise=torch.randn(ref.shape,device=ref.device,dtype=ref.dtype,
generator=torch.Generator(device='cuda').manual_seed(job['seed']+100000))
output,lat=self.pipe.inference(noise[:,restart:],[job['task']['prompt']],
initial_latent=lat[:,:restart] if restart else None,return_latents=True)
for item in trace:item['rollback_restart']=restart
finally:
for h in handles:h.remove()
self.pipe.num_frame_per_block=original_block
self.pipe.generator.model.num_frame_per_block=original_model_block
return output,lat,None,mask
LOADERS={'meissonic':Meissonic,'momask':Momask,'sd35':SD35,'self_forcing':SelfForcing,'showo':Showo}