Excalibro's picture
download
raw
21.1 kB
"""Paired plain sparse12/6 -> fixed distilled shape6; evaluation only."""
import os, sys, json, hashlib, gc, time, subprocess, fcntl, signal
from pathlib import Path
import numpy as np
import torch
import torch.utils.checkpoint
import torch.nn as nn
import torch.nn.functional as F
from safetensors.torch import load_file
from huggingface_hub import hf_hub_download
ROOT=Path(os.environ['T2_ROOT'])
C=json.loads((ROOT/'config.json').read_text())
RUN=ROOT/'run'; OUT=RUN/'outputs'; INPUT=ROOT/'inputs'
for p in (RUN,OUT,INPUT): p.mkdir(parents=True,exist_ok=True)
sys.path.insert(0,str(ROOT/'TRELLIS.2'))
STOP=False
def stop(*args):
global STOP
STOP=True
print('Finishing this item and backing it up before stopping.',flush=True)
signal.signal(signal.SIGTERM,stop)
signal.signal(signal.SIGINT,stop)
def dump(x,p):
p=Path(p);p.parent.mkdir(parents=True,exist_ok=True)
q=p.with_suffix(p.suffix+'.tmp');q.write_text(json.dumps(x,indent=2));q.replace(p)
def save(x,p):
p=Path(p);p.parent.mkdir(parents=True,exist_ok=True)
q=p.with_suffix(p.suffix+'.tmp');torch.save(x,q);q.replace(p)
def read(p):return torch.load(p,map_location='cpu',weights_only=True)
def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest()
def hash_json(x):return hashlib.sha256(json.dumps(x,sort_keys=True).encode()).hexdigest()
def bridge(action,payload):
job=ROOT/'bucket_job.json';dump(payload,job)
subprocess.run([os.environ['T2_BUCKET_PY'],str(ROOT/'bucket_bridge.py'),action,str(job)],check=True)
return json.loads(job.with_suffix('.result.json').read_text())
def download(pairs):bridge('download',dict(bucket=C['bucket'],files=[[a,str(b)] for a,b in pairs]))
def upload(paths):
pairs=[[str(p),C['output_prefix']+'/'+Path(p).relative_to(RUN).as_posix()] for p in paths if Path(p).is_file()]
if pairs:bridge('upload',dict(bucket=C['bucket'],files=pairs))
def log(event,**kw):
x=dict(event=event,utc=time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),**kw)
print(json.dumps(x),flush=True)
with (OUT/'events.jsonl').open('a') as f:f.write(json.dumps(x)+'\n')
def clear():gc.collect();torch.cuda.empty_cache()
def seed(text):return int.from_bytes(hashlib.sha256(text.encode()).digest()[:4],'little') ^ C['seed']
def meta():return json.loads((INPUT/'shape/inference.json').read_text())
def manifest():return meta()['manifest']
def rows():return json.loads((RUN/'selection.json').read_text())['rows']
def cases():return [(r,k) for r in rows() for k in range(C['seeds_per_image'])]
def stem(r,k):return r['id']+'_seed'+str(k)
def cond(r):
d=read(INPUT/'native/conditions'/f"{r['id']}.pt")
expected=json.loads((RUN/'provenance.json').read_text())['native_signature']
if d.get('signature')!=expected:raise RuntimeError('Native conditioning provenance mismatch')
x=d['cond']
if x.ndim!=3 or x.shape[0]!=1 or x.shape[-1]!=1024 or not torch.isfinite(x).all():raise RuntimeError('Invalid condition')
return x.cuda().float()
def fetch():
bridge('restore',dict(bucket=C['bucket'],prefix=C['output_prefix'],root=str(RUN)))
sp=C['shape_prefix'];nprefix=C['native_prefix']
download([(sp+'/outputs/inference.json',INPUT/'shape/inference.json'),
(sp+'/outputs/best_adapter.safetensors',INPUT/'shape/best_adapter.safetensors'),
(sp+'/selection.json',INPUT/'shape/selection.json'),
(nprefix+'/selection.json',INPUT/'native/selection.json'),
(nprefix+'/identity.json',INPUT/'native/identity.json'),
(nprefix+'/manifest.json',INPUT/'native/manifest.json')])
m=meta();nm=json.loads((INPUT/'native/manifest.json').read_text())
if m['model_key']!='shape_slat_flow_model_512' or m['steps']!=6 or m['step']<=0 or m.get('adapter_scale')!=1:
raise RuntimeError('Expected a trained, unscaled SCFM shape6 adapter for the 512 flow')
if m['method']!='trellis2_scfm_fastslow_shape6_v1':raise RuntimeError('Unrecognized shape adapter recipe')
if manifest()['source_commit']!=C['source_commit'] or nm['source_commit']!=C['source_commit']:raise RuntimeError('Source revisions differ')
for key in ('sparse_structure_flow_model','sparse_structure_decoder'):
if manifest()['models'][key]!=nm['models'][key]:raise RuntimeError('Sparse model revisions differ')
selected=json.loads((INPUT/'native/selection.json').read_text())
rr=[r for r in selected['rows'] if r['split']=='test']
if len(rr)!=10 or len({r['id'] for r in rr})!=10:raise RuntimeError('Expected the fixed ten native test images')
identity=json.loads((INPUT/'native/identity.json').read_text())
sig=hash_json(dict(config=identity,selection=selected['rows'],manifest=nm))
shape_rows=json.loads((INPUT/'shape/selection.json').read_text())['rows']
seen={r['id']:r['split'] for r in shape_rows}
for r in rr:r['shape_training_overlap']=seen.get(r['id'],'not_in_shape_pilot')
prov=dict(config=C,shape_metadata=m,adapter_sha256=sha(INPUT/'shape/best_adapter.safetensors'),
native_signature=sig,native_manifest=nm,rows=rr,implementation_sha256=sha(__file__),
note='Paired mesh comparison, not new independent model selection. Native Facebook DINO conditions feed both branches; the older shape pilot used a timm encoder. This also tests that adapter transfer.')
p=RUN/'provenance.json'
if p.exists() and json.loads(p.read_text())!=prov:raise RuntimeError('Comparison inputs/settings changed. Choose a new RUN_NAME.')
dump(prov,p);dump(dict(rows=rr),RUN/'selection.json')
jobs=[]
for r in rr:
jobs.extend([(nprefix+'/'+r['image'],RUN/r['image']),
(nprefix+'/conditions/'+r['id']+'.pt',INPUT/'native/conditions'/f"{r['id']}.pt")])
download(jobs)
for name in ('compare.py','geometry_import.py','bucket_bridge.py','config.json'):
target=RUN/'code'/name;target.parent.mkdir(exist_ok=True);target.write_bytes((ROOT/name).read_bytes())
upload([p,RUN/'selection.json']+list((RUN/'code').iterdir())+[RUN/r['image'] for r in rr])
print('Ready: 10 fixed images x',C['seeds_per_image'],'seeds; shape checkpoint',m['step'],flush=True)
def load_model(key,dtype=None):
if key=='shape_slat_decoder':
import geometry_import
geometry_import.activate()
from trellis2 import models
d=manifest()['models'][key]
cfg=json.loads(Path(hf_hub_download(d['repo'],d['prefix']+'.json',revision=d['revision'])).read_text())
args=dict(cfg['args'])
if 'dtype' in args:args['dtype']='float32'
if key=='sparse_structure_decoder':args['use_fp16']=False
model=getattr(models,cfg['name'])(**args)
path=hf_hub_download(d['repo'],d['prefix']+'.safetensors',revision=d['revision'])
missing,extra=model.load_state_dict(load_file(path),strict=False)
if set(missing)-{'rope_phases','pos_emb'} or extra:raise RuntimeError(f'Weight mismatch: {missing}, {extra}')
if dtype is not None:model.convert_to(dtype)
if key=='sparse_structure_decoder':
model.float()
if hasattr(model,'convert_to_fp32'):model.convert_to_fp32()
for b in getattr(model,'blocks',[]):
if hasattr(b,'use_checkpoint'):b.use_checkpoint=False
return model.eval().requires_grad_(False).cuda()
def sampler(name):
from trellis2.pipelines import samplers
d=manifest()['pipeline']['args'][name]
return getattr(samplers,d['name'])(**d['args']),dict(d['params'])
def timed(fn):
torch.cuda.synchronize();torch.cuda.reset_peak_memory_stats();start=time.perf_counter()
y=fn();torch.cuda.synchronize()
return y,dict(seconds=time.perf_counter()-start,peak_allocated_gib=torch.cuda.max_memory_allocated()/2**30)
@torch.no_grad()
def structures():
model=load_model('sparse_structure_flow_model',torch.bfloat16)
sm,params=sampler('sparse_structure_sampler')
warm=False
for index,(r,k) in enumerate(cases()):
p=RUN/'structures'/f'{stem(r,k)}.pt'
if p.exists():upload([p]);continue
cg=cond(r)
noise=torch.randn((1,model.in_channels,model.resolution,model.resolution,model.resolution),generator=torch.Generator().manual_seed(seed(r['id']+f'/native/{k}'))).cuda()
def sample(n):return sm.sample(model,noise.clone(),cond=cg,neg_cond=torch.zeros_like(cg),**dict(params,steps=n),verbose=False).samples.float()
if not warm:
sample(12);torch.cuda.synchronize();warm=True
d=dict(id=r['id'],seed_index=k)
for n in ((12,6) if index%2==0 else (6,12)):
x,t=timed(lambda:sample(n))
if not torch.isfinite(x).all():raise RuntimeError('Nonfinite sparse output')
d['sparse'+str(n)]=dict(latent=x.cpu(),flow=t)
del x
save(d,p);upload([p]);log('sparse_pair',case=stem(r,k),seconds12=d['sparse12']['flow']['seconds'],seconds6=d['sparse6']['flow']['seconds'])
del d,noise,cg;clear()
if STOP:return
@torch.no_grad()
def coordinates():
model=load_model('sparse_structure_decoder')
for r,k in cases():
p=RUN/'coords'/f'{stem(r,k)}.pt'
if p.exists():upload([p]);continue
d=read(RUN/'structures'/f'{stem(r,k)}.pt');o=dict(id=r['id'],seed_index=k)
for name in ('sparse12','sparse6'):
x=d[name]['latent'].cuda().float()
def decode():
logits=model(x)
if not torch.isfinite(logits).all():raise RuntimeError('Nonfinite structure decoder output')
mask=logits>0
if mask.shape[2]!=32:
if mask.shape[2]%32:raise RuntimeError('Unexpected grid size')
ratio=mask.shape[2]//32;mask=F.max_pool3d(mask.float(),ratio,ratio)>.5
return torch.argwhere(mask)[:,[0,2,3,4]].int()
co,t=timed(decode)
if not 0<len(co)<=C['max_tokens']:raise RuntimeError(f'{stem(r,k)} {name}: {len(co)} voxels; limit {C["max_tokens"]}. No thinning or replacement performed.')
o[name]=dict(coords=co.cpu(),decode=t,flow=d[name]['flow'])
del x,co
save(o,p);upload([p]);log('coords_pair',case=stem(r,k),tokens12=len(o['sparse12']['coords']),tokens6=len(o['sparse6']['coords']))
clear()
if STOP:return
class ShapeLoRA(nn.Module):
def __init__(self,base,a,b):
super().__init__();self.base=base
self.register_buffer('a',a.to(base.weight.device).float())
self.register_buffer('b',b.to(base.weight.device).float())
def forward(self,x):
y=self.base(x)
return y+F.linear(F.linear(x.float(),self.a),self.b).to(y.dtype)
def install_shape(model,state):
expected=set();count=0
for name,module in list(model.named_modules()):
if type(module) is nn.Linear and name.startswith('blocks.') and any(t in name for t in ('.self_attn.','.cross_attn.')):
ka,kb=name+'.a',name+'.b';expected.update((ka,kb))
if ka not in state or kb not in state:raise RuntimeError('Missing shape adapter keys: '+name)
a,b=state[ka],state[kb]
if a.shape!=(meta()['rank'],module.in_features) or b.shape!=(module.out_features,meta()['rank']):raise RuntimeError('Adapter dimensions differ')
if not torch.isfinite(a).all() or not torch.isfinite(b).all():raise RuntimeError('Nonfinite shape adapter')
parent,leaf=name.rsplit('.',1);setattr(model.get_submodule(parent),leaf,ShapeLoRA(module,a,b));count+=1
if set(state)!=expected or not count:raise RuntimeError('Unexpected adapter keys or empty adapter')
if not any(bool(v.count_nonzero()) for n,v in state.items() if n.endswith('.b')):raise RuntimeError('Zero shape adapter')
print('Loaded fixed shape LoRA modules:',count,flush=True)
return model
def coordinate_noise(coords,channels,noise_seed):
# A shared dense noise field gives identical noise at every overlapping coordinate.
xyz=coords[:,1:].long().cpu()
index=(xyz[:,0]*32+xyz[:,1])*32+xyz[:,2]
dense=torch.randn((32**3,channels),generator=torch.Generator().manual_seed(noise_seed))
return dense[index]
@torch.no_grad()
def shapes():
from trellis2.modules.sparse import SparseTensor
model=install_shape(load_model('shape_slat_flow_model_512',torch.bfloat16),load_file(str(INPUT/'shape/best_adapter.safetensors')))
sm,params=sampler('shape_slat_sampler');params.update(meta()['sampler']);params['steps']=6
warm=False
for i,(r,k) in enumerate(cases()):
d=read(RUN/'coords'/f'{stem(r,k)}.pt');cg=cond(r)
for name in (('sparse12','sparse6') if i%2==0 else ('sparse6','sparse12')):
p=RUN/'shape_latents'/f'{stem(r,k)}_{name}.pt'
if p.exists():upload([p]);continue
coords=d[name]['coords'].cuda()
noise=coordinate_noise(coords,model.in_channels,seed(r['id']+f'/shape/{k}')).cuda()
def sample():
x=SparseTensor(feats=noise.clone(),coords=coords)
return sm.sample(model,x,cond=cg,neg_cond=torch.zeros_like(cg),**params,verbose=False).samples.feats.float()
if not warm:sample();torch.cuda.synchronize();warm=True
features,t=timed(sample)
if not torch.isfinite(features).all():raise RuntimeError('Nonfinite shape latent')
save(dict(coords=coords.cpu(),features=features.cpu(),shape=t,structure=d[name]),p);upload([p])
log('shape_done',case=stem(r,k),branch=name,**t)
del features,noise,coords;clear()
if STOP:return
@torch.no_grad()
def decode_one(case_name,branch):
# Separate process per mesh bounds allocator growth and releases all memory after OOM.
import geometry_import
geometry_import.activate()
from trellis2.modules.sparse import SparseTensor
import trimesh
p=RUN/'meshes'/case_name/branch;p.parent.mkdir(parents=True,exist_ok=True)
if p.with_suffix('.json').exists():return
d=read(RUN/'shape_latents'/f'{case_name}_{branch}.pt')
norm=manifest()['pipeline']['args']['shape_slat_normalization']
features=d['features'].cuda().float()*torch.tensor(norm['std'],device='cuda')+torch.tensor(norm['mean'],device='cuda')
model=load_model('shape_slat_decoder');model.set_resolution(512)
slat=SparseTensor(feats=features,coords=d['coords'].cuda().int())
meshes,t=timed(lambda:model(slat))
mesh=meshes[0]
v=mesh.vertices.float().cpu().numpy();f=mesh.faces.long().cpu().numpy()
if not len(v) or not len(f) or not np.isfinite(v).all():raise RuntimeError('Invalid or empty decoded mesh')
tm=trimesh.Trimesh(v,f,process=False)
temp=p.with_suffix('.tmp.glb');tm.export(temp);temp.replace(p.with_suffix('.glb'))
tm.export(p.with_suffix('.ply'))
info=dict(case=case_name,branch=branch,valid=True,vertices=len(v),faces=len(f),bounds=tm.bounds.tolist(),
shape=d['shape'],flow=d['structure']['flow'],structure_decode=d['structure']['decode'],mesh_decode=t)
dump(info,p.with_suffix('.json'));upload([p.with_suffix(ext) for ext in ('.json','.glb','.ply')])
log('mesh_done',**info)
def meshes():
for r,k in cases():
for branch in ('sparse12','sparse6'):
q=RUN/'meshes'/stem(r,k)/f'{branch}.json'
if q.exists():
upload([q,q.with_suffix('.glb'),q.with_suffix('.ply')]);continue
rc=subprocess.run([sys.executable,__file__,'decode_one',stem(r,k),branch]).returncode
if rc:
log('decode_failed',case=stem(r,k),branch=branch,returncode=rc)
upload([OUT/'events.jsonl'])
raise RuntimeError('Mesh decode failed. Completed GLBs are backed up. Keep this runtime; rerun the mesh cell after resolving the error. No lower-resolution substitution.')
if STOP:return
def surface_points(mesh,n,sd):
rng=np.random.default_rng(sd);tri=np.asarray(mesh.triangles)
areas=np.asarray(mesh.area_faces);total=areas.sum()
if not np.isfinite(total) or total<=0:raise RuntimeError('Zero mesh surface area')
t=tri[rng.choice(len(tri),n,p=areas/total)]
uv=rng.random((n,2));u=np.sqrt(uv[:,0]);v=uv[:,1]
return (1-u[:,None])*t[:,0]+(u*(1-v))[:,None]*t[:,1]+(u*v)[:,None]*t[:,2]
def compare_surfaces(a,b,n,sd):
from scipy.spatial import cKDTree
pa=surface_points(a,n,sd);pb=surface_points(b,n,sd+1)
da=cKDTree(pb).query(pa,workers=-1)[0];db=cKDTree(pa).query(pb,workers=-1)[0]
diag=max(float(np.linalg.norm(a.bounds[1]-a.bounds[0])),1e-8)
recall=float(np.mean(da<.01*diag));precision=float(np.mean(db<.01*diag))
return dict(surface_chamfer_relative=float((da.mean()+db.mean())/(2*diag)),
surface_p95_relative=float(max(np.quantile(da,.95),np.quantile(db,.95))/diag),
fscore_at_1pct=2*precision*recall/max(precision+recall,1e-12),
precision_at_1pct=precision,recall_at_1pct=recall)
def make_preview(a,b,target):
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
fig=plt.figure(figsize=(10,6),facecolor='white')
# Uniform area samples preserve surfaces and holes better than face-order subsampling.
for row,(mesh,title) in enumerate(((a,'Sparse12 + distilled shape6'),(b,'Sparse6 + distilled shape6'))):
points=surface_points(mesh,45000,123)
for col,(az,el) in enumerate(((35,20),(125,20),(35,80))):
ax=fig.add_subplot(2,3,row*3+col+1,projection='3d')
ax.scatter(*points.T,s=.1,c=points[:,2],cmap='bone',rasterized=True)
ax.set(xlim=(-.55,.55),ylim=(-.55,.55),zlim=(-.55,.55));ax.set_box_aspect((1,1,1));ax.view_init(el,az);ax.axis('off')
if col==1:ax.set_title(title,fontsize=10)
fig.tight_layout();fig.savefig(target,dpi=150);plt.close(fig)
def report():
import trimesh
records=[]
for r,k in cases():
folder=RUN/'meshes'/stem(r,k)
a=trimesh.load(folder/'sparse12.glb',force='mesh',process=False)
b=trimesh.load(folder/'sparse6.glb',force='mesh',process=False)
metric=compare_surfaces(a,b,C['surface_samples'],seed(stem(r,k)))
# Independent samples from the reference quantify the point-sampling noise floor.
floor=compare_surfaces(a,a,C['surface_samples'],seed(stem(r,k)))
timings={}
for branch in ('sparse12','sparse6'):
t=json.loads((folder/f'{branch}.json').read_text())
timings[branch]=dict(flow_seconds=t['flow']['seconds'],shape_seconds=t['shape']['seconds'],
structure_decode_seconds=t['structure_decode']['seconds'],mesh_decode_seconds=t['mesh_decode']['seconds'],
summed_compute_seconds=sum(t[key]['seconds'] for key in ('flow','shape','structure_decode','mesh_decode')),
vertices=t['vertices'],faces=t['faces'])
rec=dict(id=r['id'],seed_index=k,**metric,reference_sampling_floor=floor,timings=timings,
shape_training_overlap=r['shape_training_overlap'])
records.append(rec)
preview=folder/'comparison.png'
if not preview.exists():make_preview(a,b,preview)
dump(rec,folder/'comparison.json');upload([preview,folder/'comparison.json'])
mean=lambda key:float(np.mean([r[key] for r in records]))
t12=sum(r['timings']['sparse12']['summed_compute_seconds'] for r in records)
t6=sum(r['timings']['sparse6']['summed_compute_seconds'] for r in records)
summary=dict(images=len(rows()),pairs=len(records),meshes=2*len(records),shape_step=meta()['step'],
mean_surface_chamfer_relative=mean('surface_chamfer_relative'),mean_fscore_at_1pct=mean('fscore_at_1pct'),
worst_surface_p95_relative=max(r['surface_p95_relative'] for r in records),
sparse_flow_speedup=sum(r['timings']['sparse12']['flow_seconds'] for r in records)/sum(r['timings']['sparse6']['flow_seconds'] for r in records),
summed_compute_speedup=t12/t6,mean_summed_compute_sparse12=t12/len(records),mean_summed_compute_sparse6=t6/len(records),
status='REQUIRES_VISUAL_REVIEW',
note='Distances measure agreement with sparse12+shape6, not ground truth. No automatic mesh-quality pass. Surface metrics use area samples; reference_sampling_floor measures sampling error. Timings are synchronized compute, excluding model loads, conditions, uploads and exports. Per-mesh decoder timing includes first-use kernel overhead. No texture, Q4, or mobile inference tested.')
dump(dict(summary=summary,records=records),OUT/'comparison_report.json');upload([OUT/'comparison_report.json',OUT/'events.jsonl'])
print(json.dumps(summary,indent=2),flush=True)
if __name__=='__main__':
action=sys.argv[1]
# The decode child is serialized by its locked parent.
if action!='decode_one':
lock=(ROOT/'.compare.lock').open('w')
try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
except BlockingIOError:raise RuntimeError('Another comparison worker is still running')
if action=='decode_one':decode_one(*sys.argv[2:])
else:
dict(fetch=fetch,structures=structures,coordinates=coordinates,shapes=shapes,meshes=meshes,report=report)[action]()
if STOP:raise SystemExit('Paused safely; rerun the same cell to finish before continuing.')

Xet Storage Details

Size:
21.1 kB
·
Xet hash:
34eb92e641f2e6a9437c4c8b2a6ef457dc96f34d68e3e7c88bd074f5dacdab87

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.