File size: 11,916 Bytes
434f330
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
"""Fresh training of the one shared fuse-then-gaze architecture. Validation only."""
from pathlib import Path
import argparse,csv,hashlib,json,math,random,time
import copy
import numpy as np
from PIL import Image
import torch
import torch.nn.functional as F
from torch.utils.data import DataLoader
from model import TransposeCNNGazePyramidDecoder,segmentation_loss
from data import FeatureImageDataset,collate,load_torch


def emit(x):print(json.dumps(x),flush=True)
def atomic_json(path,value):
    temp=path.with_suffix('.tmp');temp.write_text(json.dumps(value,indent=2));temp.replace(path)
def patient(name):return name.rsplit('_',1)[0]
def fingerprint(model):
    digest=hashlib.sha256()
    for name,p in model.state_dict().items():digest.update(name.encode());digest.update(p.detach().cpu().contiguous().numpy().tobytes())
    return digest.hexdigest()
def inputs(batch):
    return (batch['image'].cuda(non_blocking=True).float(),
            [f.cuda(non_blocking=True) for f in batch['features']],batch['gaze'].cuda(non_blocking=True).float())
def dice_iou(probability,target):
    p=probability>.5;t=target>.5;inter=(p&t).sum().item()
    return (2*inter+1e-6)/(p.sum().item()+t.sum().item()+1e-6),(inter+1e-6)/((p|t).sum().item()+1e-6)


@torch.no_grad()
def evaluate(model,loader,masks=None):
    model.eval();records=[]
    for batch in loader:
        probabilities=model(*inputs(batch)).sigmoid().float().cpu()
        for i,name in enumerate(batch['id']):
            native=batch['native'][i]
            p=F.interpolate(probabilities[i:i+1],size=native.shape[-2:],mode='bilinear',align_corners=False)[0]
            dice,iou=dice_iou(p,native)
            records.append({'id':name,'patient':patient(name),'dice':dice,'iou':iou})
            if masks is not None:Image.fromarray(((p[0]>.5).numpy()*255).astype(np.uint8)).save(masks/(name+'.png'))
    return records


def summary(records):
    result={'n':len(records)}
    for key in ['dice','iou']:
        values=np.array([r[key] for r in records])*100
        result[key+'_mean']=float(values.mean());result[key+'_case_sd']=float(values.std(ddof=1))
    return result


def main():
    ap=argparse.ArgumentParser()
    ap.add_argument('--dataset',choices=['kvasir','prostate'],required=True)
    ap.add_argument('--manifest',required=True);ap.add_argument('--existing-cache',required=True)
    ap.add_argument('--sigma-px',type=float,required=True);ap.add_argument('--output',required=True)
    ap.add_argument('--seed',type=int,default=42);ap.add_argument('--epochs',type=int,default=80)
    ap.add_argument('--patience',type=int,default=15);ap.add_argument('--batch-size',type=int,default=4)
    ap.add_argument('--accumulation',type=int,default=2);ap.add_argument('--lr',type=float,default=2e-4)
    args=ap.parse_args();assert args.seed==42 and torch.cuda.is_available()
    torch.set_num_threads(4);random.seed(args.seed);np.random.seed(args.seed)
    torch.manual_seed(args.seed);torch.cuda.manual_seed_all(args.seed)
    torch.backends.cudnn.benchmark=False;torch.backends.cudnn.deterministic=True;torch.backends.cuda.matmul.allow_tf32=True
    root=Path(args.output);root.mkdir(parents=True,exist_ok=True);assert not (root/'settings.json').exists()
    manifest=json.loads(Path(args.manifest).read_text());assert manifest['dataset']==args.dataset
    for a,b in [('train','validation'),('train','test'),('validation','test')]:
        assert not set(manifest[a])&set(manifest[b])
        if args.dataset=='prostate':assert not {patient(x) for x in manifest[a]}&{patient(x) for x in manifest[b]}
    cache=Path(args.existing_cache);ids=manifest['train']+manifest['validation']+manifest['test']
    assert args.sigma_px==(32. if args.dataset=='kvasir' else 8.)
    meta=json.loads((cache/'cache_metadata.json').read_text())
    assert meta['dataset']==args.dataset and meta['resolution']==512 and meta['blocks']==[6,12,18,24]
    assert meta['ids_sha256']==hashlib.sha256('\n'.join(ids).encode()).hexdigest()
    assert meta['training_code_sha256']=='661fe6a153c3519229f6e8dcb3fb500dff5737a090faff86998f861a80df6b3d'
    atomic_json(root/'cache_metadata.json',meta)

    sources={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in Path(__file__).parent.glob('*.py') if p.name in ['train.py','model.py','data.py','image_data.py','heatmap.py','base_model.py','wavelet.py','reference_model.py','width_reference.py','flat_reference.py','pointwise_reference.py','pyramid_reference.py','conv3_reference.py','selected_reference.py','transpose_reference.py']}
    model=TransposeCNNGazePyramidDecoder();params=sum(p.numel() for p in model.parameters())
    initial_parameters={n:p.detach().cpu().clone() for n,p in model.named_parameters()}
    settings={**vars(args),'architecture':'gaze_pyramid_transpose_cnn_decoder_rgb','resolution':512,'counts':manifest['counts'],
              'initialization':'all learned decoder/projection/conditioning weights newly random; wavelet Haar-initialized; only frozen pretrained DINO features retained',
              'segmentation_checkpoint_loaded':False,'pretrained_encoder':'frozen DINOv3 ViT-L/16 cached features','dino_used':True,'wavelet_used':True,'dino_trainable_parameters':0,
              'wavelet_channels':[12,12,12,12],'dino_blocks':[6,12,18,24],'dino_projection_channels':[32,64,128,256],'gaze_conditioned_widths':[32,64,128,256],
              'decoder_widths':[32,32,32,32,32],'bottleneck_width':32,'decoder_input_channels':[268,172,108,76,35],'deep_fusion_input_channels':268,'full_resolution_skip':'original normalized RGB3 concatenated with upsampled decoder32','decoder_block':'one ConvTranspose2d3x3 per block:268/172/108/76/35->32; strides2/2/2/2/1,padding1,output_padding1/1/1/1/0,biasFalse,GN8,GELU; then Conv2d32->32 kernel3,padding1,biasFalse,GN8,GELU','skip_merge':'concatenate previous transposed-decoder32 at the current gaze/wavelet grid; fuse before next learned upsampling',
              'gaze_duration_weighted':True,'heatmap_from_original_fixations':True,'sigma_units':'input pixels',
              'background_subtraction':True,'prototype_features':'projected DINO32/64/128/256 ONLY, resized to each scale; never wavelet-fused',
              'conditioning':'concat projected DINO,H1,C1; Conv1x1 inputs34/66/130/258, outputs32/64/128/256, GN8/GELU',
              'encoder_recurrence':'DINO-only conditioning independently per depth; wavelet branch bypasses gaze and enters only deep decoder fusion and three decoder concatenations',
              'pooling':'one learned8-coefficient lifting wavelet then three fixed average pools','upsampling':'decoder learned ConvTranspose2d; only DINO projection resizing remains fixed bilinear',
              'knn':False,'prototype_iterations':0,'tta':False,'threshold':.5,
              'loss':'BCEWithLogits + soft Dice, final head only','optimizer':'AdamW, weight_decay1e-4',
              'schedule':'5 epoch linear warmup then cosine to 0.05x LR',
              'augmentation':'paired H/V image, cached feature, full-resolution heatmap and mask flips; frozen DINO not re-encoded',
              'data_rng_seed':args.seed+1000,'effective_batch_size':args.batch_size*args.accumulation,
              'checkpoint_selection':'highest native validation mean Dice; no test use','test_evaluated':False,
              'torch':torch.__version__,'gpu':torch.cuda.get_device_name(),
              'manifest_sha256':hashlib.sha256(Path(args.manifest).read_bytes()).hexdigest(),
              'sources_sha256':sources,'initial_model_fingerprint':fingerprint(model),'same_shaped_retained_modules_start_as_baseline':True,'trainable_parameters':params,
              'component_parameters':{n:sum(p.numel() for p in getattr(model,n).parameters()) for n in ['wavelet','dino_projection','gaze_conditioning','decoder','mask_head']}}
    atomic_json(root/'settings.json',settings);emit({'phase':'random_initialization',**settings})
    model=model.cuda()
    images={p.stem:p for p in (Path(manifest['root'])/'images').iterdir() if p.is_file()};assert set(ids)<=set(images)
    def loader(split,augment=False):
        dataset=FeatureImageDataset(cache,images,manifest[split],manifest,args.sigma_px,augment)
        atomic_json(root/f'{split}_gaze_audit.json',dataset.gaze_audit)
        return DataLoader(dataset,batch_size=args.batch_size,shuffle=augment,num_workers=0,pin_memory=True,collate_fn=collate)
    train_loader=loader('train',True);val_loader=loader('validation')
    optimizer=torch.optim.AdamW(model.parameters(),lr=args.lr,weight_decay=1e-4)
    def multiplier(epoch):
        if epoch<5:return (epoch+1)/5
        progress=min(1.,(epoch-5)/max(args.epochs-5,1))
        return .05+.95*.5*(1+math.cos(math.pi*progress))
    scheduler=torch.optim.lr_scheduler.LambdaLR(optimizer,multiplier)
    torch.manual_seed(args.seed+1000);torch.cuda.manual_seed_all(args.seed+1000)
    initial=summary(evaluate(model,val_loader));atomic_json(root/'initial_validation.json',initial);emit({'phase':'initial_validation','metrics':initial})
    best=initial['dice_mean'];best_epoch=0;stale=0;history=[];start=time.monotonic()
    torch.save({'model':model.state_dict(),'epoch':0,'validation':initial,'settings':settings},root/'best.pt')
    for epoch in range(1,args.epochs+1):
        model.train();total_loss=0.;seen=0;optimizer.zero_grad(set_to_none=True)
        for index,batch in enumerate(train_loader):
            logits=model(*inputs(batch));target=batch['mask'].cuda(non_blocking=True).float()
            loss=segmentation_loss(logits,target);assert torch.isfinite(loss)
            group_start=(index//args.accumulation)*args.accumulation
            examples=min(args.batch_size*args.accumulation,len(train_loader.dataset)-group_start*args.batch_size)
            (loss*(len(batch['id'])/examples)).backward()
            if (index+1)%args.accumulation==0 or index+1==len(train_loader):
                torch.nn.utils.clip_grad_norm_(model.parameters(),1.);optimizer.step();optimizer.zero_grad(set_to_none=True)
            total_loss+=loss.item()*len(batch['id']);seen+=len(batch['id'])
        val=summary(evaluate(model,val_loader));lr=optimizer.param_groups[0]['lr'];scheduler.step()
        record={'epoch':epoch,'train_loss':total_loss/seen,'lr':lr,'validation':val,'seconds':round(time.monotonic()-start,1)}
        history.append(record);atomic_json(root/'history.json',history);emit({'phase':'epoch',**record})
        if val['dice_mean']>best+1e-5:
            best=val['dice_mean'];best_epoch=epoch;stale=0
            torch.save({'model':model.state_dict(),'epoch':epoch,'validation':val,'settings':settings},root/'best.pt')
        else:stale+=1
        if stale>=args.patience:break
    selected=load_torch(root/'best.pt');model.load_state_dict(selected['model'])
    records=evaluate(model,val_loader);chosen=summary(records);assert abs(chosen['dice_mean']-best)<1e-7
    with (root/'validation_per_image.csv').open('w',newline='') as stream:
        writer=csv.DictWriter(stream,fieldnames=records[0]);writer.writeheader();writer.writerows(records)
    for name,digest in sources.items():assert hashlib.sha256((Path(__file__).parent/name).read_bytes()).hexdigest()==digest
    changes={n:float((p.detach().cpu()-initial_parameters[n]).norm()) for n,p in model.named_parameters()}
    assert all(v>1e-10 and np.isfinite(v) for v in changes.values())
    atomic_json(root/'parameter_updates.json',changes)
    result={'dataset':args.dataset,'seed':args.seed,'best_epoch':best_epoch,'epochs_completed':len(history),
            'validation':chosen,'training_seconds':time.monotonic()-start,'test_evaluated':False,
            'initial_model_fingerprint':settings['initial_model_fingerprint'],
            'checkpoint_sha256':hashlib.sha256((root/'best.pt').read_bytes()).hexdigest()}
    atomic_json(root/'train_summary.json',result);emit({'phase':'training_complete',**result})

if __name__=='__main__':main()