Spaces:
Sleeping
Sleeping
File size: 4,654 Bytes
434f330 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | from pathlib import Path
import csv,hashlib,json,shutil,tarfile
import numpy as np
import torch
from torch.utils.data import DataLoader
from PIL import Image
from model import TransposeCNNGazePyramidDecoder
from data import FeatureImageDataset,collate,load_torch
from train import evaluate,summary,atomic_json
root=Path(__file__).resolve().parent.parent;report=root/'report';report.mkdir()
torch.set_num_threads(4)
torch.backends.cudnn.benchmark=False;torch.backends.cudnn.deterministic=True;torch.backends.cuda.matmul.allow_tf32=True
for name,digest in json.loads((root/'source_hashes.json').read_text()).items():assert hashlib.sha256((root/'code'/name).read_bytes()).hexdigest()==digest
selected={};checkpoints={}
for ds in ['kvasir','prostate']:
folder=root/f'{ds}_seed42';settings=json.loads((folder/'settings.json').read_text())
trained=json.loads((folder/'train_summary.json').read_text());history=json.loads((folder/'history.json').read_text())
assert settings['dino_used'] and settings['dino_trainable_parameters']==0
assert trained['best_epoch']>0 and trained['best_epoch'] in [r['epoch'] for r in history]
assert abs(max(r['validation']['dice_mean'] for r in history)-trained['validation']['dice_mean'])<1e-7
digest=hashlib.sha256((folder/'best.pt').read_bytes()).hexdigest();assert digest==trained['checkpoint_sha256']
checkpoints[ds]={'path':str(folder/'best.pt'),'sha256':digest,'best_epoch':trained['best_epoch'],'validation':trained['validation']}
atomic_json(report/'final_selection.json',checkpoints)
comparison={};verified={}
for ds in ['kvasir','prostate']:
folder=root/f'{ds}_seed42';settings=json.loads((folder/'settings.json').read_text())
manifest=json.loads((root/'manifests'/f'{ds}_manifest.json').read_text())
assert settings['manifest_sha256']==hashlib.sha256((root/'manifests'/f'{ds}_manifest.json').read_bytes()).hexdigest()
model=TransposeCNNGazePyramidDecoder();state=load_torch(folder/'best.pt');model.load_state_dict(state['model']);model=model.cuda()
assert sum(p.numel() for p in model.parameters())==settings['trainable_parameters']
images={p.stem:p for p in (Path(manifest['root'])/'images').iterdir() if p.is_file()}
dataset=FeatureImageDataset(Path(settings['existing_cache']),images,manifest['test'],manifest,settings['sigma_px'])
loader=DataLoader(dataset,batch_size=4,num_workers=0,pin_memory=True,collate_fn=collate)
masks=folder/'test_masks';masks.mkdir();records=evaluate(model,loader,masks)
assert [r['id'] for r in records]==manifest['test']
for r in records:
p=np.asarray(Image.open(masks/(r['id']+'.png')))>0
target=load_torch(Path(settings['existing_cache'])/(r['id']+'.pt'))['native'].numpy().squeeze()>.5
intersection=(p&target).sum();dice=(2*intersection+1e-6)/(p.sum()+target.sum()+1e-6)
iou=(intersection+1e-6)/((p|target).sum()+1e-6)
np.testing.assert_allclose([r['dice'],r['iou']],[dice,iou],atol=1e-12,rtol=0)
metrics={**summary(records),'dataset':ds,'seed':42,'trainable_parameters':settings['trainable_parameters'],
'best_epoch':state['epoch'],'threshold':.5,'dino_used':True,'frozen_pretrained_dino':True,'segmentation_checkpoint_imported':False}
atomic_json(folder/'test_summary.json',metrics);atomic_json(folder/'test_gaze_audit.json',dataset.gaze_audit)
with (folder/'test_per_image.csv').open('w',newline='') as f:
w=csv.DictWriter(f,fieldnames=records[0]);w.writeheader();w.writerows(records)
dest=report/f'{ds}_seed42';dest.mkdir()
for p in folder.iterdir():
if p.suffix in ['.json','.csv']:shutil.copy2(p,dest/p.name)
shutil.copytree(masks,dest/'test_masks')
comparison[ds]=metrics;verified[ds]={'native_masks_verified':len(records),'checkpoint_sha256':checkpoints[ds]['sha256']}
print(json.dumps(metrics),flush=True);del model,dataset,loader;torch.cuda.empty_cache()
atomic_json(report/'comparison.json',comparison);atomic_json(report/'verification.json',verified)
shutil.copytree(root/'code',report/'code',ignore=shutil.ignore_patterns('__pycache__'))
shutil.copytree(root/'manifests',report/'manifests')
for name in ['data_preflight.json','source_hashes.json']:shutil.copy2(root/name,report/name)
archive=root/'gaze_pyramid_transpose_cnn_decoder_rgb_seed42_report.tar.gz'
with tarfile.open(archive,'w:gz') as stream:stream.add(report,arcname='report')
archive.with_suffix('.gz.sha256').write_text(hashlib.sha256(archive.read_bytes()).hexdigest()+' '+archive.name+'\n')
jobs=json.loads((root/'jobs.json').read_text());jobs.update(status='completed',test_metrics=comparison)
atomic_json(root/'jobs.json',jobs)
|