Buckets:
| #!/usr/bin/env python | |
| """Ekspor encoder Phase 2 -> berkas yang bisa dibaca Phase 3. | |
| Dijalankan di vast (/workspace). Tidak mengimpor notebook: kelas encoder dibangun | |
| ulang di sini supaya skrip bisa jalan tanpa mengeksekusi sel mana pun. | |
| Format keluaran: {'encoder_state_dict': ..., ...meta} dengan kunci TANPA prefiks | |
| 'body.' — Phase 3 memanggil `encoder_module.body.load_state_dict(state, strict=True)`, | |
| jadi prefiks EncoderModule.body harus sudah dibuang. Nama berkas `<run_tag>_encoder.pth` | |
| cocok dengan parser Phase 3 (nokd: p2_<student>, kd: p2_<student>+<teacher>-<grids>). | |
| Pemakaian: | |
| python export_encoder.py # semua run final di phase_2_out | |
| python export_encoder.py p2_efficientnetb3 # satu run | |
| python export_encoder.py --all --out-dir /workspace/enc | |
| python export_encoder.py --list | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import sys | |
| from pathlib import Path | |
| import torch | |
| import torch.nn as nn | |
| import torchvision.models as models | |
| ENCODER_SUFFIX = '_encoder.pth' | |
| CKPT_SUFFIXES = ('_best.pth', '_last.pth') | |
| # -------------------------------------------------------------------------- | |
| # Encoder, disalin strukturnya dari Phase 2 (nama atribut harus identik supaya | |
| # strict=True cocok). pretrained=False: bobot akan ditimpa sepenuhnya. | |
| # -------------------------------------------------------------------------- | |
| class EfficientNetB3Encoder(nn.Module): | |
| def __init__(self, in_channels=3): | |
| super().__init__() | |
| base = models.efficientnet_b3(weights=None) | |
| if in_channels != 3: | |
| base.features[0][0] = nn.Conv2d( | |
| in_channels, base.features[0][0].out_channels, | |
| base.features[0][0].kernel_size, base.features[0][0].stride, | |
| base.features[0][0].padding, bias=False) | |
| self.features = base.features | |
| class ResNet50Encoder(nn.Module): | |
| def __init__(self, in_channels=3): | |
| super().__init__() | |
| trunk = models.resnet50(weights=None) | |
| trunk.fc = nn.Identity() | |
| if in_channels != 3: | |
| trunk.conv1 = nn.Conv2d(in_channels, trunk.conv1.out_channels, | |
| trunk.conv1.kernel_size, trunk.conv1.stride, | |
| trunk.conv1.padding, bias=False) | |
| self.conv1, self.bn1, self.relu, self.maxpool = ( | |
| trunk.conv1, trunk.bn1, trunk.relu, trunk.maxpool) | |
| self.layer1, self.layer2 = trunk.layer1, trunk.layer2 | |
| self.layer3, self.layer4 = trunk.layer3, trunk.layer4 | |
| class SwinV2TinyEncoder(nn.Module): | |
| def __init__(self, in_channels=3): | |
| super().__init__() | |
| base = models.swin_v2_t(weights=None) | |
| if in_channels != 3: | |
| base.features[0][0] = nn.Conv2d( | |
| in_channels, base.features[0][0].out_channels, | |
| base.features[0][0].kernel_size, base.features[0][0].stride, | |
| base.features[0][0].padding, bias=False) | |
| self.features, self.norm = base.features, base.norm | |
| ENCODERS = {'efficientnetb3': EfficientNetB3Encoder, | |
| 'resnet50': ResNet50Encoder, | |
| 'swinv2tiny': SwinV2TinyEncoder} | |
| # -------------------------------------------------------------------------- | |
| def load_ckpt(path): | |
| ck = torch.load(path, map_location='cpu', weights_only=False) | |
| if not isinstance(ck, dict): | |
| raise SystemExit('%s: bukan checkpoint dict' % path) | |
| return ck | |
| def encoder_state(ck, path): | |
| """state_dict encoder tanpa prefiks. Menerima dua bentuk checkpoint.""" | |
| if isinstance(ck.get('encoder_state_dict'), dict): | |
| state = ck['encoder_state_dict'] | |
| source = 'encoder_state_dict' | |
| elif isinstance(ck.get('model_state_dict'), dict): | |
| state = ck['model_state_dict'] | |
| source = 'model_state_dict' | |
| else: | |
| raise SystemExit('%s: tidak ada encoder_state_dict / model_state_dict' % path) | |
| prefixed = [k for k in state if k.startswith('encoder.body.')] | |
| if prefixed: | |
| state = {k[len('encoder.body.'):]: v for k, v in state.items() | |
| if k.startswith('encoder.body.')} | |
| source += ' -> buang prefiks encoder.body.' | |
| else: | |
| dropped = [k for k in state if not k.startswith('body.')] | |
| if dropped and any(k.startswith('body.') for k in state): | |
| state = {k[len('body.'):]: v for k, v in state.items() | |
| if k.startswith('body.')} | |
| source += ' -> buang prefiks body.' | |
| elif dropped: | |
| # kunci campuran (encoder + head): buang yang bukan encoder | |
| keep = {k: v for k, v in state.items() | |
| if not k.startswith(('head.', 'kd.', 'teacher.'))} | |
| if len(keep) != len(state): | |
| source += ' -> buang %d kunci non-encoder' % (len(state) - len(keep)) | |
| state = keep | |
| return {k: v for k, v in state.items() if torch.is_tensor(v)}, source | |
| def build_and_verify(student, in_channels, state, ck_name): | |
| if student not in ENCODERS: | |
| raise SystemExit('%s: student %r tidak didukung (%s)' | |
| % (ck_name, student, tuple(ENCODERS))) | |
| enc = ENCODERS[student](in_channels=in_channels) | |
| target = enc.state_dict() | |
| missing = sorted(set(target) - set(state)) | |
| unexpected = sorted(set(state) - set(target)) | |
| if missing or unexpected: | |
| raise SystemExit( | |
| '%s: tidak cocok dengan %s(in_channels=%d)\n missing %d: %s\n unexpected %d: %s' | |
| % (ck_name, student, in_channels, len(missing), missing[:4], | |
| len(unexpected), unexpected[:4])) | |
| enc.load_state_dict(state, strict=True) | |
| shapes_bad = [(k, tuple(state[k].shape), tuple(target[k].shape)) | |
| for k in target if tuple(state[k].shape) != tuple(target[k].shape)] | |
| if shapes_bad: | |
| raise SystemExit('%s: bentuk tensor beda: %s' % (ck_name, shapes_bad[:3])) | |
| n = sum(v.numel() for v in state.values()) | |
| return enc, n | |
| def find_runs(root): | |
| """Direktori run yang punya checkpoint _best.pth atau _last.pth.""" | |
| out = [] | |
| for d in sorted(Path(root).iterdir()): | |
| if not d.is_dir(): | |
| continue | |
| for suf in ('_best.pth', '_last.pth'): | |
| p = d / ('%s%s' % (d.name, suf)) | |
| if p.exists(): | |
| out.append(p) | |
| break | |
| return out | |
| def resolve(target, root): | |
| """target boleh tag, direktori run, atau path .pth.""" | |
| p = Path(target) | |
| if p.suffix == '.pth' and p.exists(): | |
| return [p] | |
| d = p if p.is_dir() else Path(root) / target | |
| for suf in ('_best.pth', '_last.pth'): | |
| c = d / ('%s%s' % (d.name, suf)) | |
| if c.exists(): | |
| return [c] | |
| raise SystemExit('tidak ada checkpoint untuk %r (dicari di %s)' % (target, d)) | |
| def export_one(ckpt_path, out_dir, name, force): | |
| ck = load_ckpt(ckpt_path) | |
| cfg = ck.get('config') or {} | |
| student = cfg.get('STUDENT') | |
| in_ch = int(cfg.get('SLICE_CONTEXT', 3)) | |
| run_tag = ck.get('run_tag') or ckpt_path.name.rsplit('_', 1)[0] | |
| if not student: | |
| raise SystemExit('%s: checkpoint tanpa config[STUDENT]' % ckpt_path) | |
| state, source = encoder_state(ck, ckpt_path) | |
| enc, n_param = build_and_verify(student, in_ch, state, ckpt_path.name) | |
| fname = name or ('%s%s' % (run_tag, ENCODER_SUFFIX)) | |
| if not fname.endswith(ENCODER_SUFFIX): | |
| fname += ENCODER_SUFFIX | |
| out = Path(out_dir) / fname | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| if out.exists() and not force: | |
| print('[skip] %s sudah ada (pakai --force)' % out) | |
| return out, False | |
| payload = { | |
| 'encoder_state_dict': state, | |
| 'student': student, | |
| 'in_channels': in_ch, | |
| 'source': str(ckpt_path), | |
| 'run_tag': run_tag, | |
| 'epoch': ck.get('epoch'), | |
| 'weights': ck.get('weights'), | |
| 'best_epoch': ck.get('best_epoch'), | |
| 'val_f1_patient': ck.get('val_f1_patient'), | |
| 'val_auc': ck.get('val_auc'), | |
| 'slice_context': in_ch, | |
| 'cls_grid': cfg.get('CLS_GRID'), | |
| 'cls_hidden': cfg.get('CLS_HIDDEN'), | |
| 'use_distill': cfg.get('USE_DISTILL'), | |
| 'use_ema': cfg.get('USE_EMA'), | |
| } | |
| torch.save(payload, out) | |
| # Provenans terpisah: mudah dibaca tanpa torch.load. | |
| meta = {k: v for k, v in payload.items() if k != 'encoder_state_dict'} | |
| meta['n_tensor'] = len(state) | |
| meta['n_param'] = n_param | |
| meta['state_source'] = source | |
| meta['file'] = out.name | |
| (out.parent / (out.stem + '.json')).write_text( | |
| json.dumps(meta, indent=1, ensure_ascii=False, default=str), encoding='utf-8') | |
| print('[ok] %-58s %s(%d ch) %d tensor / %s param | epoch %s bobot %s | %s' | |
| % (out.name, student, in_ch, len(state), '{:,}'.format(n_param), | |
| ck.get('epoch'), ck.get('weights'), source)) | |
| return out, True | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__, | |
| formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument('targets', nargs='*', help='tag / dir run / path .pth') | |
| ap.add_argument('--all', action='store_true', help='semua run di --root') | |
| ap.add_argument('--root', default='/workspace/phase_2_out', | |
| help='direktori phase_2_out (default /workspace/phase_2_out)') | |
| ap.add_argument('--out-dir', default=None, | |
| help='tujuan (default: <root>/enc)') | |
| ap.add_argument('--name', default=None, | |
| help='nama berkas untuk satu target (default <run_tag>_encoder.pth)') | |
| ap.add_argument('--force', action='store_true', help='timpa berkas yang ada') | |
| ap.add_argument('--list', action='store_true', help='daftar checkpoint lalu keluar') | |
| a = ap.parse_args() | |
| root = Path(a.root) | |
| if not root.exists(): | |
| alt = Path.cwd() / 'phase_2_out' | |
| if alt.exists(): | |
| root = alt | |
| else: | |
| raise SystemExit('--root tidak ada: %s' % a.root) | |
| out_dir = Path(a.out_dir) if a.out_dir else root / 'enc' | |
| if a.list: | |
| for p in find_runs(root): | |
| ck = load_ckpt(p) | |
| cfg = ck.get('config') or {} | |
| print('%-62s %-14s ep %-4s bobot %-4s val F1(pas) %s' | |
| % (p.parent.name, cfg.get('STUDENT'), ck.get('epoch'), | |
| ck.get('weights'), ck.get('val_f1_patient'))) | |
| return 0 | |
| if a.name and (a.all or len(a.targets) != 1): | |
| raise SystemExit('--name hanya untuk satu target') | |
| if a.all: | |
| paths = find_runs(root) | |
| elif a.targets: | |
| paths = [p for t in a.targets for p in resolve(t, root)] | |
| else: | |
| raise SystemExit('sebutkan target, atau pakai --all / --list') | |
| if not paths: | |
| raise SystemExit('tidak ada checkpoint di %s' % root) | |
| print('[enc] root %s | out %s | %d kandidat' % (root, out_dir, len(paths))) | |
| written = 0 | |
| for p in paths: | |
| try: | |
| _, did = export_one(p, out_dir, a.name, a.force) | |
| written += int(did) | |
| except SystemExit as e: | |
| print('[gagal] %s: %s' % (p.parent.name, e)) | |
| print('[enc] %d berkas ditulis ke %s' % (written, out_dir)) | |
| return 0 if written else 1 | |
| if __name__ == '__main__': | |
| sys.exit(main()) | |
Xet Storage Details
- Size:
- 11.2 kB
- Xet hash:
- 3df72a1131f54db23b1145b1a95258ad4169b5fb1dd79719e6b4b173fc164669
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.