Ambabidi's picture
download
raw
11.2 kB
#!/usr/bin/env python
"""Ekspor encoder Phase 2 -> berkas yang bisa dibaca Phase 3.
Dijalankan di vast (/workspace). Tidak mengimpor notebook: kelas encoder dibangun
ulang di sini supaya skrip bisa jalan tanpa mengeksekusi sel mana pun.
Format keluaran: {'encoder_state_dict': ..., ...meta} dengan kunci TANPA prefiks
'body.' — Phase 3 memanggil `encoder_module.body.load_state_dict(state, strict=True)`,
jadi prefiks EncoderModule.body harus sudah dibuang. Nama berkas `<run_tag>_encoder.pth`
cocok dengan parser Phase 3 (nokd: p2_<student>, kd: p2_<student>+<teacher>-<grids>).
Pemakaian:
python export_encoder.py # semua run final di phase_2_out
python export_encoder.py p2_efficientnetb3 # satu run
python export_encoder.py --all --out-dir /workspace/enc
python export_encoder.py --list
"""
import argparse
import json
import os
import sys
from pathlib import Path
import torch
import torch.nn as nn
import torchvision.models as models
ENCODER_SUFFIX = '_encoder.pth'
CKPT_SUFFIXES = ('_best.pth', '_last.pth')
# --------------------------------------------------------------------------
# Encoder, disalin strukturnya dari Phase 2 (nama atribut harus identik supaya
# strict=True cocok). pretrained=False: bobot akan ditimpa sepenuhnya.
# --------------------------------------------------------------------------
class EfficientNetB3Encoder(nn.Module):
def __init__(self, in_channels=3):
super().__init__()
base = models.efficientnet_b3(weights=None)
if in_channels != 3:
base.features[0][0] = nn.Conv2d(
in_channels, base.features[0][0].out_channels,
base.features[0][0].kernel_size, base.features[0][0].stride,
base.features[0][0].padding, bias=False)
self.features = base.features
class ResNet50Encoder(nn.Module):
def __init__(self, in_channels=3):
super().__init__()
trunk = models.resnet50(weights=None)
trunk.fc = nn.Identity()
if in_channels != 3:
trunk.conv1 = nn.Conv2d(in_channels, trunk.conv1.out_channels,
trunk.conv1.kernel_size, trunk.conv1.stride,
trunk.conv1.padding, bias=False)
self.conv1, self.bn1, self.relu, self.maxpool = (
trunk.conv1, trunk.bn1, trunk.relu, trunk.maxpool)
self.layer1, self.layer2 = trunk.layer1, trunk.layer2
self.layer3, self.layer4 = trunk.layer3, trunk.layer4
class SwinV2TinyEncoder(nn.Module):
def __init__(self, in_channels=3):
super().__init__()
base = models.swin_v2_t(weights=None)
if in_channels != 3:
base.features[0][0] = nn.Conv2d(
in_channels, base.features[0][0].out_channels,
base.features[0][0].kernel_size, base.features[0][0].stride,
base.features[0][0].padding, bias=False)
self.features, self.norm = base.features, base.norm
ENCODERS = {'efficientnetb3': EfficientNetB3Encoder,
'resnet50': ResNet50Encoder,
'swinv2tiny': SwinV2TinyEncoder}
# --------------------------------------------------------------------------
def load_ckpt(path):
ck = torch.load(path, map_location='cpu', weights_only=False)
if not isinstance(ck, dict):
raise SystemExit('%s: bukan checkpoint dict' % path)
return ck
def encoder_state(ck, path):
"""state_dict encoder tanpa prefiks. Menerima dua bentuk checkpoint."""
if isinstance(ck.get('encoder_state_dict'), dict):
state = ck['encoder_state_dict']
source = 'encoder_state_dict'
elif isinstance(ck.get('model_state_dict'), dict):
state = ck['model_state_dict']
source = 'model_state_dict'
else:
raise SystemExit('%s: tidak ada encoder_state_dict / model_state_dict' % path)
prefixed = [k for k in state if k.startswith('encoder.body.')]
if prefixed:
state = {k[len('encoder.body.'):]: v for k, v in state.items()
if k.startswith('encoder.body.')}
source += ' -> buang prefiks encoder.body.'
else:
dropped = [k for k in state if not k.startswith('body.')]
if dropped and any(k.startswith('body.') for k in state):
state = {k[len('body.'):]: v for k, v in state.items()
if k.startswith('body.')}
source += ' -> buang prefiks body.'
elif dropped:
# kunci campuran (encoder + head): buang yang bukan encoder
keep = {k: v for k, v in state.items()
if not k.startswith(('head.', 'kd.', 'teacher.'))}
if len(keep) != len(state):
source += ' -> buang %d kunci non-encoder' % (len(state) - len(keep))
state = keep
return {k: v for k, v in state.items() if torch.is_tensor(v)}, source
def build_and_verify(student, in_channels, state, ck_name):
if student not in ENCODERS:
raise SystemExit('%s: student %r tidak didukung (%s)'
% (ck_name, student, tuple(ENCODERS)))
enc = ENCODERS[student](in_channels=in_channels)
target = enc.state_dict()
missing = sorted(set(target) - set(state))
unexpected = sorted(set(state) - set(target))
if missing or unexpected:
raise SystemExit(
'%s: tidak cocok dengan %s(in_channels=%d)\n missing %d: %s\n unexpected %d: %s'
% (ck_name, student, in_channels, len(missing), missing[:4],
len(unexpected), unexpected[:4]))
enc.load_state_dict(state, strict=True)
shapes_bad = [(k, tuple(state[k].shape), tuple(target[k].shape))
for k in target if tuple(state[k].shape) != tuple(target[k].shape)]
if shapes_bad:
raise SystemExit('%s: bentuk tensor beda: %s' % (ck_name, shapes_bad[:3]))
n = sum(v.numel() for v in state.values())
return enc, n
def find_runs(root):
"""Direktori run yang punya checkpoint _best.pth atau _last.pth."""
out = []
for d in sorted(Path(root).iterdir()):
if not d.is_dir():
continue
for suf in ('_best.pth', '_last.pth'):
p = d / ('%s%s' % (d.name, suf))
if p.exists():
out.append(p)
break
return out
def resolve(target, root):
"""target boleh tag, direktori run, atau path .pth."""
p = Path(target)
if p.suffix == '.pth' and p.exists():
return [p]
d = p if p.is_dir() else Path(root) / target
for suf in ('_best.pth', '_last.pth'):
c = d / ('%s%s' % (d.name, suf))
if c.exists():
return [c]
raise SystemExit('tidak ada checkpoint untuk %r (dicari di %s)' % (target, d))
def export_one(ckpt_path, out_dir, name, force):
ck = load_ckpt(ckpt_path)
cfg = ck.get('config') or {}
student = cfg.get('STUDENT')
in_ch = int(cfg.get('SLICE_CONTEXT', 3))
run_tag = ck.get('run_tag') or ckpt_path.name.rsplit('_', 1)[0]
if not student:
raise SystemExit('%s: checkpoint tanpa config[STUDENT]' % ckpt_path)
state, source = encoder_state(ck, ckpt_path)
enc, n_param = build_and_verify(student, in_ch, state, ckpt_path.name)
fname = name or ('%s%s' % (run_tag, ENCODER_SUFFIX))
if not fname.endswith(ENCODER_SUFFIX):
fname += ENCODER_SUFFIX
out = Path(out_dir) / fname
out.parent.mkdir(parents=True, exist_ok=True)
if out.exists() and not force:
print('[skip] %s sudah ada (pakai --force)' % out)
return out, False
payload = {
'encoder_state_dict': state,
'student': student,
'in_channels': in_ch,
'source': str(ckpt_path),
'run_tag': run_tag,
'epoch': ck.get('epoch'),
'weights': ck.get('weights'),
'best_epoch': ck.get('best_epoch'),
'val_f1_patient': ck.get('val_f1_patient'),
'val_auc': ck.get('val_auc'),
'slice_context': in_ch,
'cls_grid': cfg.get('CLS_GRID'),
'cls_hidden': cfg.get('CLS_HIDDEN'),
'use_distill': cfg.get('USE_DISTILL'),
'use_ema': cfg.get('USE_EMA'),
}
torch.save(payload, out)
# Provenans terpisah: mudah dibaca tanpa torch.load.
meta = {k: v for k, v in payload.items() if k != 'encoder_state_dict'}
meta['n_tensor'] = len(state)
meta['n_param'] = n_param
meta['state_source'] = source
meta['file'] = out.name
(out.parent / (out.stem + '.json')).write_text(
json.dumps(meta, indent=1, ensure_ascii=False, default=str), encoding='utf-8')
print('[ok] %-58s %s(%d ch) %d tensor / %s param | epoch %s bobot %s | %s'
% (out.name, student, in_ch, len(state), '{:,}'.format(n_param),
ck.get('epoch'), ck.get('weights'), source))
return out, True
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument('targets', nargs='*', help='tag / dir run / path .pth')
ap.add_argument('--all', action='store_true', help='semua run di --root')
ap.add_argument('--root', default='/workspace/phase_2_out',
help='direktori phase_2_out (default /workspace/phase_2_out)')
ap.add_argument('--out-dir', default=None,
help='tujuan (default: <root>/enc)')
ap.add_argument('--name', default=None,
help='nama berkas untuk satu target (default <run_tag>_encoder.pth)')
ap.add_argument('--force', action='store_true', help='timpa berkas yang ada')
ap.add_argument('--list', action='store_true', help='daftar checkpoint lalu keluar')
a = ap.parse_args()
root = Path(a.root)
if not root.exists():
alt = Path.cwd() / 'phase_2_out'
if alt.exists():
root = alt
else:
raise SystemExit('--root tidak ada: %s' % a.root)
out_dir = Path(a.out_dir) if a.out_dir else root / 'enc'
if a.list:
for p in find_runs(root):
ck = load_ckpt(p)
cfg = ck.get('config') or {}
print('%-62s %-14s ep %-4s bobot %-4s val F1(pas) %s'
% (p.parent.name, cfg.get('STUDENT'), ck.get('epoch'),
ck.get('weights'), ck.get('val_f1_patient')))
return 0
if a.name and (a.all or len(a.targets) != 1):
raise SystemExit('--name hanya untuk satu target')
if a.all:
paths = find_runs(root)
elif a.targets:
paths = [p for t in a.targets for p in resolve(t, root)]
else:
raise SystemExit('sebutkan target, atau pakai --all / --list')
if not paths:
raise SystemExit('tidak ada checkpoint di %s' % root)
print('[enc] root %s | out %s | %d kandidat' % (root, out_dir, len(paths)))
written = 0
for p in paths:
try:
_, did = export_one(p, out_dir, a.name, a.force)
written += int(did)
except SystemExit as e:
print('[gagal] %s: %s' % (p.parent.name, e))
print('[enc] %d berkas ditulis ke %s' % (written, out_dir))
return 0 if written else 1
if __name__ == '__main__':
sys.exit(main())

Xet Storage Details

Size:
11.2 kB
·
Xet hash:
3df72a1131f54db23b1145b1a95258ad4169b5fb1dd79719e6b4b173fc164669

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.