diff --git "a/scripts/train_ink_denoise.py" "b/scripts/train_ink_denoise.py" deleted file mode 100644--- "a/scripts/train_ink_denoise.py" +++ /dev/null @@ -1,1118 +0,0 @@ -# -*- coding: utf-8 -*- -"""И100 (22.10, «гораздо лучше чистая генерация с самого начала»): NON-autoregressive line generator = a diffusion DENOISER -over the whole line, conditioned point-by-point on the letter plan (FastSpeech-style; docs/literature_cursive_generation_2026-10-22.md, -ставка 3; DiffInk / ChiroDiff / BoundInk). - plan per point : current char, PREVIOUS char (BoundInk window), next char, phase inside the char (0..1, + NPH bin), char duration - style : 256-d vector of the FROZEN style encoder of lstmal_v6isl3 (train_ink_al2.Model.style over K=3 lines of the same page) - diffused x0 : per point [ (x - R_ADV*i)/XS , y/YS , pen ±1 ] — ABSOLUTE coords in line heights (x relative to a straight ramp so - the range is ~unit), pen = «pen up after this point» as an analog bit - denoiser : ConvNeXt-1D (dilated depthwise k=7) -> BiGRU 2x512 -> ConvNeXt-1D, FiLM by (noise level, style) in every block, - self-conditioning on the previous x0 estimate; predicts x0 (+ a pen logit) - diffusion : cosine schedule, T=100 training steps, DDIM (eta 0) with S=20 steps at sampling; CFG over letters (C, Cp, Cn -> null - with p=0.15 per line; phase/durations and style kept) - loss : L1 on positions (z units) + L1 on point-to-point increments (x20, the local stroke shape) + BCE pen - + optional CTC of the frozen recogniser ctc3 on the predicted x0 (CTCW, only for t < CTC_TMAX*T) -Training data: train_ink_al2.Inks (v6, align_islands3c_clean.npz = 3c without 364 dirty lines) — the true per-point char index gives the plan; the same affine AUG. -At generation the plan comes from the duration head of lstmal_v6isl3 (DUR=head) or from the true alignment (DUR=oracle). -MODES - MODE=train : env PROJ, HW_RUN (denoise1), INK_DS (ink_lines_v6), META_DS (ink_dataset_v6_sorted), ALIGN (align_islands3c.npz), - BASE (lstmal_v6isl3 fp16 ckpt: style encoder + duration head), STEPS, BS (64), LR (2e-4), CTCW (0), CTC (ckpt dir of ctc.pt) - MODE=judge : writes SEQS_OUT json for scripts/judge_lines_qwen.py SEQS_IN (exactly the standard judge's 60 v2-valid texts for SEED, - refs 0,16,17 of v2 valid); env CKPT (denoise ckpt), DUR=head|oracle (oracle = 3c durations when the v2 line is found in - v6, else ctc3 align_all), CFG (1.0 = no guidance), SSTEPS (20), SEED, N (60) - MODE=acts : И143 (SAE): residual stream at 'mid'/'post' on real train lines + line attributes -> ACTS_OUT npz (see acts()) - MODE=steer : И143: steering by SAE latents while sampling -> attributes + CTC CER per alpha (see steer()) - MODE=svdirs : И143: attribute directions in the style-vector space (within writer) -> SVDIR_OUT npz; MODE=steer SVDIR=npz uses them - MODE=gradcos: cosines of the gradients of the denoising / CTC / style (lightly noised and DRaFT-1) losses -> OUT json - MODE=sheet : eyeball sheet of v6-valid lines: real | oracle durations | head durations (OUT png) - MODE=blind : blind sheet «real / denoiser» of 20 v2-valid lines (OUT png + OUT.key.json) -""" -import os, sys, json, math, time, glob, copy, threading, queue, numpy as np -MODE=os.environ.get('MODE','train') -if MODE in ('judge','blind','splits','steer'): # the standard judge's dataset (judge_std.sh): v2 valid, ctc3 alignment, v2 meta — set BEFORE the imports read env - os.environ['INK_DS']='ink_lines_v2'; os.environ['ALIGN']='align_all.npz'; os.environ['META_DS']='ink_dataset_v2_sorted' -if os.environ.get('PRESS')=='1' or MODE=='pres_eval': os.environ['PRES']='1' # denoise4: the npz pressure column is loaded as Y[...,4] by train_ink.Inks -os.environ.setdefault('HW_RUN','denoise1'); os.environ.setdefault('INK_DS','ink_lines_v6'); os.environ.setdefault('META_DS','ink_dataset_v6_sorted') -os.environ.setdefault('ALIGN','align_islands3c_clean.npz'); os.environ.setdefault('MAXPTS','1200'); os.environ.setdefault('MAXTOK','200') -os.environ.setdefault('HID','512'); os.environ.setdefault('AUG','1') # HID = the BASE (v6isl3) LSTM width, not the denoiser's -sys.path.insert(0,os.path.dirname(os.path.abspath(__file__))) -import torch, torch.nn as nn, torch.nn.functional as F -import train_ink_al2 as T2 -from train_ink_al2 import dev, stoi, itos, V, SCALE, PROJ, MAXPTS, NPH, DUR_SCALE, render -E=os.environ.get -RUN=E('HW_RUN'); CKPT=os.path.join(PROJ,'ckpt_'+RUN); SAMP=os.path.join(PROJ,'samples_'+RUN); os.makedirs(os.path.join(CKPT,'fp16'),exist_ok=True); os.makedirs(SAMP,exist_ok=True) -BASE=E('BASE',os.path.join(PROJ,'ckpt_dl','lstmal_v6isl3','step100000_fp16.pt')) -STEPS=int(E('STEPS','100000')); BS=int(E('BS','64')); LR=float(E('LR','2e-4')); WD=float(E('WD','0.01')); WARM=int(E('WARM','1000')) -TD=int(E('TD','100')); SSTEPS=int(E('SSTEPS','20')); PDROP=float(E('PDROP','0.15')); PSTYLE_DROP=float(E('PSTYLE_DROP','0.1')); SELFCOND=E('SELFCOND','1')=='1' -DN_D=int(E('DN_D','384')); DN_GRU=int(E('DN_GRU','512')); DN_PRE=int(E('DN_PRE','6')); DN_POST=int(E('DN_POST','4')) -R_ADV=float(E('R_ADV','0.017')); XS=float(E('XS','0.6')); YS=float(E('YS','0.25')) # measured 22.10 on 2 000 v6 train lines: x advance 0.017/pt, resid. x sd ~0.5, y sd ~0.2 -DELW=float(E('DELW','1.0')); PENW=float(E('PENW','3.0')); POSW=float(E('POSW','1.0')) -CTCW=float(E('CTCW','0')); CTC_TMAX=float(E('CTC_TMAX','0.5')); CTC_START=int(E('CTC_START','0')); CTC_DIR=E('CTC',os.path.join(PROJ,'ckpt_ctc3')); CTC_HID=int(E('CTC_HID','384')) -PRESS=int(E('PRESS','0')); PRESW_DN=float(E('PRESW_DN','1.0')); PRES_WRITER=E('PRES_WRITER','w01'); WMED={} # denoise4: pressure as a 4th analog channel, normalised per writer (median -> 0.5) -def zc(): return 4 if PRESS else 3 -STYLE_ENC=int(E('STYLE_ENC','0')); AUXW=float(E('AUXW','0.2')); AUX_P=int(E('AUX_P','8')); AUX_K=int(E('AUX_K','4')); CROPR=int(E('CROPR','600')) # denoise4: own style encoder (trained jointly) -REFATT=int(E('REFATT','0')); REFATT_TRAIN=int(E('REFATT_TRAIN','0')) # 25.10 (И125, w03: «можно привязываться к референсной строке»): cross-attention to EVERY point of the reference line(s) (+ a sink token); point features from a style encoder initialised from the writer classifier, frozen unless REFATT_TRAIN -STYLE_SEMB=E('STYLE_SEMB','') # 26.10 (И137, w03: «эмбеддинг из этого эмбеддера и использовать как вектор этого стиля»): style vector = Linear(frozen SupCon writer embedding of ANOTHER line of the writer); tag of data//semb__.npy (scripts/semb_precompute.py); generation embeds the reference line(s) with SEMB_CKPT -SEMB_CKPT=E('SEMB_CKPT','ckpt_semb_A/best.pt') -WTOK=int(E('WTOK','0')); WT_LIST=[]; FREEZE_BASE=int(E('FREEZE_BASE','0')) # 01.11 (И154 этап 2, Лев: «слой внимания будет для всех одинаковый, а вектор для каждого писца свой»): WTOK learned tokens [WTOK,256] per training writer (WT_LIST, last row = null/unknown), read by the two shared cross-attention layers; FREEZE_BASE=1 trains only those layers + the tokens -STYLE_TOK=int(E('STYLE_TOK','0')) # 24.10 (И124, w03: «нормально хранить файл с почерком»): N style tokens [N,256] from the reference line(s), the denoiser cross-attends to them -REFK=[(int(a),float(b)) for a,b in (x.split(':') for x in E('REFK','').split(',') if x)]; REFK=([k for k,_ in REFK],[w/sum(w_ for _,w_ in REFK) for _,w in REFK]) if REFK else None # e.g. '1:0.5,2:0.25,3:0.25' -FOCUS=E('FOCUS',''); FOCUS_P=float(E('FOCUS_P','0.4')); BALANCE=int(E('BALANCE','0')); BAL_MIN=int(E('BAL_MIN','200')); BAL_P=float(E('BAL_P','1.0')); REFS_PAGES=int(E('REFS_PAGES','0')); LR_MIN=float(E('LR_MIN','0')) # 23.10: per-writer fine-tune (FOCUS share of the batch), writer-balanced batches, references from OTHER pages of the writer -STYLE_CON_W=float(E('STYLE_CON_W','0')); STYLE_CON_TMAX=float(E('STYLE_CON_TMAX','0.5')); STYLE_CON_TAU=float(E('STYLE_CON_TAU','0.1')) # 25.10 (И128): contrastive style reward with a FROZEN pre-trained writer embedder (scripts/style_embed_train.py, ckpt STYLE_CON_CKPT) -WKEY={} # writer string -> int, for the contrastive reward's same-writer mask -STYLE_CLS_W=float(E('STYLE_CLS_W','0')); STYLE_CLS_TMAX=float(E('STYLE_CLS_TMAX','0.5')) # frozen writer classifier (ckpt_stylecls) as a style loss on the predicted x0 -SC_WRITERS=['w01','w03','w05','w06','w07','w08','w09','w10','w11','w12','w13','w14','w15','w16','w17'] # = scripts/style_classifier.py WRITERS -SRC_FLAG=int(E('SRC_FLAG','0')); SRC_DROP=float(E('SRC_DROP','0.2')) # 23.10 SRC_FLAG: learnt source embedding (0 tablet, 1 scan_traced, 2 unknown) added to the style vector -PEN_JOIN=float(E('PEN_JOIN','0'));DN_ETA=float(E('DN_ETA','0')); STYLE_IN=int(E('STYLE_IN','0')); NOSTYLE=int(E('NOSTYLE','0')); STYLE_NOISE=float(E('STYLE_NOISE','0')); DUR_JIT=float(E('DUR_JIT','0')); SV_NOISE=float(E('SV_NOISE','0')) # denoise3 (22.10 ночь): style also added to every point's input; style-vector noise (train); plan-duration / style jitter (sampling) for diversity -AMP=E('AMP','1')=='1' and dev=='cuda'; CK_EVERY=int(E('CK_EVERY','2000')); LOG_EVERY=int(E('LOG_EVERY','100')) -# 26.10 base (w03: «скачивал бы раз в 1000, а в самом начале (первые 5000) раз в 100, интересно динамику смотреть»): fp16 EMA weights on their own -# schedule, apart from last.pt + sample sheet (CK_EVERY): every FP16_EVERY steps, every FP16_FINE steps up to FP16_FINE_UNTIL (0 = off, old behaviour). -FP16_EVERY=int(E('FP16_EVERY','0')); FP16_FINE=int(E('FP16_FINE','0')); FP16_FINE_UNTIL=int(E('FP16_FINE_UNTIL','0')) -def fp16_due(st): return (FP16_EVERY>0 and st%FP16_EVERY==0) or (FP16_FINE>0 and st<=FP16_FINE_UNTIL and st%FP16_FINE==0) -NULL=V # extra embedding row = «letter dropped» (CFG) -torch.backends.cuda.matmul.allow_tf32=True; torch.backends.cudnn.allow_tf32=True - -# ---------------- diffusion schedule (cosine, discrete T=TD) -def alpha_bar(Tn=TD,s=0.008): - t=np.arange(1,Tn+1)/Tn; f=np.cos((t+s)/(1+s)*math.pi/2)**2/math.cos(s/(1+s)*math.pi/2)**2; return np.clip(f,1e-4,0.9999).astype(np.float32) -AB=torch.tensor(alpha_bar(),device=dev) # AB[t], t=0 (cleanest) .. TD-1 (pure noise) - -# ---------------- data: al2 lines + previous char, continuous phase, duration per point -class Inks(T2.Inks): - def __init__(s,split): - super().__init__(split) - s.semb=np.load(os.path.join(PROJ,'data',os.environ.get('INK_DS'),f'semb_{STYLE_SEMB}_{split}.npy')).astype(np.float32) if STYLE_SEMB else None - s.srct={} # 23.10: line id -> 0 tablet / 1 scan_traced (SRC_FLAG) - s.wid={} # line id -> writer_id (unified meta has it; v6 meta: src) - meta=os.path.join(PROJ,'data',os.environ.get('META_DS','ink_dataset_all_sorted'),'meta_'+split+'.jsonl'); k=0 - for l in open(meta): - r=json.loads(l) - if k1e-3: acc.setdefault(s.page[j][0],[]).append(float(np.median(p))) - s.wmed={k:float(np.median(v)) for k,v in acc.items()}; print(split,'writer pressure medians',{k:round(v,3) for k,v in s.wmed.items()},flush=True) - def refs(s,j,rng): - if not (STYLE_ENC or REFS_PAGES or getattr(s,'xpage',False)): return super().refs(j,rng) # xpage: the DRaFT scan pool (И144) - if not hasattr(s,'_bw'): - s._bw={} - for q in s.idx: s._bw.setdefault(s.wid.get(q,s.page[q][0]),[]).append(q) - c=s._bw[s.wid.get(j,s.page[j][0])]; out=[] - nk=T2.K if not REFK else int(rng.choice(REFK[0],p=REFK[1])) # 24.10 (w03: «база — под снятие почерка по одной строке»): 1..K references, mostly 1 - for _ in range(nk): - q=j - for _t in range(6): # another line of the same WRITER, preferably from another page (DiffusionPen/DiffInk-style references) - q=c[rng.integers(len(c))] - if q!=j and s.page[q]!=s.page[j]: break - d=s.pts[s.off[q]:s.off[q+1]][:MAXPTS].copy() - if len(d)>CROPR: a=rng.integers(0,len(d)-CROPR+1); d=d[a:a+CROPR] - d[:,:2]*=SCALE; out.append(d[:,:3]) - return out - def batch(s,ids,rng=None,aug=None): - X,Y,Mk,Tk,Tm,C,Ph,Cn,Du,Dm,R,RL=super().batch(ids,rng,aug); Bn,Tn=Mk.shape - Cp=np.zeros((Bn,Tn),np.int64); Pc=np.zeros((Bn,Tn),np.float32); Dp=np.zeros((Bn,Tn),np.float32) - for b,i in enumerate(ids): - j=s.idx[i]; ci=s.al[j].astype(np.int64); n=len(ci); toks=s.toks[j]; tid=np.array([stoi.get(t,0) for t in toks]+[0]) - cnt=np.bincount(ci,minlength=len(toks)); start=np.concatenate([[0],np.cumsum(cnt)[:-1]]); pos=np.arange(n)-start[ci] - Cp[b,:n]=np.where(ci>0,tid[np.maximum(ci-1,0)],0); Pc[b,:n]=pos/np.maximum(cnt[ci],1); Dp[b,:n]=cnt[ci] - f=lambda a: torch.from_numpy(a).to(dev) - Pw=np.array([getattr(s,'wmed',{}).get(s.page[s.idx[i]][0],0.5) for i in ids],np.float32) if PRESS else np.zeros(len(ids),np.float32) - Wid=np.array([SC_WRITERS.index(s.wid.get(s.idx[i])) if s.wid.get(s.idx[i]) in SC_WRITERS else -1 for i in ids],np.int64) - Wn=np.array([WT_LIST.index(s.wid.get(s.idx[i])) if s.wid.get(s.idx[i]) in WT_LIST else len(WT_LIST) for i in ids],np.int64) # WTOK row (last = unknown writer) - Src=np.array([s.srct.get(s.idx[i],0) for i in ids],np.int64) - Wk=np.array([WKEY.setdefault(str(s.wid.get(s.idx[i],s.page[s.idx[i]][0])),len(WKEY)) for i in ids],np.int64) - Se=np.zeros((len(ids),256),np.float32) - if STYLE_SEMB: # И137: the embedding of ANOTHER line of the same writer (another page when possible) — never the target line itself unless it is the writer's only one - if not hasattr(s,'_bw'): - s._bw={} - for q in s.idx: s._bw.setdefault(s.wid.get(q,s.page[q][0]),[]).append(q) - rg=rng if rng is not None else np.random.default_rng() - for b,i in enumerate(ids): - j=s.idx[i]; c=s._bw[s.wid.get(j,s.page[j][0])]; q=j - for _t in range(6): - q=c[rg.integers(len(c))] - if q!=j and s.page[q]!=s.page[j]: break - Se[b]=s.semb[q] - return dict(Se=f(Se),Y=Y,Mk=Mk,Tk=Tk,Tm=Tm,C=C,Ph=Ph,Cn=Cn,Cp=f(Cp),Pc=f(Pc),Dp=f(Dp),R=R,RL=RL,Pw=f(Pw),Wid=f(Wid),Wn=f(Wn),Src=f(Src),Wk=f(Wk)) -def to_z(Y,Mk,Pw=None): - """Y [B,T,>=3] (dx,dy scaled by SCALE, pen[, -, pressure]) -> z0 [B,T,3|4] and raw positions P [B,T,2] (line heights); with PRESS the - 4th channel = 2*clip(p/(2*writer median),0,1)-1 and z0 gets an attribute-free companion: the per-line pressure mask (returned as 3rd item).""" - d=Y[...,:2]/SCALE; P=torch.cumsum(d*Mk[...,None],1); i=torch.arange(Y.size(1),device=Y.device,dtype=torch.float32)[None] - ch=[(P[...,0]-R_ADV*i)/XS,P[...,1]/YS,Y[...,2]*2-1]; pm=None - if PRESS: - pv=Y[...,4] if Y.size(-1)>4 else torch.full_like(Y[...,0],-1.0); pm=(pv[:,0]>=0).float() # -1 = the line has no real pressure - pn=(pv/(2*Pw[:,None].clamp(min=1e-3))).clamp(0,1); ch.append((2*pn-1)*pm[:,None]) - z=torch.stack(ch,-1)*Mk[...,None]; return (z,P,pm) if PRESS else (z,P) -def x0cat(xy,pl): - """net outputs -> x0 in z layout [x, y, pen(tanh), (pressure)]""" - return torch.cat([xy[...,:2],torch.tanh(pl/2)[...,None],xy[...,2:]],-1) -def from_z(zxy,Tn=None): - """zxy [B,T,2] -> raw positions [B,T,2].""" - i=torch.arange(zxy.size(1),device=zxy.device,dtype=torch.float32)[None]; return torch.stack([zxy[...,0]*XS+R_ADV*i,zxy[...,1]*YS],-1) - -# ---------------- denoiser -def tembed(t,dim=128): - h=dim//2; fr=torch.exp(-math.log(10000)*torch.arange(h,device=t.device)/h); a=t.float()[:,None]*fr[None]*(1000.0/TD); return torch.cat([a.sin(),a.cos()],-1) -class FiLM(nn.Module): - def __init__(s,G,D): super().__init__(); s.l=nn.Linear(G,2*D); nn.init.zeros_(s.l.weight); nn.init.zeros_(s.l.bias) - def forward(s,x,g): a,b=s.l(g)[:,None].chunk(2,-1); return x*(1+a)+b -class CNX(nn.Module): - def __init__(s,D,G,dil): - super().__init__(); s.dw=nn.Conv1d(D,D,7,padding=3*dil,dilation=dil,groups=D); s.ln=nn.LayerNorm(D); s.film=FiLM(G,D); s.f1=nn.Linear(D,4*D); s.f2=nn.Linear(4*D,D); s.gam=nn.Parameter(torch.full((D,),0.1)) - def forward(s,x,m,g): - h=s.dw((x*m).transpose(1,2)).transpose(1,2); h=s.film(s.ln(h),g); h=s.f2(F.gelu(s.f1(h))); return (x+s.gam*h)*m -class StyleEnc(nn.Module): - """Trajectory style encoder (architecture of scripts/style_classifier.Net: Conv1d stem x4 -> BiGRU 2x192 -> mean+max), embedding 256, - writer head over SC_WRITERS. Input per point: [dx*SCALE, dy*SCALE, pen, y - median(y)] (the classifier's features).""" - def __init__(s,d=192): - super().__init__() - s.stem=nn.Sequential(nn.Conv1d(4,128,5,padding=2),nn.GELU(),nn.Conv1d(128,192,5,stride=2,padding=2),nn.GELU(),nn.Conv1d(192,256,5,stride=2,padding=2),nn.GELU()) - s.gru=nn.GRU(256,d,num_layers=2,batch_first=True,bidirectional=True,dropout=0.1) - if STYLE_TOK: s.tq=nn.Parameter(torch.randn(STYLE_TOK,256)*0.02); s.tkv=nn.Linear(2*d,256); s.tatt=nn.MultiheadAttention(256,4,batch_first=True); s.tln=nn.LayerNorm(256) - if REFATT: s.rkv=nn.Sequential(nn.Linear(2*d,256),nn.LayerNorm(256)); s.sink=nn.Parameter(torch.zeros(1,256)) - s.proj=nn.Linear(4*d,256); s.cls=nn.Sequential(nn.GELU(),nn.Linear(256,len(SC_WRITERS))) # proj+cls = the classifier's head (Linear-GELU-Linear) - def embed(s,X,L): - h=s.stem(X.transpose(1,2)).transpose(1,2); L2=torch.clamp((L+3)//4,min=1) - with torch.autocast('cuda',enabled=False): - p=nn.utils.rnn.pack_padded_sequence(h.float(),L2.cpu(),batch_first=True,enforce_sorted=False); o,_=s.gru(p); o,_=nn.utils.rnn.pad_packed_sequence(o,batch_first=True) - m=(torch.arange(o.size(1),device=o.device)[None] style tokens [B,STYLE_TOK,256]: learned queries attend over the points of ALL real references""" - Bn,Kn,Tn,_=R.shape; x=R.flatten(0,1).float(); L=RL.flatten().clamp(min=1); msk=(torch.arange(Tn,device=x.device)[None]0).to(pm.device)[:,None]; kv=s.tkv(o.to(s.tkv.weight.dtype)).view(Bn,-1,256); pm=pm.view(Bn,-1) - q=s.tq[None].expand(Bn,-1,-1); a,_=s.tatt(q,kv,kv,key_padding_mask=~pm); return s.tln(a+q) - def refpts(s,R,RL): - """И125: R [B,K,T,3], RL [B,K] -> (keys [B,1+K*T',256] = sink + every (4x-subsampled) point of every real reference, valid mask [B,1+K*T'])""" - Bn,Kn,Tn,_=R.shape; x=R.flatten(0,1).float(); L=RL.flatten().clamp(min=1); msk=(torch.arange(Tn,device=x.device)[None]0).to(pm.device)[:,None]; kv=s.rkv(o.to(s.rkv[0].weight.dtype)).view(Bn,-1,256); pm=pm.view(Bn,-1) - return torch.cat([s.sink[None].expand(Bn,-1,-1).to(kv.dtype),kv],1),torch.cat([torch.ones(Bn,1,dtype=torch.bool,device=pm.device),pm],1) - def style(s,R,RL): - """R [B,K,T,3] (dx,dy scaled by SCALE, pen), RL [B,K] -> mean embedding [B,256]""" - Bn,Kn,Tn,_=R.shape; x=R.flatten(0,1).float(); L=RL.flatten().clamp(min=1); msk=(torch.arange(Tn,device=x.device)[None]0).float().to(v.device)[...,None]; return (v*w).sum(1)/w.sum(1).clamp(min=1) # 24.10: mean over REAL references only (REFK: 1..K per sample, empty slots RL=0) -class Denoiser(nn.Module): - def __init__(s): - super().__init__(); D=DN_D; G=256 - s.ec=nn.Embedding(V+1,D); s.ep=nn.Embedding(V+1,D); s.en=nn.Embedding(V+1,D); s.eph=nn.Embedding(NPH,D) - s.inp=nn.Linear(zc()+zc()+3+2,D); s.tm=nn.Sequential(nn.Linear(128,G),nn.SiLU(),nn.Linear(G,G)); s.sp=nn.Linear(256,G); s.null_style=nn.Parameter(torch.zeros(256)) - if STYLE_ENC or REFATT: s.enc=StyleEnc() - if REFATT and not (STYLE_ENC or REFATT_TRAIN): # frozen point features (the classifier's space): only rkv/sink/xatt learn - for n_,p_ in s.enc.named_parameters(): - if n_.startswith(('stem.','gru.')): p_.requires_grad_(False) - if SRC_FLAG: s.src_emb=nn.Parameter(torch.zeros(3,256)) - if STYLE_SEMB: s.semb_proj=nn.Linear(256,256) # И137: writer embedding -> the style vector the denoiser takes (init: least squares to the base style, see train()) # zero-init: a warm start is the same function - if STYLE_IN: s.sin=nn.Linear(256,D); nn.init.zeros_(s.sin.weight); nn.init.zeros_(s.sin.bias) # zero-init: a warm start from denoise1/2 begins as the same function - dil=[1,2,4,8,16,1,2,4,8,16] - s.pre=nn.ModuleList([CNX(D,G,dil[k%len(dil)]) for k in range(DN_PRE)]); s.gln=nn.LayerNorm(D); s.gfilm=FiLM(G,D) - s.gru=nn.GRU(D,DN_GRU,num_layers=2,batch_first=True,bidirectional=True); s.gout=nn.Linear(2*DN_GRU,D) - if WTOK: s.wtok=nn.Parameter(torch.randn(len(WT_LIST)+1,WTOK,256)*0.02) - if STYLE_TOK or REFATT or WTOK: # two cross-attention layers on the style tokens / reference points (after the GRU, before the head); out_proj zero-init = a warm start is the same function - if STYLE_TOK: s.null_tok=nn.Parameter(torch.zeros(STYLE_TOK,256)) - s.xln=nn.ModuleList([nn.LayerNorm(D) for _ in range(2)]); s.xatt=nn.ModuleList([nn.MultiheadAttention(D,4,kdim=256,vdim=256,batch_first=True) for _ in range(2)]) - for a_ in s.xatt: nn.init.zeros_(a_.out_proj.weight); nn.init.zeros_(a_.out_proj.bias) - s.post=nn.ModuleList([CNX(D,G,dil[k%len(dil)]) for k in range(DN_POST)]); s.oln=nn.LayerNorm(D); s.head=nn.Linear(D,3+(1 if PRESS else 0)) - def _hook(s,name,x,m): - """И143 (SAE): s._cap = {} -> the residual stream at 'mid' (after the GRU) / 'post' (before the head) is stored there; - s._steer = {name: vector [D] or [B,T,D]} -> added to the residual stream at that point (masked), every call = every sampling step""" - if getattr(s,'_cap',None) is not None: s._cap[name]=x.detach() - st=getattr(s,'_steer',None) - if st and name in st: x=(x+st[name].to(x.dtype))*m - return x - def forward(s,zt,t,cond,sv,sc): - """zt [B,T,3] noisy z; t [B] long; cond dict(C,Cp,Cn,Ph,Pc,Dp,Mk); sv [B,256] style; sc [B,T,3] self-cond x0 estimate (zeros = none) - -> x0 estimate of (zx, zy) [B,T,2] and pen logit [B,T].""" - if NOSTYLE: sv=s.null_style[None].expand(zt.size(0),-1) # 31.10 (И154 stage 1, w03: «обучим модель вообще без привязки к стилю письма»): every call — training, held-out, sampling — sees the one learned null style - Mk=cond['Mk']; m=Mk[...,None]; Tn=zt.size(1); pos=torch.arange(Tn,device=zt.device,dtype=torch.float32)[None,:,None].expand(zt.size(0),-1,-1)/600.0 - pc=cond['Pc'][...,None]; feats=torch.cat([zt,sc,pc,torch.sin(2*math.pi*pc),torch.cos(2*math.pi*pc),torch.log1p(cond['Dp'])[...,None]/3.0,pos],-1) - x=(s.inp(feats)+s.ec(cond['C'])+s.ep(cond['Cp'])+s.en(cond['Cn'])+s.eph(cond['Ph'])+(s.sin(sv)[:,None] if STYLE_IN else 0))*m - g=F.silu(s.tm(tembed(t))+s.sp(sv)) - for b in s.pre: x=b(x,m,g) - h=s.gfilm(s.gln(x),g); L=Mk.sum(1).long().clamp(min=1) - with torch.autocast('cuda',enabled=False): - p=nn.utils.rnn.pack_padded_sequence(h.float(),L.cpu(),batch_first=True,enforce_sorted=False); o,_=s.gru(p); o,_=nn.utils.rnn.pad_packed_sequence(o,batch_first=True,total_length=Tn) - x=(x+s.gout(o.to(x.dtype)))*m - x=s._hook('mid',x,m) - tok=getattr(s,'_tok',None) if (STYLE_TOK or REFATT or WTOK) else None; kpm=None - if isinstance(tok,tuple): tok,kpm=tok[0],~tok[1] # REFATT: (keys, valid mask) - if tok is not None: a_,_=s.xatt[0](s.xln[0](x),tok.to(x.dtype),tok.to(x.dtype),key_padding_mask=kpm); x=(x+a_)*m - for b in s.post: x=b(x,m,g) - if tok is not None: a_,_=s.xatt[1](s.xln[1](x),tok.to(x.dtype),tok.to(x.dtype),key_padding_mask=kpm); x=(x+a_)*m - x=s._hook('post',x,m) - o=s.head(s.oln(x)).float(); xy=torch.cat([o[...,:2],o[...,3:]],-1) if PRESS else o[...,:2]; return xy*m,o[...,2] # PRESS: xy = (x, y, pressure) - -# ---------------- frozen base: style encoder + duration head (lstmal_v6isl3) -_SEMB=[None] -def semb_vec(seqs): - """И137: mean of the frozen SupCon writer embeddings of the reference lines (raw dx,dy,pen), embedded exactly as at precompute/scoring""" - import style_classifier as SC - if _SEMB[0] is None: - st_=torch.load(os.path.join(PROJ,SEMB_CKPT),map_location=dev); n_=SC.Net(nc=256,d=st_['D']).to(dev); n_.load_state_dict(st_['m']); n_.eval(); _SEMB[0]=n_ - with torch.no_grad(): - fs=[SC.feats(SC.resample(np.asarray(q,np.float32)[:,:3])) for q in seqs]; X,L=SC.make_batch(fs,False); e=F.normalize(_SEMB[0](X,L).float(),dim=-1).mean(0,keepdim=True) - return F.normalize(e,dim=-1).to(dev) -def style_of(net,base,seqs): - """style vector [1,256] of reference lines (raw dx,dy,pen) with the encoder the denoiser was trained with: its own (STYLE_ENC) or the base's""" - if STYLE_SEMB and hasattr(net,'semb_proj'): - with torch.no_grad(): return net.semb_proj(semb_vec(seqs)) - if hasattr(net,'enc') and STYLE_ENC: # REFATT alone has an encoder for the reference points, but its style vector stays the base's - L=max(len(q) for q in seqs); R=np.zeros((1,len(seqs),L,3),np.float32); RL=np.zeros((1,len(seqs)),np.int64) - for k,q in enumerate(seqs): d=np.asarray(q,np.float32)[:MAXPTS].copy(); d[:,:2]*=SCALE; R[0,k,:len(d)]=d[:,:3]; RL[0,k]=len(d) - with torch.no_grad(): return net.enc.style(torch.from_numpy(R).to(dev),torch.from_numpy(RL).to(dev)) - return T2.style_from_seqs(base,seqs) -def cat_tok(ts): - """batch a list of per-item tokens: tensors [1,N,256] -> cat; REFATT tuples (keys [1,Lk,256], mask [1,Lk]) -> padded to the longest""" - if not isinstance(ts[0],tuple): return torch.cat(ts,0) - Lm=max(t[0].shape[1] for t in ts); K_=torch.zeros(len(ts),Lm,256,device=ts[0][0].device,dtype=ts[0][0].dtype); M_=torch.zeros(len(ts),Lm,dtype=torch.bool,device=ts[0][0].device) - for i,(k,mk) in enumerate(ts): K_[i,:k.shape[1]]=k[0]; M_[i,:k.shape[1]]=mk[0] - return K_,M_ -def tokens_of(net,seqs): - """style tokens [1,N,256] of reference lines (raw dx,dy,pen); REFATT: (keys, mask); None for a net without either""" - if not ((STYLE_TOK or REFATT) and hasattr(net,'enc')): return None - L=max(len(q) for q in seqs); R=np.zeros((1,len(seqs),L,3),np.float32); RL=np.zeros((1,len(seqs)),np.int64) - for k,q in enumerate(seqs): d=np.asarray(q,np.float32)[:MAXPTS].copy(); d[:,:2]*=SCALE; R[0,k,:len(d)]=d[:,:3]; RL[0,k]=len(d) - with torch.no_grad(): return (net.enc.refpts if REFATT else net.enc.tokens)(torch.from_numpy(R).to(dev),torch.from_numpy(RL).to(dev)) -def load_base(path=BASE): - b=T2.Model().to(dev); st=torch.load(path,map_location=dev); sd=b.state_dict() - for k,v in st.get('e',st).items(): # PRES=1 env (page writer) widens the unused output layer: copy the common rows - v=v.float() - if k in sd and sd[k].shape!=v.shape: t=sd[k].clone(); t[:v.shape[0]]=v; v=t - sd[k]=v - b.load_state_dict(sd); b.eval(); b._dn_base=True - for p in b.parameters(): p.requires_grad_(False) - return b -@torch.no_grad() -def head_durs(base,toks_list): - Bn=len(toks_list); Un=max(len(t) for t in toks_list)+2; Tk=torch.zeros(Bn,Un,dtype=torch.long,device=dev); Tm=torch.zeros(Bn,Un,device=dev) - for b,t in enumerate(toks_list): ids=[1]+[stoi.get(x,0) for x in t]+[2]; Tk[b,:len(ids)]=torch.tensor(ids); Tm[b,:len(ids)]=1 - d=(torch.expm1(base.dur(base.encode(Tk,Tm))[...,0]).clamp(min=0)*DUR_SCALE).round().long().cpu().numpy() - return [list(d[b,1:1+len(t)]) for b,t in enumerate(toks_list)] -_WDS=None -def writer_durs(toks_list,durs,w): - """23.10: the writer's own per-letter rhythm — median points per letter of writer w (out/dur_stats_writer.json from scripts/dur_stats_writer.py, - unified train alignment); letters the writer has < 5 times keep the head's value. The head is text-only (w01's rhythm for everyone).""" - global _WDS - if _WDS is None: _WDS=json.load(open(E('DUR_STATS_W',os.path.join(PROJ,'out','dur_stats_writer.json')))) - st=_WDS[w]; a=float(E('WDUR_A','1')) # 23.10: WDUR_A<1 = soft rhythm, a*writer + (1-a)*head (full writer rhythm cramps letters, readability drops) - return [[int(round(a*st[c]+(1-a)*x)) if c in st else int(x) for c,x in zip(t,d)] for t,d in zip(toks_list,durs)] -def plan(tok_ids,durs): - """per-point plan exactly as in training: C, Cp, Cn, Ph (NPH bins), Pc (pos/cnt), Dp (cnt).""" - C=[];Cp=[];Cn=[];Ph=[];Pc=[];Dp=[] - for k,(t,d) in enumerate(zip(tok_ids,durs)): - d=max(1,int(d)); pv=tok_ids[k-1] if k>0 else 0; nx=tok_ids[k+1] if k+1 P[i+1] becomes a cubic Hermite bridge whose end - tangents follow the strokes (1.5 points of context each side), sampled at the line's median step; other channels are copied from P[i].""" - if len(ups)==0: return d - P=np.cumsum(d[:,:2].astype(np.float64),0); st=np.hypot(d[1:,0],d[1:,1]); step=float(np.median(st[st>0])) if (st>0).any() else 0.02 - rows=[]; last=0; ups=sorted(int(u) for u in ups) - for i in ups: - rows.append(np.c_[P[last:i+1],d[last:i+1,2:]]); p0,p1=P[i],P[i+1]; g=float(np.hypot(*(p1-p0))) - k=int(g/max(step,1e-6))-1 - if k>0 and i>=1 and i+20.5)[0]; jmp=np.hypot(d[up+1,0],d[up+1,1]); done=[] - for i,jm in zip(up,jmp): - if jm>=PEN_JOIN: continue - c1=itos[int(C[i])] if int(C[i])=1 and i+2= JOIN_MIN_N times - v=li.get(c1,[0,0]) # (26.10: capital Э had 4 samples -> smoothed to the global ~2 % lift rate -> joined 33 times, hooks) - if v[1] next letter: P(join pair), smoothed toward the letter's exit rate - e=ex.get(c1,[0,0]); pe=(e[0]+K*gj)/(e[1]+K); v=pr.get(c1+c2,[0,0]); p=(v[0]+K*pe)/(v[1]+K) - else: continue - if rg.random()next as ONE stroke, there is no lift to keep; - # at a letter boundary WITHOUT any lift (last third of c1 .. first third of c2 — the same window as scripts/join_stats.py) the pen is lifted - # at c1's last point with probability 1 - P(join pair) when that is below JOIN_SPLIT_P (0.5): only letters the writer usually leaves open - bnd=np.where(np.diff(inst)==1)[0] - for i in bnd: - if i+1>=n-1 or i<1: continue - c1=itos[int(C[i])] if int(C[i])=float(E('JOIN_SPLIT_P','0.5')): continue - i0=np.where(inst==inst[i])[0]; i1=np.where(inst==inst[i+1])[0]; a0=i0[int(len(i0)*0.66)] if len(i0) else i; b0=i1[min(len(i1)-1,int(len(i1)*0.34))] if len(i1) else i+1 - if (d[a0:b0,2]>0.5).any(): continue - if rg.random()<1-p: d[i,2]=1; _JLOG['split_'+c1]=_JLOG.get('split_'+c1,0)+1 - else: _JLOG['kept_'+c1]=_JLOG.get('kept_'+c1,0)+1 - return spline_joins(d,done) if E('JOIN_SPLINE','0')=='1' else d # default straight: the page pipeline needs len == plan (pressure model); bridges are curved later (write_page SPLINE_BRIDGES) - -# ---------------- sampling (DDIM, x0-prediction, CFG over letters) -_ISLET={i:(len(c)==1 and c.isalpha()) for c,i in stoi.items()} -@torch.no_grad() -def sample(net,toks_list,durs_list,sv,cfg=None,steps=None,pen_thr=0.0,maxlen=None,trace=None): - """toks_list: list of char lists; durs_list: per-char point counts; sv [1 or B,256] style. Returns list of [n,3] (dx,dy,pen) raw units.""" - cfg=float(E('CFG','1.0')) if cfg is None else cfg; steps=steps or SSTEPS; net.eval(); Bn=len(toks_list) - if DUR_JIT>0: durs_list=[[max(1,int(round(float(x)*math.exp(DUR_JIT*float(torch.randn(())))))) for x in d] for d in durs_list] # log-normal jitter per letter (plan diversity) - global LAST_DURS; LAST_DURS=durs_list # the durations actually used (denoise_page builds the returned plans from them) - pl=[plan([stoi.get(x,0) for x in t],d) for t,d in zip(toks_list,durs_list)]; Tn=min(maxlen or MAXPTS,max(len(p[0]) for p in pl)); L=[min(Tn,len(p[0])) for p in pl] - cond={k:torch.zeros(Bn,Tn,dtype=torch.long,device=dev) for k in ('C','Cp','Cn','Ph')}; cond.update(Pc=torch.zeros(Bn,Tn,device=dev),Dp=torch.zeros(Bn,Tn,device=dev),Mk=torch.zeros(Bn,Tn,device=dev)) - for b,p in enumerate(pl): - n=L[b] - for k,a in zip(('C','Cp','Cn','Ph','Pc','Dp'),p): cond[k][b,:n]=torch.tensor(a[:n],dtype=cond[k].dtype) - cond['Mk'][b,:n]=1 - sv=sv.expand(Bn,-1).clone() - if SRC_FLAG and hasattr(net,'src_emb'): sv=sv+net.src_emb[{'tablet':0,'scan':1,'none':2}[E('DN_SRC','tablet')]][None] # 23.10: generate as tablet ink by default - if SV_NOISE>0: n_=torch.randn_like(sv); sv=sv+n_/n_.norm(dim=-1,keepdim=True)*SV_NOISE*sv.norm(dim=-1,keepdim=True) - scfg=float(E('SCFG','1.0')) # SCFG>1 (22.10 evening): guidance by STYLE — x0 += (SCFG-1)*(x0(style) - x0(null style)), letters kept - if scfg!=1.0 and cfg==1.0: cfg=1.0+1e-6 # route through the batched branch - nb=1 - if cfg!=1.0: # cond and uncond in one batch - un={k:(torch.where(cond['Mk']>0,torch.full_like(v,NULL),v) if k in ('C','Cp','Cn') else v) for k,v in cond.items()} - nb=3 if scfg!=1.0 else 2; cond2={k:torch.cat([cond[k],un[k]]+([cond[k]] if nb==3 else []),0) for k in cond}; sv2=torch.cat([sv,sv]+([net.null_style[None].expand(Bn,-1)] if nb==3 else []),0) - if STYLE_TOK: # tokens set by the caller (tokens_of) or the null tokens (sheets); CFG rows: cond, uncond (same style), null style - tb=getattr(net,'_tok_base',None); tk=(tb if tb is not None else net.null_tok[None]).expand(Bn,-1,-1) - net._tok=torch.cat([tk,tk]+([net.null_tok[None].expand(Bn,-1,-1)] if nb==3 else []),0) if cfg!=1.0 else tk - if WTOK: # writer tokens set by the caller (net._tok_base = net.wtok[[row]] or inverted tokens) or the null row; CFG rows: cond, uncond (same writer), null - nt=net.wtok[-1][None]; tb=getattr(net,'_tok_base',None); tk=(tb if tb is not None else nt).expand(Bn,-1,-1) - net._tok=torch.cat([tk,tk]+([nt.expand(Bn,-1,-1)] if nb==3 else []),0) if cfg!=1.0 else tk - if REFATT: # reference points set by the caller (tokens_of/cat_tok) or the sink alone (= no reference); null-style CFG row: sink only - tb=getattr(net,'_tok_base',None) - if tb is None: tb=(net.enc.sink[None].expand(Bn,-1,-1),torch.ones(Bn,1,dtype=torch.bool,device=dev)) - kk,mm=tb; kk=kk.expand(Bn,-1,-1); mm=mm.expand(Bn,-1); mn=mm.clone(); mn[:,1:]=False - net._tok=(torch.cat([kk,kk]+([kk] if nb==3 else []),0),torch.cat([mm,mm]+([mn] if nb==3 else []),0)) if cfg!=1.0 else (kk,mm) - m=cond['Mk'][...,None]; z=torch.randn(Bn,Tn,zc(),device=dev)*m; sc=torch.zeros_like(z); ts=np.linspace(TD-1,0,steps).round().astype(int) - for k,t in enumerate(ts): - tt=torch.full((Bn,),int(t),device=dev,dtype=torch.long) - with torch.autocast('cuda',dtype=torch.bfloat16,enabled=AMP): - if cfg!=1.0: - xy,pl_=net(torch.cat([z]*nb,0),torch.cat([tt]*nb,0),cond2,sv2,torch.cat([sc]*nb,0) if SELFCOND else torch.zeros(nb*Bn,Tn,zc(),device=dev)) - xc,pc=xy[:Bn],pl_[:Bn]; xg=xy[Bn:2*Bn]+cfg*(xc-xy[Bn:2*Bn]); pg=pl_[Bn:2*Bn]+cfg*(pc-pl_[Bn:2*Bn]) - if nb==3: xg=xg+(scfg-1)*(xc-xy[2*Bn:]); pg=pg+(scfg-1)*(pc-pl_[2*Bn:]) - xy,pl_=xg,pg - else: xy,pl_=net(z,tt,cond,sv,sc if SELFCOND else torch.zeros_like(z)) - x0=x0cat(xy.float(),pl_.float())*m; sc=x0 - if trace is not None: trace.append(dict(k=k,t=int(t),ab=float(AB[t]),z=z.detach().float().cpu(),x0=x0.detach().cpu(),pl=pl_.detach().float().cpu(),L=list(L))) # denoising animation (scripts/denoise_anim.py) - if k==len(ts)-1: break - a=AB[t]; a2=AB[ts[k+1]]; eps=(z-a.sqrt()*x0)/(1-a).sqrt() - if DN_ETA>0: # 23.10 (DPO pairs need different candidates; DDIM eta=0 gives near-identical ones): stochastic DDIM step - sg=DN_ETA*((1-a2)/(1-a)).sqrt()*(1-a/a2).clamp(min=0).sqrt(); z=(a2.sqrt()*x0+(1-a2-sg**2).clamp(min=0).sqrt()*eps+sg*torch.randn_like(z))*m - else: z=(a2.sqrt()*x0+(1-a2).sqrt()*eps)*m - P=from_z(x0[...,:2]).cpu().numpy(); pen=(pl_.float()>pen_thr).float().cpu().numpy(); res=[] - for b in range(Bn): - n=L[b]; d=np.zeros((n,4 if PRESS else 3),np.float32); d[1:,:2]=np.diff(P[b,:n],axis=0); d[:,2]=pen[b,:n]; d[-1,2]=1 - if PEN_JOIN>0 and E('JOIN_STATS'): # 26.10 (w03: «э рисуется с соединением, где я редко его делаю, и б тоже, надо аккуратно подбирать»): join a short lift - d=join_lifts(d,pl[b],n) # with the WRITER'S OWN probability (scripts/join_stats.py): inside a letter 1 - P(lift inside it), between letters P(join pair) - elif PEN_JOIN>0: # 23.10 (w03: more breaks inside words than in his hand): drop a pen-up whose jump to the next point is < PEN_JOIN line heights - up=np.where(d[:-1,2]>0.5)[0]; j=np.hypot(d[up+1,0],d[up+1,1]); Cb=np.asarray(pl[b][0][:n]) # (gen median jump 0.20 vs real w03 0.51: short lift-and-resume) - lt=np.array([_ISLET.get(int(c),False) for c in Cb]); ok=(j 0.5) - res.append(d) - return res - -# ---------------- CTC regulariser / monitor (frozen ctc3) -def load_rec(): - from train_ink_grpo import Rec - r=Rec(CTC_HID,V).to(dev); r.load_state_dict(torch.load(os.path.join(CTC_DIR,'ctc.pt'),map_location=dev)['m']); r.train() # train mode: cuDNN RNN backward needs it; Rec has no dropout -> same outputs - for p in r.parameters(): p.requires_grad_(False) - return r -def rec_feats(P,pen,Mk): - """P raw positions [B,T,2], pen prob [B,T] -> ctc3 input [B,T,4] (dx*20, dy*20, pen, y).""" - d=torch.cat([torch.zeros_like(P[:,:1]),P[:,1:]-P[:,:-1]],1); return torch.stack([d[...,0]*20,d[...,1]*20,pen,P[...,1]],-1)*Mk[...,None] -def greedy(lp,L): - out=[] - for b in range(lp.size(0)): - a=lp[b,:L[b]].argmax(-1).tolist(); prev=0; s=[] - for c in a: - if c!=0 and c!=prev: s.append(c) - prev=c - out.append(s) - return out -def cer(a,b): - if not a and not b: return 0.0 - dp=list(range(len(b)+1)) - for i_,ca in enumerate(a,1): - prev,dp[0]=dp[0],i_ - for j_,cb in enumerate(b,1): prev,dp[j_]=dp[j_],min(dp[j_]+1,dp[j_-1]+1,prev+(ca!=cb)) - return dp[len(b)]/max(1,len(a)) - -# ---------------- eyeball sheet: real | oracle durations | head durations (v6 valid, style = other lines of the same page) -def sheet(net,base,va,path,n=8,seed=1): - from PIL import Image, ImageDraw, ImageFont - rng=np.random.default_rng(seed); pos=list(range(0,len(va),max(1,len(va)//n)))[:n]; f=ImageFont.truetype(os.path.join(PROJ,'DejaVuSans.ttf'),14); ims=[] - for i in pos: - j=va.idx[i]; d=va.pts[va.off[j]:va.off[j+1]]; toks=va.toks[j]; refs=[q.copy() for q in va.refs(j,rng)] - for q in refs: q[:,:2]/=SCALE - sv=enc_style(net,refs) if STYLE_ENC else T2.style_from_seqs(base,refs); od=list(np.bincount(va.al[j].astype(np.int64),minlength=len(toks))); hd=head_durs(base,[toks])[0] - torch.manual_seed(seed); g1,g2=sample(net,[toks,toks],[od,hd],sv) - rows=[render(d),render(g1),render(g2)]; Wd=max(r.width for r in rows) - c=Image.new('L',(max(Wd,400),sum(r.height for r in rows)+22),255); y=0 - for r in rows: c.paste(r,(0,y)); y+=r.height - ImageDraw.Draw(c).text((2,y+2),'real / oracle dur / head dur: '+''.join(toks)[:80],fill=0,font=f); ims.append(c) - W=max(c.width for c in ims); out=Image.new('L',(W,sum(c.height for c in ims)+10*len(ims)),255); y=0 - for c in ims: out.paste(c,(0,y)); y+=c.height+10 - out.save(path); return path - -@torch.no_grad() -def enc_style(net,seqs): - """reference lines (raw units, as style_from_seqs takes them) -> style vector [1,256] of the denoiser's own encoder""" - net.enc.eval(); L=max(len(q[:MAXPTS]) for q in seqs); R=np.zeros((1,len(seqs),L,3),np.float32); RL=np.zeros((1,len(seqs)),np.int64) - for k,q in enumerate(seqs): d=np.asarray(q,np.float32)[:MAXPTS,:3].copy(); d[:,:2]*=SCALE; R[0,k,:len(d)]=d; RL[0,k]=len(d) - return net.enc.style(torch.from_numpy(R).to(dev),torch.from_numpy(RL).to(dev)) -def install_enc(net): - """callers compute style as T2.style_from_seqs(base, refs) with the BASE model (load_base): route those calls to the denoiser's encoder; - any other model (e.g. the LSTM) keeps the original function (T2._orig_style_from_seqs).""" - if not hasattr(T2,'_orig_style_from_seqs'): T2._orig_style_from_seqs=T2.style_from_seqs - orig=T2._orig_style_from_seqs - def patched(model,seqs): return enc_style(net,seqs) if getattr(model,'_dn_base',False) else orig(model,seqs) - T2.style_from_seqs=patched -def load_net(path): - st=torch.load(path,map_location=dev) - for k_,v_ in (st.get('stats') or {}).items(): globals()[k_]=v_ # R_ADV/XS/YS (+ STYLE_IN) the net was trained with — before building it - net=Denoiser().to(dev); net.load_state_dict({k:v.float() for k,v in st.get('e',st).items()}); net.eval() - if STYLE_ENC: install_enc(net) - return net,st.get('step') - -# ---------------- training -# ---------------- И142/И143 (w03 29.10: «добавить в лосс кусок, который будет подтягивать узнаваемость ... скалярное произведение с читаемостью») -DRAFT_CON_W=float(E('DRAFT_CON_W','0')); DRAFT_CTC_W=float(E('DRAFT_CTC_W','0')); DRAFT_EVERY=int(E('DRAFT_EVERY','1')); DRAFT_BS=int(E('DRAFT_BS','16')); DRAFT_STEPS=int(E('DRAFT_STEPS','8')) -DRAFT_DS=E('DRAFT_DS',''); DRAFT_SCAN_N=int(E('DRAFT_SCAN_N','8')) # И144 (w03 29.10 «Давай первый»): scan writers ONLY as style references of the DRaFT branch -def draft_inks(): - """И144: the train lines of DRAFT_DS (data/ink_lines_u1scan = scan_traced lines of u1hwrG, best 20% per writer by ctc4 CER) as a second - Inks — the module-level dataset dir / alignment / meta of train_ink / train_ink_al / env are swapped only while it loads. References come - from OTHER pages of the same writer (xpage), as in the unseen one-line probe. Its lines never enter the denoising / CTC losses.""" - import train_ink as B_ - sv_=(B_.D,T2.A.ALIGN,os.environ.get('META_DS')); B_.D=os.path.join(PROJ,'data',DRAFT_DS); T2.A.ALIGN=os.path.join(B_.D,E('DRAFT_ALIGN',E('ALIGN'))); os.environ['META_DS']=DRAFT_DS - try: d=Inks('train') - finally: B_.D,T2.A.ALIGN=sv_[:2]; os.environ['META_DS']=sv_[2] - d.xpage=True; return d -def cat_pad(a,b): # concatenate along the batch, zero-padding every other dim to the larger size - sh=[max(x,y) for x,y in zip(a.shape[1:],b.shape[1:])] - pad=lambda t: F.pad(t,[p_ for d_ in range(t.dim()-1,0,-1) for p_ in (0,sh[d_-1]-t.shape[d_])]) - return torch.cat([pad(a),pad(b)],0) -def load_scon(): - import style_classifier as SC_ - st_=torch.load(E('STYLE_CON_CKPT',os.path.join(PROJ,'ckpt_semb_A','best.pt')),map_location=dev); n_=SC_.Net(nc=256,d=st_['D']).to(dev); n_.load_state_dict(st_['m']) - n_.gru.dropout=0.0; n_.head[2].p=0.0; n_.train() # train mode for the cuDNN RNN backward, dropout off - for p_ in n_.parameters(): p_.requires_grad_(False) - return n_ -def con_loss(SCON,Ph_,pl,Mk,R,RL,Wk): - """contrastive style loss of generated lines (raw positions Ph_, pen logits pl): the frozen embedder must place each line nearest to ITS - writer's reference lines among the batch's other writers (same writer elsewhere in the batch is not a negative). Same as the И128 code.""" - Lq=Mk.sum(1).long().clamp(min=1); dq=torch.cat([torch.zeros_like(Ph_[:,:1]),Ph_[:,1:]-Ph_[:,:-1]],1); yq=Ph_[...,1]; med=torch.nanmedian(yq.masked_fill(Mk<0.5,float('nan')),1).values - Xq=torch.stack([dq[...,0]*SCALE,dq[...,1]*SCALE,torch.sigmoid(pl),yq-med[:,None]],-1)*Mk[...,None] - with torch.no_grad(): - nI,Kn_,Tn_,_=R.shape; xr=R.flatten(0,1).float(); Lr=RL.flatten().clamp(min=1); mr=(torch.arange(Tn_,device=dev)[None]0).float().to(dev)[...,None]; rr=F.normalize((er*wr).sum(1)/wr.sum(1).clamp(min=1),dim=-1) - with torch.autocast('cuda',enabled=False): g_=F.normalize(SCON(Xq.float(),Lq).float(),dim=-1); lg=g_@rr.T/STYLE_CON_TAU - eye=torch.eye(len(Wk),device=dev,dtype=torch.bool); lg=lg.masked_fill((Wk[:,None]==Wk[None])&~eye,-1e4) - return F.cross_entropy(lg,torch.arange(len(Wk),device=dev)) -def ctc_loss_of(rec,Ph_,pl,Mk,Tk,Tm): - L=Mk.sum(1).long().clamp(min=1); lp=rec(rec_feats(Ph_,torch.sigmoid(pl),Mk),L).float(); tl=(Tm.sum(1)-2).long(); tg=torch.cat([Tk[b,1:1+tl[b]] for b in range(len(L))]) - return (F.ctc_loss(lp.transpose(0,1),tg,L,tl,blank=0,reduction='none',zero_infinity=True)/tl.clamp(min=1).float()).mean() -def draft_gen(net,bt,sv,n): - """DRaFT-1 (Clark et al. 2023): the first n lines of the batch generated FROM PURE NOISE with their real plan and their writer's references - (no CFG, DRAFT_STEPS DDIM steps, no gradient), the LAST step with gradient -> (raw positions, pen logits, mask) of the final x0. - Unlike the И128 loss on a lightly noised REAL line (where the style is already in the input), here the style can only come from the references.""" - Mk=bt['Mk'][:n]; m=Mk[...,None]; cond=dict(Mk=Mk,**{k:bt[k][:n] for k in ('Ph','Pc','Dp','C','Cp','Cn')}); sv=sv[:n] - if REFATT: kk,mm=net.enc.refpts(bt['R'][:n],bt['RL'][:n]); net._tok=(kk,mm) - if STYLE_TOK: net._tok=net.enc.tokens(bt['R'][:n],bt['RL'][:n]) - z=torch.randn(n,Mk.size(1),zc(),device=dev)*m; sc=torch.zeros_like(z); ts=np.linspace(TD-1,0,DRAFT_STEPS).round().astype(int) - with torch.no_grad(): - for k,t in enumerate(ts[:-1]): - tt=torch.full((n,),int(t),device=dev,dtype=torch.long) - with torch.autocast('cuda',dtype=torch.bfloat16,enabled=AMP): xy,pl_=net(z,tt,cond,sv,sc if SELFCOND else torch.zeros_like(z)) - x0=x0cat(xy.float(),pl_.float())*m; sc=x0; a=AB[t]; a2=AB[ts[k+1]]; eps=(z-a.sqrt()*x0)/(1-a).sqrt(); z=(a2.sqrt()*x0+(1-a2).sqrt()*eps)*m - tt=torch.full((n,),int(ts[-1]),device=dev,dtype=torch.long) - with torch.autocast('cuda',dtype=torch.bfloat16,enabled=AMP): xy,pl_=net(z,tt,cond,sv,sc if SELFCOND else torch.zeros_like(z)) - return from_z(xy.float()[...,:2]),pl_.float(),Mk - -def gradcos(): - """w03 29.10: «какое скалярное произведение этой части лосса и той, которая отвечает за читаемость». On NB train batches of INK_DS, the - gradients w.r.t. the trainable weights of: main (denoising: POSW*pos + DELW*del + pen, t < CTC_TMAX*TD so that the lightly-noised losses - apply to the same x0), ctc (CTC on that x0 — the clean1 legibility term), con (the И128 style loss on that x0), dcon / dctc (DRaFT-1: style / - CTC on a line generated from noise, draft_gen). Cosines of every pair per batch -> mean, sd, share < 0; and the norms. Out: OUT json.""" - net,step=load_net(E('CKPT')); base=load_base(); tr=Inks('train'); rng=np.random.default_rng(int(E('SEED','11'))); NB=int(E('NB','30')); BSg=int(E('BS','16')) - if REFATT and not (STYLE_ENC or REFATT_TRAIN): - for n_,p_ in net.enc.named_parameters(): - if n_.startswith(('stem.','gru.')): p_.requires_grad_(False) - net.train(); P_=[p_ for p_ in net.parameters() if p_.requires_grad]; SC=load_scon(); rec=load_rec(); names=['main','ctc','con','dcon','dctc']; C={}; N={k:[] for k in names}; S_={} - for b in range(NB): - bt=tr.batch(rng.choice(len(tr),BSg,replace=False),rng); Mk=bt['Mk']; z0,Pp,*_=to_z(bt['Y'],Mk,bt.get('Pw')) - sv=base.style(bt['R'],bt['RL']) if not STYLE_ENC else net.enc.style(bt['R'],bt['RL']) - if SRC_FLAG: sv=sv+net.src_emb[bt['Src']] - if REFATT: net._tok=net.enc.refpts(bt['R'],bt['RL']) - if STYLE_TOK: net._tok=net.enc.tokens(bt['R'],bt['RL']) - cond=dict(Mk=Mk,**{k:bt[k] for k in ('Ph','Pc','Dp','C','Cp','Cn')}); t=torch.randint(0,int(CTC_TMAX*TD),(BSg,),device=dev); a=AB[t][:,None,None] - zt=(a.sqrt()*z0+(1-a).sqrt()*torch.randn_like(z0))*Mk[...,None]; xy,pl=net(zt,t,cond,sv,torch.zeros_like(zt)); xy=xy.float(); pl=pl.float(); nm=Mk.sum() - Ph_=from_z(xy[...,:2]); dm=Mk[:,1:]*Mk[:,:-1] - L={'main':POSW*((xy[...,:2]-z0[...,:2]).abs().sum(-1)*Mk).sum()/nm+DELW*((((Ph_[:,1:]-Ph_[:,:-1])-(Pp[:,1:]-Pp[:,:-1]))*20).abs().sum(-1)*dm).sum()/dm.sum() - +(F.binary_cross_entropy_with_logits(pl,(z0[...,2]>0).float(),reduction='none',pos_weight=torch.tensor(PENW,device=dev))*Mk).sum()/nm, - 'ctc':ctc_loss_of(rec,Ph_,pl,Mk,bt['Tk'],bt['Tm']),'con':con_loss(SC,Ph_,pl,Mk,bt['R'],bt['RL'],bt['Wk'])} - G={k:torch.cat([(g if g is not None else torch.zeros_like(p_)).flatten() for g,p_ in zip(torch.autograd.grad(v,P_,retain_graph=True,allow_unused=True),P_)]) for k,v in L.items()} - Pd,pd,Md=draft_gen(net,bt,sv,BSg); Ld={'dcon':con_loss(SC,Pd,pd,Md,bt['R'],bt['RL'],bt['Wk']),'dctc':ctc_loss_of(rec,Pd,pd,Md,bt['Tk'],bt['Tm'])} - G.update({k:torch.cat([(g if g is not None else torch.zeros_like(p_)).flatten() for g,p_ in zip(torch.autograd.grad(v,P_,retain_graph=True,allow_unused=True),P_)]) for k,v in Ld.items()}) - for k in names: N[k].append(float(G[k].norm())) - for k in names: S_[k]=G[k].detach().clone() if k not in S_ else S_[k]+G[k].detach() # the summed gradient = the direction an optimiser actually follows - for i,x in enumerate(names): - for y in names[i+1:]: C.setdefault(f'{x}·{y}',[]).append(float(F.cosine_similarity(G[x],G[y],dim=0))) - print(f'batch {b+1}/{NB}: losses '+' '.join(f'{k} {float(v):.3f}' for k,v in {**L,**Ld}.items())+' | cos '+' '.join(f'{k} {v[-1]:+.3f}' for k,v in C.items()),flush=True) - R={k:dict(mean=round(float(np.mean(v)),4),sd=round(float(np.std(v)),4),neg=round(float(np.mean(np.array(v)<0)),3)) for k,v in C.items()} - for k,v in R.items(): print(f'COS {k:10s} mean {v["mean"]:+.4f} sd {v["sd"]:.4f} share<0 {v["neg"]:.2f}',flush=True) - print('NORM '+' '.join(f'{k} {np.mean(v):.3g}' for k,v in N.items()),flush=True) - CS={f'{x}·{y}':round(float(F.cosine_similarity(S_[x],S_[y],dim=0)),4) for i,x in enumerate(names) for y in names[i+1:]} - for k,v in CS.items(): print(f'COS_OF_SUMS {k:10s} {v:+.4f}',flush=True) - json.dump(dict(ckpt=E('CKPT'),step=step,nb=NB,bs=BSg,cos=R,cos_of_sums=CS,norm={k:float(np.mean(v)) for k,v in N.items()},per_batch=C),open(E('OUT'),'w'),indent=1) - print('GRADCOS DONE',E('OUT'),flush=True) - -def train(): - tr=Inks('train'); va=Inks('valid'); base=load_base() - if WTOK: WT_LIST[:]=sorted({str(w) for w in tr.wid.values()}); print('WTOK',WTOK,'tokens x',len(WT_LIST),'writers:',WT_LIST,flush=True) - net=Denoiser().to(dev); ema=copy.deepcopy(net).eval() - for p in ema.parameters(): p.requires_grad_(False) - opt=torch.optim.AdamW(net.parameters(),LR,weight_decay=WD,betas=(0.9,0.99)); step=0; last=os.path.join(CKPT,'last.pt'); resumed=os.path.exists(last); st={} - if resumed: - st=torch.load(last,map_location=dev); net.load_state_dict(st['m']); ema.load_state_dict(st['e']); (None if FREEZE_BASE else opt.load_state_dict(st['o'])); step=st['step']; print('resumed',step,flush=True) - elif E('INIT_FROM'): # warm start (e.g. a CTC-regularised fork from the main run's EMA): weights only, fresh optimiser and step 0 - st=torch.load(E('INIT_FROM'),map_location=dev); w={k:v.float() for k,v in st.get('e',st).items()} - if PRESS and w['inp.weight'].shape[1]==11: # 3-channel net -> 4 channels: new input columns (zt_p, sc_p) and the pressure head row start at zero - o_=w['inp.weight']; n_=torch.zeros(o_.shape[0],13); n_[:,0:3]=o_[:,0:3]; n_[:,4:7]=o_[:,3:6]; n_[:,8:13]=o_[:,6:11]; w['inp.weight']=n_ - for k_ in ('head.weight','head.bias'): o_=w[k_]; n_=torch.zeros((4,)+tuple(o_.shape[1:])); n_[:3]=o_; w[k_]=n_ - print('warm-start: 3 -> 4 channels (pressure), new weights zero',flush=True) - r_=net.load_state_dict(w,strict=False) - if (STYLE_ENC or REFATT) and any(k.startswith('enc.stem') for k in r_.missing_keys): # fresh encoder: stem+GRU from the writer classifier (ckpt_stylecls) - sc_=torch.load(E('STYLE_INIT',os.path.join(PROJ,'ckpt_stylecls','best.pt')),map_location=dev)['m']; e_={k:v for k,v in sc_.items() if k.startswith(('stem.','gru.'))}; e_.update({'proj.weight':sc_['head.0.weight'],'proj.bias':sc_['head.0.bias'],'cls.1.weight':sc_['head.3.weight'],'cls.1.bias':sc_['head.3.bias']}) - net.enc.load_state_dict(e_,strict=False); print('style encoder: stem+GRU from',E('STYLE_INIT',os.path.join(PROJ,'ckpt_stylecls','best.pt')),len(e_),'tensors',flush=True) - if STYLE_SEMB and 'semb_proj.weight' in r_.missing_keys: # (01.11: only when the init has none — a warm start from a STYLE_SEMB net keeps its trained projection) И137: start the projection at the least-squares map writer embedding -> base style vector (the warm-started denoiser then gets in-distribution style vectors at step 0) - rg_=np.random.default_rng(0); Xs=[]; Ys=[] - with torch.no_grad(): - for _ in range(48): - b_=tr.batch(rg_.choice(len(tr.idx),64,replace=False).tolist(),rg_); Xs.append(b_['Se']); Ys.append(base.style(b_['R'],b_['RL']).float()) - Xs=torch.cat(Xs); Ys=torch.cat(Ys); Xa=torch.cat([Xs,torch.ones_like(Xs[:,:1])],1); Wls=torch.linalg.lstsq(Xa.cpu(),Ys.cpu(),driver='gelsd').solution.to(dev) - net.semb_proj.weight.data.copy_(Wls[:-1].T); net.semb_proj.bias.data.copy_(Wls[-1]); r2=1-((Xa@Wls-Ys)**2).sum()/((Ys-Ys.mean(0))**2).sum() - print(f'semb_proj: least-squares init on {len(Xs)} lines, R^2 to the base style {float(r2):.3f}',flush=True) - ema.load_state_dict(net.state_dict()); print('warm-start from',E('INIT_FROM'),'step',st.get('step'),'new (zero-init):',r_.missing_keys,flush=True) - if FREEZE_BASE: # И154 этап 2: the stage-1 generator stays as it is; only the shared attention layers and the writer tokens learn - for n_,p_ in net.named_parameters(): p_.requires_grad_(n_.startswith(('xln.','xatt.','wtok'))) - opt=torch.optim.AdamW([p_ for p_ in net.parameters() if p_.requires_grad],LR,weight_decay=WD,betas=(0.9,0.99)) - if resumed: opt.load_state_dict(st['o']) - print('FREEZE_BASE: trainable',sum(p_.numel() for p_ in net.parameters() if p_.requires_grad),'of',sum(p_.numel() for p_ in net.parameters()),flush=True) - rec=load_rec() if (CTCW>0 or E('MON_CTC','1')=='1') and os.path.exists(os.path.join(CTC_DIR,'ctc.pt')) else None - print(f"denoise[{RUN}] dev={dev} V={V} D={DN_D} GRU={DN_GRU} pre/post={DN_PRE}/{DN_POST} BS={BS} LR={LR} T={TD} params {sum(p.numel() for p in net.parameters())/1e6:.1f}M " - f"target {STEPS} align {T2.A.ALIGN} CTCW={CTCW} (rec {'on' if rec is not None else 'off'}) selfcond={SELFCOND} amp={AMP}",flush=True) - stats=dict(R_ADV=R_ADV,XS=XS,YS=YS,STYLE_IN=STYLE_IN,NOSTYLE=NOSTYLE,PRESS=PRESS,WMED=getattr(tr,'wmed',{}),STYLE_ENC=STYLE_ENC,SRC_FLAG=SRC_FLAG,STYLE_TOK=STYLE_TOK,REFATT=REFATT,STYLE_SEMB=STYLE_SEMB,SEMB_CKPT=SEMB_CKPT,WTOK=WTOK,WT_LIST=WT_LIST) - U=None - if STYLE_ENC and AUXW>0: # writer classification + batch-hard triplet on unified train lines (all writers with data), every step - ad=E('AUX_DS','ink_lines_u1'); ad=ad if os.path.exists(os.path.join(PROJ,'data',ad,'train_raw.npz')) else E('INK_DS') # a merged INK_DS starts with u1's lines - uz=np.load(os.path.join(PROJ,'data',ad,'train_raw.npz'),allow_pickle=True); uo=uz['pt_off']; up=uz['pts']; uw={} - for i,l in enumerate(open(os.path.join(PROJ,'data',ad,'meta_train.jsonl'))): - w_=json.loads(l)['writer_id'] - if w_ in SC_WRITERS and uo[i+1]-uo[i]>=40: uw.setdefault(w_,[]).append(i) - U=(up,uo,{w_:v for w_,v in uw.items() if len(v)>=AUX_K}); print('aux style data: unified train',{w_:len(v) for w_,v in U[2].items()},flush=True) - def aux_batch(rng): - up,uo,uw=U; ws=list(uw); pick=rng.choice(len(ws),min(AUX_P,len(ws)),replace=False); xs=[];ys=[] - for k in pick: - for i in rng.choice(uw[ws[k]],AUX_K,replace=False): - d=up[uo[i]:uo[i+1]][:,:3].astype(np.float32) - if len(d)>512: a=rng.integers(0,len(d)-511); d=d[a:a+512] - y=np.cumsum(d[:,1]); xs.append(np.stack([d[:,0]*SCALE,d[:,1]*SCALE,d[:,2],y-np.median(y)],1)); ys.append(SC_WRITERS.index(ws[k])) - L=np.array([len(x) for x in xs]); X=np.zeros((len(xs),L.max(),4),np.float32) - for b,x in enumerate(xs): X[b,:len(x)]=x - return torch.from_numpy(X).to(dev),torch.from_numpy(L).to(dev),torch.tensor(ys,device=dev) - q=queue.Queue(maxsize=6) - bypos={} - for i_,j_ in enumerate(tr.idx): bypos.setdefault(tr.wid.get(j_,tr.page[j_][0]),[]).append(i_) - if FOCUS and E('FOCUS_KIND')=='prose': bypos[FOCUS]=[i_ for i_ in bypos.get(FOCUS,[]) if '$' not in str(tr.labels[tr.idx[i_]])] # 24.10 (w03: «дообучи на моей прозе») - if FOCUS: print('focus writer',FOCUS,len(bypos.get(FOCUS,[])),'lines, share',FOCUS_P,flush=True) - balw=[w_ for w_,v_ in bypos.items() if len(v_)>=BAL_MIN] - if BALANCE: print('balanced writers',{w_:len(bypos[w_]) for w_ in balw},flush=True) - def pick(rng): - if FOCUS: - nf=int(round(BS*FOCUS_P)); return np.concatenate([rng.choice(bypos[FOCUS],nf),rng.integers(0,len(tr),BS-nf)]) - if BALANCE: # BAL_P (29.10, base2): that share of the batch writer-balanced, the rest uniform over lines (1.0 = all balanced, as before) - nb=int(round(BS*BAL_P)); return np.concatenate([np.array([rng.choice(bypos[balw[k_]]) for k_ in rng.integers(0,len(balw),nb)],np.int64),rng.integers(0,len(tr),BS-nb)]) - return rng.integers(0,len(tr),BS) - def loader(seed): - rng=np.random.default_rng(seed) - while True: q.put(tr.batch(pick(rng),rng)) - for k in range(2): threading.Thread(target=loader,args=(1000+step+k,),daemon=True).start() - DR=None - if DRAFT_DS: - DR=draft_inks(); qd=queue.Queue(maxsize=6); print(f'DRaFT scan pool {DRAFT_DS}: {len(DR)} lines, {len(set(DR.wid.values()))} writers, {DRAFT_SCAN_N} per step',flush=True) - def dloader(seed): - rng=np.random.default_rng(seed) - while True: qd.put(DR.batch(rng.integers(0,len(DR),DRAFT_SCAN_N),rng)) - threading.Thread(target=dloader,args=(2000+step,),daemon=True).start() - t0=time.time(); s0=step; acc={}; nbad=0; arng=np.random.default_rng(7+step); SCL=None; l_scl=torch.zeros((),device=dev) - if STYLE_CLS_W>0: - import style_classifier as SC_ - SCL=SC_.Net().to(dev); SCL.load_state_dict(torch.load(E('STYLE_CLS_CKPT',os.path.join(PROJ,'ckpt_stylecls','best.pt')),map_location=dev)['m']) - SCL.gru.dropout=0.0; SCL.head[2].p=0.0; SCL.train() # train mode for the cuDNN RNN backward, dropout off = deterministic - for p_ in SCL.parameters(): p_.requires_grad_(False) - print('style loss: frozen writer classifier, weight',STYLE_CLS_W,flush=True) - SCON=None; l_con=torch.zeros((),device=dev) - DSCON=load_scon() if DRAFT_CON_W>0 else None; l_dcon=torch.zeros((),device=dev); l_dctc=torch.zeros((),device=dev) - if DSCON is not None or DRAFT_CTC_W>0: print(f'DRaFT-1: style {DRAFT_CON_W} (embedder',E('STYLE_CON_CKPT','ckpt_semb_A/best.pt'),f') ctc {DRAFT_CTC_W}, every {DRAFT_EVERY} steps, {DRAFT_BS} lines, {DRAFT_STEPS} sampling steps',flush=True) - if STYLE_CON_W>0: # И128: frozen embedder trained by SupCon on real lines of ~200 writers (never the unseen-test ones) - import style_classifier as SC_ - st_=torch.load(E('STYLE_CON_CKPT',os.path.join(PROJ,'ckpt_semb_A','best.pt')),map_location=dev); SCON=SC_.Net(nc=256,d=st_['D']).to(dev); SCON.load_state_dict(st_['m']) - SCON.gru.dropout=0.0; SCON.head[2].p=0.0; SCON.train() # train mode for the cuDNN RNN backward, dropout off - for p_ in SCON.parameters(): p_.requires_grad_(False) - print('style reward: frozen SupCon embedder',E('STYLE_CON_CKPT','ckpt_semb_A/best.pt'),'eval',st_.get('eval'),'weight',STYLE_CON_W,'tmax',STYLE_CON_TMAX,flush=True) - # 24.10: held-out loss of the EMA on fixed valid lines (fixed noise levels covering 0..TD-1, fixed noise, no self-cond, no drops) every - # VAL_EVERY steps -> val.csv (+ per source type). SCHED=plateau: LR = LR*scale, scale*=PLAT_F after PATIENCE checks without a VAL_TOL - # relative improvement, stop when scale < MIN_SCALE (STEPS is then only a cap); best.pt (fp16 EMA) = best held-out loss. VAL=1 = measure only. - SCHED=E('SCHED','cos'); VAL_EVERY=int(E('VAL_EVERY','2000')); VAL_TOL=float(E('VAL_TOL','0.003')); PATIENCE=int(E('PATIENCE','3')) - PLAT_F=float(E('PLAT_F','0.5')); MIN_SCALE=float(E('MIN_SCALE','0.03')); VB=[]; stop=False - SS=dict(scale=1.0,best=float('inf'),bad=0,cuts=[]) - if resumed and 'ss' in st: SS.update(st['ss']); print('plateau state',SS,flush=True) - if SCHED=='plateau' or E('VAL','0')=='1': - vr=np.random.default_rng(99); g_=torch.Generator(device=dev).manual_seed(99); nv=min(int(E('VAL_N','512')),len(va)) - for a_ in range(0,nv,BS): - bt_=va.batch(np.arange(a_,min(nv,a_+BS)),vr); z0_,*_=to_z(bt_['Y'],bt_['Mk'],bt_.get('Pw')) - VB.append((bt_,(torch.arange(a_,a_+len(z0_),device=dev)*37)%TD,torch.randn(z0_.shape,generator=g_,device=dev))) - print(f'held-out loss: {nv} valid lines every {VAL_EVERY} steps, sched {SCHED} (tol {VAL_TOL}, patience {PATIENCE}, x{PLAT_F}, stop < {MIN_SCALE})',flush=True) - def val_loss(model): - per={} - with torch.no_grad(), torch.autocast('cuda',dtype=torch.bfloat16,enabled=AMP): - for bt_,t_,eps_ in VB: - Mk_=bt_['Mk']; z0_,P_,*_=to_z(bt_['Y'],Mk_,bt_.get('Pw')); sv_=model.semb_proj(bt_['Se']) if STYLE_SEMB else (model.enc.style(bt_['R'],bt_['RL']) if STYLE_ENC else base.style(bt_['R'],bt_['RL'])) - if SRC_FLAG: sv_=sv_+model.src_emb[bt_['Src']] - if STYLE_TOK: model._tok=model.enc.tokens(bt_['R'],bt_['RL']) - if WTOK: model._tok=model.wtok[bt_['Wn']] - if REFATT: model._tok=model.enc.refpts(bt_['R'],bt_['RL']) - a_=AB[t_][:,None,None]; zt_=(a_.sqrt()*z0_+(1-a_).sqrt()*eps_)*Mk_[...,None] - cond_=dict(Mk=Mk_,Ph=bt_['Ph'],Pc=bt_['Pc'],Dp=bt_['Dp'],C=bt_['C'],Cp=bt_['Cp'],Cn=bt_['Cn']) - xy_,pl_=model(zt_,t_,cond_,sv_,torch.zeros_like(zt_)); xy_=xy_.float(); pl_=pl_.float(); n_=Mk_.sum(1).clamp(min=1) - lp=((xy_[...,:2]-z0_[...,:2]).abs().sum(-1)*Mk_).sum(1)/n_ - Ph2=from_z(xy_[...,:2]); dm=Mk_[:,1:]*Mk_[:,:-1]; ld=((((Ph2[:,1:]-Ph2[:,:-1])-(P_[:,1:]-P_[:,:-1]))*20).abs().sum(-1)*dm).sum(1)/dm.sum(1).clamp(min=1) - le=(F.binary_cross_entropy_with_logits(pl_,(z0_[...,2]>0).float(),reduction='none',pos_weight=torch.tensor(PENW,device=dev))*Mk_).sum(1)/n_ - for s_,v_ in zip(bt_['Src'].tolist(),(POSW*lp+DELW*ld+le).tolist()): per.setdefault(s_,[]).append(v_) - return float(np.mean([v for vs in per.values() for v in vs])),{k:round(float(np.mean(v)),4) for k,v in sorted(per.items())} - while step0: # «same writer, another day»: noise of norm STYLE_NOISE*|style| (the scale is not differentiated; the style still is) - with torch.no_grad(): n_=torch.randn_like(sv); n_=n_/n_.norm(dim=-1,keepdim=True)*STYLE_NOISE*sv.norm(dim=-1,keepdim=True) - sv=sv+n_ - sv=torch.where(drop,net.null_style[None].expand_as(sv),sv) # learnable null style (outside no_grad: it is trained) - if WTOK: net._tok=net.wtok[torch.where(drop[:,0],torch.full_like(bt['Wn'],len(WT_LIST)),bt['Wn'])] - if STYLE_TOK: tk=net.enc.tokens(bt['R'],bt['RL']); net._tok=torch.where(drop[...,None],net.null_tok[None].expand_as(tk),tk) - if REFATT: kk,mm=net.enc.refpts(bt['R'],bt['RL']); mm=mm.clone(); mm[:,1:]&=~drop; net._tok=(kk,mm) # a dropped style sees the sink only - if SRC_FLAG: # source embedding, dropped to 'unknown' (2) with prob SRC_DROP - si=torch.where(torch.rand(Bn,device=dev)0).float(),reduction='none',pos_weight=torch.tensor(PENW,device=dev))*Mk).sum()/nm - loss=POSW*l_pos+DELW*l_del+l_pen; l_ctc=torch.zeros((),device=dev); l_pr=torch.zeros((),device=dev); l_aux=torch.zeros((),device=dev) - if SCL is not None: # frozen writer classifier on the predicted x0 (low noise, style not dropped, known writer) - sel=(t=0) - if sel.any(): - ii=sel.nonzero()[:,0]; Pq=Ph_[ii]; Mq=Mk[ii]; Lq=Mq.sum(1).long(); dq=torch.cat([torch.zeros_like(Pq[:,:1]),Pq[:,1:]-Pq[:,:-1]],1) - yq=Pq[...,1]; med=torch.nanmedian(yq.masked_fill(Mq<0.5,float('nan')),1).values - Xq=torch.stack([dq[...,0]*SCALE,dq[...,1]*SCALE,torch.sigmoid(pl[ii]),yq-med[:,None]],-1)*Mq[...,None] - with torch.autocast('cuda',enabled=False): l_scl=F.cross_entropy(SCL(Xq.float(),Lq),bt['Wid'][ii]) - loss=loss+STYLE_CLS_W*l_scl - if SCON is not None: # contrastive: the predicted line (low noise, style kept) must be nearest to ITS references among the batch's other writers - sel=(t=4: - ii=sel.nonzero()[:,0]; Pq=Ph_[ii]; Mq=Mk[ii]; Lq=Mq.sum(1).long().clamp(min=1); dq=torch.cat([torch.zeros_like(Pq[:,:1]),Pq[:,1:]-Pq[:,:-1]],1) - yq=Pq[...,1]; med=torch.nanmedian(yq.masked_fill(Mq<0.5,float('nan')),1).values - Xq=torch.stack([dq[...,0]*SCALE,dq[...,1]*SCALE,torch.sigmoid(pl[ii]),yq-med[:,None]],-1)*Mq[...,None] - with torch.no_grad(): - Rr=bt['R'][ii]; RLr=bt['RL'][ii]; nI,Kn_,Tn_,_=Rr.shape; xr=Rr.flatten(0,1).float(); Lr=RLr.flatten().clamp(min=1); mr=(torch.arange(Tn_,device=dev)[None]0).float().to(dev)[...,None]; rr=F.normalize((er*wr).sum(1)/wr.sum(1).clamp(min=1),dim=-1) - with torch.autocast('cuda',enabled=False): - g_=F.normalize(SCON(Xq.float(),Lq).float(),dim=-1); lg=g_@rr.T/STYLE_CON_TAU - wk=bt['Wk'][ii]; eye=torch.eye(len(ii),device=dev,dtype=torch.bool); lg=lg.masked_fill((wk[:,None]==wk[None])&~eye,-1e4) # same writer elsewhere in the batch: not a negative - l_con=F.cross_entropy(lg,torch.arange(len(ii),device=dev)); loss=loss+STYLE_CON_W*l_con - if U is not None: - Xa,La,ya=aux_batch(arng); ea=net.enc.embed(Xa,La).float(); l_ce=F.cross_entropy(net.enc.cls(ea),ya) - en=F.normalize(ea,dim=-1); dd=torch.cdist(en,en); same=(ya[:,None]==ya[None]); eye=torch.eye(len(ya),device=dev,dtype=torch.bool) - dp=(dd*(same&~eye)).max(1).values; dn_=dd.masked_fill(same,1e4).min(1).values; l_tri=F.relu(dp-dn_+0.2).mean() - l_aux=l_ce+l_tri; loss=loss+AUXW*l_aux - if PRESS: # L1 on the pressure channel, only lines with real pressure - pmk=Mk*pm_[0][:,None]; l_pr=((xy[...,2]-z0[...,3]).abs()*pmk).sum()/pmk.sum().clamp(min=1); loss=loss+PRESW_DN*l_pr - if rec is not None and CTCW>0 and step>=CTC_START: - sel=(t0 and rec is not None)) and step%DRAFT_EVERY==0: # DRaFT-1: losses on a line generated from noise - def dsv(b_,n_): - with torch.no_grad(): v_=base.style(b_['R'][:n_],b_['RL'][:n_]) if not (STYLE_ENC or STYLE_SEMB) else None - if v_ is None: v_=net.semb_proj(b_['Se'][:n_]) if STYLE_SEMB else net.enc.style(b_['R'][:n_],b_['RL'][:n_]) - return v_+net.src_emb[b_['Src'][:n_]] if SRC_FLAG else v_ - nd=min(DRAFT_BS-(DRAFT_SCAN_N if DR is not None else 0),Bn); Pd,pd,Md=draft_gen(net,bt,dsv(bt,nd),nd) - Rd,RLd,Wd,Tkd,Tmd=bt['R'][:nd],bt['RL'][:nd],bt['Wk'][:nd],bt['Tk'][:nd],bt['Tm'][:nd] - if DR is not None: # И144: + DRAFT_SCAN_N lines of scan writers (their plan, their reference from another page), same losses - sb=qd.get(); ns=len(sb['Mk']); Ps,ps,Ms=draft_gen(net,sb,dsv(sb,ns),ns) - Pd,pd,Md=cat_pad(Pd,Ps),cat_pad(pd,ps),cat_pad(Md,Ms); Rd,RLd=cat_pad(Rd,sb['R']),cat_pad(RLd,sb['RL']); Wd=torch.cat([Wd,sb['Wk']]); Tkd,Tmd=cat_pad(Tkd,sb['Tk']),cat_pad(Tmd,sb['Tm']) - if DSCON is not None: l_dcon=con_loss(DSCON,Pd,pd,Md,Rd,RLd,Wd); loss=loss+DRAFT_CON_W*l_dcon - if DRAFT_CTC_W>0 and rec is not None: l_dctc=ctc_loss_of(rec,Pd,pd,Md,Tkd,Tmd); loss=loss+DRAFT_CTC_W*l_dctc - if not torch.isfinite(loss): - nbad+=1; print(f' non-finite loss at step {step} (#{nbad})',flush=True) - if nbad>=50: print('DENOISE ABORT: persistent NaN',flush=True); raise SystemExit(3) - continue - opt.zero_grad(set_to_none=True); loss.backward(); gn=torch.nn.utils.clip_grad_norm_(net.parameters(),1.0) - if not torch.isfinite(gn): opt.zero_grad(set_to_none=True); continue - opt.step(); step+=1; dec=min(0.999,(1+step)/(10+step)) - with torch.no_grad(): - for pe,pm in zip(ema.parameters(),net.parameters()): pe.mul_(dec).add_(pm.detach(),alpha=1-dec) - for k_,v_ in (('pos',l_pos),('del',l_del),('pen',l_pen),('ctc',l_ctc))+((('pres',l_pr),) if PRESS else ())+((('aux',l_aux),) if U is not None else ())+((('scl',l_scl),) if SCL is not None else ())+((('con',l_con),) if SCON is not None else ())+((('dcon',l_dcon),) if DSCON is not None else ())+((('dctc',l_dctc),) if DRAFT_CTC_W>0 else ()): acc[k_]=acc.get(k_,0)+float(v_.detach()) - if step%LOG_EVERY==0: - sp=(step-s0)/(time.time()-t0); msg=' '.join(f"{k_} {v_/LOG_EVERY:.4f}" for k_,v_ in acc.items()); acc={} - if rec is not None and step%(5*LOG_EVERY)==0: # monitor: ctc3 CER of the x0 prediction at LOW noise (t<10) — readability of the denoised plan - with torch.no_grad(): - lo=(t<10).nonzero()[:,0] - if len(lo): - L=Mk.sum(1).long(); lp=rec(rec_feats(Ph_.detach(),torch.sigmoid(pl.detach()),Mk),L); hy=greedy(lp,L.tolist()); tl=(bt['Tm'].sum(1)-2).long() - cs=[cer(bt['Tk'][b,1:1+tl[b]].tolist(),hy[b]) for b in lo.tolist()]; msg+=f" | ctcCER@t<10 {np.mean(cs):.3f} (n={len(cs)})" - print(f"step {step}/{STEPS} {msg} gn {float(gn):.2f} lr {lr:.2e} {sp:.2f}it/s eta {(STEPS-step)/max(sp,1e-6)/60:.0f}min",flush=True) - open(os.path.join(CKPT,'loss.csv'),'a').write(f"{step},{msg}\n") - if VB and step%VAL_EVERY==0: - vl,vs=val_loss(ema); imp=vl=PATIENCE: - SS['scale']*=PLAT_F; SS['bad']=0; SS['cuts'].append(step); print(f' PLATEAU at {step}: lr scale -> {SS["scale"]:.3g}',flush=True) - if SS['scale'] SEQS_OUT -def oracle3c_map(): - """(label, n_points) -> 3c char_idx for every v6 line (train+valid) — the v2 valid lines that are also in v6 get the 3c durations.""" - D6=os.path.join(PROJ,'data','ink_lines_v6'); z=np.load(os.path.join(D6,E('ORACLE_ALIGN','align_islands3c_clean.npz')),allow_pickle=True); al={(str(sp),int(i)):a for i,sp,a in zip(z['ids'],z['split'],z['char_idx'])}; mp={} - for sp in ('train','valid'): - r=np.load(os.path.join(D6,f'{sp}_raw.npz'),allow_pickle=True); off=r['pt_off'] - for i,l in enumerate(r['labels']): - if (sp,i) in al: mp[(str(l),int(off[i+1]-off[i]))]=al[(sp,i)] - return mp -G_IN=float(E('SPLIT_GAP','0.2')); G_OUT=float(E('JOIN_GAP','0.08')) # line-height units (x-height ~0.3) -def word_ids(C): - """per-point word index from the per-point char ids of the plan (space -> -1).""" - sp=stoi.get(' ',-1); w=np.full(len(C),-1); k=-1; insp=True - for i,x in enumerate(C): - if x==sp: insp=True; continue - if insp: k+=1; insp=False - w[i]=k - return w -def split_join(seq,w): - """seq [n,3] deltas (raw units), w per-point word index -> (splits, joins): splits = x-gaps > SPLIT_GAP inside a word's ink - (a word torn apart: «М икросостояние»), joins = neighbouring words whose ink is closer than JOIN_GAP or overlaps (words glued).""" - P=np.cumsum(np.asarray(seq,np.float32)[:,:2],0); n=min(len(P),len(w)); P=P[:n]; w=np.asarray(w[:n]); sp=0; jn=0; ext=[] - for k in range(int(w.max())+1 if (w>=0).any() else 0): - xs=np.sort(P[w==k,0]) - if len(xs)<2: continue - sp+=int((np.diff(xs)>G_IN).sum()); ext.append((xs[0],xs[-1])) - for (a0,a1),(b0,b1) in zip(ext[:-1],ext[1:]): jn+=int(b0-a1 [DN_PLAN=head]: re-derive each line's plan (head durations are deterministic) and count - splits/joins; also the REAL ink of the same v2-valid lines (ctc3 alignment) as the human reference.""" - J=json.load(open(E('SEQS_IN'))); base=load_base(); va=T2.Inks('valid'); toks=[list(l) for l in J['labels']]; durs=J.get('durs') or head_durs(base,toks) # the plan actually used (writer durations etc.) - S=[split_join(np.asarray(q),word_ids(plan([stoi.get(x,0) for x in t],d)[0])) for q,t,d in zip(J['seqs'],toks,durs)] - R=[] - for i,t in zip(J['ids'],toks): - ci=np.asarray(va.al[i],np.int64); C=[stoi.get(t[c],0) if c [PRES_WRITER=w01 N=40 OUT=png]: pressure of generated vs REAL lines of the same writer (v6 valid, - lines with real pressure, oracle 3c plan, style = other lines of the same page). Pressure normalised per writer (median -> 0.5) for both. - Prints quantiles 10/50/90, down-minus-up stroke contrast (mean p on dy>0.01 minus dy<-0.01, y down), corr(p, phase in letter); - sheet: real | generated with pen width ~ pressure.""" - from PIL import Image, ImageDraw, ImageFont - net,step=load_net(E('CKPT')); base=load_base(); va=Inks('valid'); W=E('PRES_WRITER','w01'); med=WMED.get(W) or getattr(va,'wmed',{}).get(W,0.5) - rng=np.random.default_rng(0); cand=[j for j in va.idx if va.page[j][0]==W and '$' not in va.labels[j] and 15<=len(va.labels[j])<=60 and np.ptp(va.pres[va.off[j]:va.off[j+1]])>1e-3] - ids=[cand[k] for k in rng.choice(len(cand),min(int(E('N','40')),len(cand)),replace=False)]; R={'real':[],'gen':[]}; pairs=[] - def feats(d,pn,ph): - dy=d[:,1]; return pn,dy,ph - for j in ids: - d=va.pts[va.off[j]:va.off[j+1]]; toks=va.toks[j]; ci=va.al[j].astype(np.int64); cnt=np.bincount(ci,minlength=len(toks)); start=np.concatenate([[0],np.cumsum(cnt)[:-1]]) - ph=(np.arange(len(ci))-start[ci])/np.maximum(cnt[ci],1); pr=np.clip(va.pres[va.off[j]:va.off[j+1]]/(2*med),0,1) - refs=[q.copy() for q in va.refs(j,rng)] - for q in refs: q[:,:2]/=SCALE - sv=T2.style_from_seqs(base,refs); torch.manual_seed(len(pairs)); g=sample(net,[toks],[list(cnt)],sv)[0] - C,Cp,Cn,Ph,Pc,Dp=plan([stoi.get(x,0) for x in toks],list(cnt)); n=min(len(g),len(Pc)) - R['real'].append((pr[:len(ci)],d[:len(ci),1],ph)); R['gen'].append((g[:n,3],g[:n,1],np.asarray(Pc[:n]))); pairs.append((d,pr,g)) - def summ(L): - p=np.concatenate([a for a,_,_ in L]); dy=np.concatenate([b for _,b,_ in L]); ph=np.concatenate([c for _,_,c in L]) - q=np.percentile(p,[10,50,90]); dn=p[dy>0.01].mean()-p[dy<-0.01].mean(); r=np.corrcoef(p,ph)[0,1] - return dict(q10=round(float(q[0]),3),q50=round(float(q[1]),3),q90=round(float(q[2]),3),down_minus_up=round(float(dn),3),corr_phase=round(float(r),3),n_pts=int(len(p))) - res={k:summ(v) for k,v in R.items()}; print('PRES_EVAL',W,'lines',len(ids),'step',step,json.dumps(res),flush=True) - def draw(d,p,S=60): - xy=np.cumsum(d[:,:2],0); x0,y0=xy.min(0); im=Image.new('L',(int((xy[:,0].max()-x0)*S)+20,int((xy[:,1].max()-y0)*S)+20),255); dr=ImageDraw.Draw(im); prev=None - for k in range(len(d)): - q=(10+(xy[k,0]-x0)*S,10+(xy[k,1]-y0)*S) - if prev is not None: w=max(1,int(round(1+7*p[k]))); dr.line([prev,q],fill=0,width=w) - prev=None if d[k,2]>0.5 else q - return im - f=ImageFont.truetype(os.path.join(PROJ,'DejaVuSans.ttf'),16); ims=[] - for d,pr,g in pairs[:6]: ims+= [('настоящая (нажим писца)',draw(d,pr)),('денойзер (свой нажим)',draw(g,g[:,3]))] - Wd=max(im.width for _,im in ims)+10; H=sum(im.height+22 for _,im in ims); sh=Image.new('L',(Wd,H),255); dd=ImageDraw.Draw(sh); y=0 - for cap,im in ims: dd.text((5,y),cap,fill=90,font=f); sh.paste(im,(5,y+20)); y+=im.height+22 - out=E('OUT',os.path.join(PROJ,'figures',f'denoise_pressure_{RUN}.png')); sh.save(out); print('sheet',out,flush=True) -def line_attrs(d,n): - """И143: style attributes of one line (raw dx,dy,pen points; n characters) — what the SAE knobs should move. - slant = median angle of pen-down near-vertical segments from the vertical (deg, + = to the right), len-weighted; - width = x extent / characters (размашистость in line heights: lines are height-normalised); ink = pen path / characters; - height = 5-95% y extent; lifts = pen lifts / characters; rough = mean |2nd difference| / mean |1st difference| (неровность).""" - d=np.asarray(d,np.float32); xy=np.cumsum(d[:,:2],0); dn=d[:,2]<0.5; s_=d[1:,:2][dn[:-1]] - L=np.hypot(s_[:,0],s_[:,1]); v=np.abs(s_[:,1])>1.5*np.abs(s_[:,0]) - ang=np.degrees(np.arctan2(s_[v,0]*np.sign(-s_[v,1]+1e-9),np.abs(s_[v,1]))) if v.any() else np.zeros(1) # y grows downwards: an upstroke going right = right slant - w=L[v] if v.any() else np.ones(1); o=np.argsort(ang); cw=np.cumsum(w[o]); slant=float(ang[o][np.searchsorted(cw,cw[-1]/2)]) - d1=np.diff(xy[dn],axis=0); rough=float(np.abs(np.diff(d1,axis=0)).sum(1).mean()/max(np.hypot(*d1.T).mean(),1e-6)) if len(d1)>2 else 0.0 - n=max(n,1); return dict(slant=slant,width=float(np.ptp(xy[:,0])/n),ink=float(np.hypot(d[:,0],d[:,1])[dn].sum()/n), - height=float(np.subtract(*np.percentile(xy[:,1],[95,5]))),lifts=float((d[:,2]>0.5).sum()/n),rough=rough) -def acts(): - """И143: the residual stream of a denoiser at 'mid' and 'post' on REAL train lines (training-style forward: the line + noise at a random - t, its writer's other lines as references, no dropout) -> ACTS_OUT.npz: mid/post [N_pts, D] fp16 (every SUB-th point), line [N_pts], - t [N_lines], writer, line attributes (line_attrs). Env CKPT, INK_DS/META_DS/ALIGN (the net's training set), NL (lines, 12000), SUB (4).""" - net,step=load_net(E('CKPT')); base=load_base(); tr=Inks('train'); rng=np.random.default_rng(int(E('SEED','5'))); NL=min(int(E('NL','12000')),len(tr)); SUB=int(E('SUB','4')) - pick=np.sort(rng.choice(len(tr),NL,replace=False)); g_=torch.Generator(device=dev).manual_seed(5); out=dict(mid=[],post=[],line=[]); T_,W_,A_=[],[],[] - net._cap={} - with torch.no_grad(): - for a in range(0,NL,64): - ii=pick[a:a+64]; bt=tr.batch(ii,rng); Mk=bt['Mk']; z0,*_=to_z(bt['Y'],Mk,bt.get('Pw')); Bn=len(ii) - sv=net.semb_proj(bt['Se']) if STYLE_SEMB else (net.enc.style(bt['R'],bt['RL']) if STYLE_ENC else base.style(bt['R'],bt['RL'])) - if SRC_FLAG: sv=sv+net.src_emb[bt['Src']] - if STYLE_TOK: net._tok=net.enc.tokens(bt['R'],bt['RL']) - if REFATT: net._tok=net.enc.refpts(bt['R'],bt['RL']) - t=torch.randint(0,TD,(Bn,),device=dev,generator=g_); al=AB[t][:,None,None]; zt=(al.sqrt()*z0+(1-al).sqrt()*torch.randn(z0.shape,device=dev,generator=g_))*Mk[...,None] - cond=dict(Mk=Mk,Ph=bt['Ph'],Pc=bt['Pc'],Dp=bt['Dp'],C=bt['C'],Cp=bt['Cp'],Cn=bt['Cn']) - net(zt,t,cond,sv,torch.zeros_like(zt)) - for b,i in enumerate(ii): - n=int(Mk[b].sum()); sel=torch.arange(0,n,SUB,device=dev) - for k in ('mid','post'): out[k].append(net._cap[k][b,sel].half().cpu().numpy()) - out['line'].append(np.full(len(sel),a+b,np.int32)); j=tr.idx[i] - A_.append(line_attrs(tr.pts[tr.off[j]:tr.off[j+1]],len(str(tr.labels[j])))) - T_+=t.tolist(); W_+=[str(tr.wid.get(tr.idx[i],tr.page[tr.idx[i]][0])) for i in ii] - if a%1280==0: print(f'acts {a+Bn}/{NL} lines, {sum(len(x) for x in out["line"])} points',flush=True) - net._cap=None; K=list(A_[0]) - np.savez(E('ACTS_OUT'),mid=np.concatenate(out['mid']),post=np.concatenate(out['post']),line=np.concatenate(out['line']),t=np.array(T_),writer=np.array(W_), - ids=np.array([tr.idx[i] for i in pick]),attr_names=np.array(K),attrs=np.array([[r[k] for k in K] for r in A_],np.float32),step=step,ckpt=E('CKPT')) - print('ACTS SAVED',E('ACTS_OUT'),NL,'lines',flush=True) - -def svdirs(): - """И143 step 2 (w03 29.10 «попробуй»): knobs in the STYLE VECTOR space (one vector per line, it drives every layer through FiLM / STYLE_IN), - not in the per-point residual. Every train line of INK_DS is embedded alone as a reference with the style encoder the net uses (style_of - path: base.style for REFATT/STYLE_IN nets), next to its line_attrs. Per attribute: a ridge direction of the attribute in sv space WITHIN - writer (sv and attribute minus the writer mean), 5-fold r; u = unit direction, sd = within-writer SD of sv.u (the steering unit). - ORDER (default slant,width,height,ink,rough,lifts) + Gram-Schmidt: each direction made orthogonal to the earlier ones (u_orth) — so a - 'width' push does not carry the slant one. Out: SVDIR_OUT npz (names, u, u_orth, sd, sd_orth, r_cv, attrs, writer, sv).""" - net,step=load_net(E('CKPT')); base=load_base(); tr=Inks('train'); SV,A_,W_=[],[],[] - with torch.no_grad(): - for a in range(0,len(tr.idx),64): - js=tr.idx[a:a+64]; ds=[np.asarray(tr.pts[tr.off[j]:tr.off[j+1]],np.float32)[:MAXPTS].copy() for j in js]; L=max(len(d) for d in ds) - R=np.zeros((len(js),1,L,3),np.float32); RL=np.zeros((len(js),1),np.int64) - for b,d in enumerate(ds): d[:,:2]*=SCALE; R[b,0,:len(d)]=d[:,:3]; RL[b,0]=len(d) - SV.append(base.style(torch.from_numpy(R).to(dev),torch.from_numpy(RL).to(dev)).float().cpu().numpy()) - for j in js: A_.append(line_attrs(tr.pts[tr.off[j]:tr.off[j+1]],len(str(tr.labels[j])))); W_.append(str(tr.wid.get(j,tr.page[j][0]))) - if E('EXTRA'): # EXTRA='cer=out/clean/_ckpt_ctc4_cer.npz': a per-line attribute from an (idx, cer) npz of line_ctc_cer.py (чистота = low CER) - nm_,f_=E('EXTRA').split('='); c_=np.load(f_); mp=dict(zip(c_['idx'].tolist(),c_['cer'].tolist())) - for r,j in zip(A_,tr.idx): r[nm_]=mp.get(int(j),np.nan) - SV=np.concatenate(SV); W_=np.array(W_); names=[n for n in E('ORDER','slant,width,height,ink,rough,lifts').split(',')]; At=np.array([[r[n] for n in names] for r in A_],np.float64) - ok_=np.isfinite(At).all(1); print(f'svdirs: {ok_.sum()} of {len(ok_)} lines have every attribute',flush=True); SV,W_,At=SV[ok_],W_[ok_],At[ok_] - def within(a): - a=a.astype(np.float64).copy() - for w in np.unique(W_): m_=W_==w; a[m_]-=a[m_].mean(0) - return a - Sw=within(SV); Aw=within(At); U,R_,SD=[],[],[]; fold=np.arange(len(Sw))%5 - for k,n in enumerate(names): - y=Aw[:,k]; pr=np.zeros(len(y)) - for f in range(5): - t_=fold!=f; w=np.linalg.solve(Sw[t_].T@Sw[t_]+float(E('RIDGE','10'))*np.eye(Sw.shape[1]),Sw[t_].T@y[t_]); pr[~t_]=Sw[~t_]@w - w=np.linalg.solve(Sw.T@Sw+float(E('RIDGE','10'))*np.eye(Sw.shape[1]),Sw.T@y); u=w/np.linalg.norm(w); U.append(u); R_.append(float(np.corrcoef(pr,y)[0,1])); SD.append(float((Sw@u).std())) - U=np.array(U); Uo=U.copy() - for k in range(len(U)): - for q in range(k): Uo[k]-=(Uo[k]@Uo[q])*Uo[q] - Uo[k]/=np.linalg.norm(Uo[k]) - SDo=[float((Sw@u).std()) for u in Uo] - for k,n in enumerate(names): print(f'{n:7s} r_cv {R_[k]:+.3f} sd {SD[k]:.3f} sd_orth {SDo[k]:.3f} cos to earlier: '+' '.join(f'{names[q]} {U[k]@U[q]:+.2f}' for q in range(k)),flush=True) - np.savez(E('SVDIR_OUT'),names=np.array(names),u=U,u_orth=Uo,sd=np.array(SD),sd_orth=np.array(SDo),r_cv=np.array(R_),attrs=At,writer=W_,sv=SV,sv_norm=float(np.linalg.norm(SV,axis=1).mean()),step=step) - print('SVDIRS SAVED',E('SVDIR_OUT'),len(SV),'lines, mean |sv|',round(float(np.linalg.norm(SV,axis=1).mean()),3),flush=True) - -def steer(): - """И143: steering by SAE latents (scripts/sae_train.py) while sampling: for each latent in LAT and each alpha in ALPHAS the residual - stream at the SAE's layer gets + alpha * act_mean[l] * scale * W_dec[l] at every point and every sampling step; the same N judge texts, - the same noise (seed reset), references REF_IDX of v2 valid (the judge's: w01). Per (latent, alpha): line attributes of the generated - lines (line_attrs) and CTC CER (CTC ckpt). NRAND random latents of similar firing share = the control. Out: OUT.json, OUT.png (rows = alpha). - Env: CKPT, SAE, META (acts meta npz: between-writer SD of the attributes), LAT '1,2', ALPHAS '-4,-2,0,2,4', N 20, NRAND 2, SHEET_LINES 2.""" - from train_ink_grpo import Rec, ctc_reward - from PIL import Image, ImageDraw, ImageFont - net,step=load_net(E('CKPT')); base=load_base(); va=T2.Inks('valid'); SEED=int(E('SEED','3')); N=int(E('N','20')) - SVM=E('SVDIR') # sv-direction mode: LAT = attribute names of the SVDIR npz (+ 'rand' controls), alpha in within-writer SDs of sv.u - if SVM: sd_=np.load(SVM); dn=[str(n) for n in sd_['names']]; DU=sd_['u_orth' if E('ORTHO','1')=='1' else 'u']; DS=sd_['sd_orth' if E('ORTHO','1')=='1' else 'sd']; lay='sv' - else: sae=torch.load(E('SAE'),map_location='cpu'); lay=sae['layer']; Wd=sae['W_dec']; am=sae['act_mean']; sc=float(sae['scale']) - rng=np.random.RandomState(SEED); pool=[i for i in va.idx if 4<=len(str(va.labels[i]))<=90 and '$' not in str(va.labels[i])]; ids=rng.choice(pool,min(N,len(pool)),replace=False) - labels=[str(va.labels[i]) for i in ids]; toks=[list(l) for l in labels]; durs=head_durs(base,toks) - z=np.load(os.path.join(PROJ,'data','ink_lines_v2','valid_raw.npz'),allow_pickle=True); off=z['pt_off']; ref=[z['pts'][off[i]:off[i+1]].astype(np.float32) for i in [int(x) for x in E('REF_IDX','0,16,17').split(',')]] - sv=style_of(net,base,ref); tk=tokens_of(net,ref) if (STYLE_TOK or REFATT) else None - rec=Rec(384,len(itos)).to(dev); rec.load_state_dict(torch.load(os.path.join(PROJ,E('CTC','ckpt_ctc4'),'ctc.pt'),map_location=dev)['m']); rec.eval() - mt=np.load(E('META')); names=[str(n) for n in mt['attr_names']]; At=mt['attrs']; Wr=mt['writer'] - bsd={n:float(np.std([At[Wr==w,k].mean() for w in np.unique(Wr)])) for k,n in enumerate(names)} - if SVM: - r_=np.random.RandomState(0); lats=E('LAT',','.join(dn)).split(',') - for q in range(int(E('NRAND','2'))): v_=r_.randn(DU.shape[1]); lats.append(('rand',v_/np.linalg.norm(v_))) - else: - lats=[int(x) for x in E('LAT').split(',')] - if int(E('NRAND','2')) and not SVM: - r_=np.random.RandomState(0); live=[i for i in range(len(am)) if am[i]>0 and i not in lats]; lats+=[int(x) for x in r_.choice(live,int(E('NRAND','2')),replace=False)] - alphas=[float(a) for a in E('ALPHAS','-4,-2,0,2,4').split(',')]; res=[]; sheet=[] - if E('DURS'): lats=['dur'] # DURS='0.7,1,1.3': the plan's points per letter x m (width per letter = the planner's, not the style's) — no sv/SAE push - for li,l in enumerate(lats): - for a in ([float(x) for x in E('DURS').split(',')] if l=='dur' else alphas): - if l=='dur': sv_=sv; net._steer=None - elif SVM: - v_=l[1] if isinstance(l,tuple) else DU[dn.index(l)]; s_=float(np.mean(DS)) if isinstance(l,tuple) else float(DS[dn.index(l)]) - sv_=sv+a*s_*torch.tensor(v_,dtype=sv.dtype,device=sv.device)[None]; net._steer=None - else: sv_=sv; net._steer={lay:(a*float(am[l])*sc*Wd[l]).to(dev)} if a else None - torch.manual_seed(SEED); seqs=[] - for i in range(0,len(toks),16): - net._tok_base=cat_tok([tk]*len(toks[i:i+16])) if tk is not None else None - seqs+=sample(net,toks[i:i+16],[[max(1,int(round(c*a))) for c in d_] for d_ in durs[i:i+16]] if l=='dur' else durs[i:i+16],sv_) - at=[line_attrs(np.asarray(q)[:,:3],len(t)) for q,t in zip(seqs,toks)] - with torch.no_grad(): _,cer=ctc_reward(rec,[np.asarray(q,np.float32)[:,:3] for q in seqs],toks) - r=dict(latent=(f'rand{li}' if isinstance(l,tuple) else l),alpha=a,cer=round(float(np.mean(cer)),4),**{k:round(float(np.mean([x[k] for x in at])),4) for k in names}); res.append(r) - print('STEER',json.dumps(r),flush=True); sheet.append((r['latent'],a,[render(np.asarray(q)) for q in seqs[:int(E('SHEET_LINES','2'))]])) - net._steer=None - json.dump(dict(ckpt=E('CKPT'),sae=E('SAE'),layer=lay,step=step,between_writer_sd=bsd,labels=labels,rows=res,random_controls=[] if SVM else (lats[-int(E('NRAND','2')):] if int(E('NRAND','2')) else [])),open(E('OUT')+'.json','w'),indent=1,ensure_ascii=False) - f=ImageFont.truetype(os.path.join(PROJ,'DejaVuSans.ttf'),16); Wm=max(sum(im.width+20 for im in ims) for _,_,ims in sheet)+120; Hm=sum(max(im.height for im in ims)+8 for _,_,ims in sheet)+20*len(lats) - sh=Image.new('L',(Wm,Hm),255); dr=ImageDraw.Draw(sh); y=0; prev=None - for l,a,ims in sheet: - if l!=prev: y+=20 if prev is not None else 0; prev=l - dr.text((4,y+4),f'{l} {a:+g}',fill=0,font=f); x=120 - for im in ims: sh.paste(im,(x,y)); x+=im.width+20 - y+=max(im.height for im in ims)+8 - sh.save(E('OUT')+'.png'); print('STEER DONE',E('OUT'),flush=True) - -def judge(): - net,step=load_net(E('CKPT')); base=load_base(); va=T2.Inks('valid'); SEED=int(E('SEED','3')); N=int(E('N','60')); DUR=E('DUR','head') - rng=np.random.RandomState(SEED); pool=[i for i in va.idx if 4<=len(str(va.labels[i]))<=90 and (E('KIND','text')!='text' or '$' not in str(va.labels[i]))]; ids=rng.choice(pool,min(N,len(pool)),replace=False) # KIND=text (default, the standard judge) or KIND=all (formulas/mixed included, as judge_lines_qwen without KIND) - labels=[str(va.labels[i]) for i in ids]; toks=[list(l) for l in labels] - z=np.load(os.path.join(PROJ,'data','ink_lines_v2','valid_raw.npz'),allow_pickle=True); off=z['pt_off']; ref=[z['pts'][off[i]:off[i+1]].astype(np.float32) for i in [int(x) for x in E('REF_IDX','0,16,17').split(',')]] - sv=style_of(net,base,ref); tk=tokens_of(net,ref) if (STYLE_TOK or REFATT) and not E("NOREF") else None - if E('SV_PUSH'): # И143: 'out/sae1/svdirs_c1_cer.npz:cer:-16' = the style vector pushed along an attribute direction (u_orth, alpha in sd_orth) - f_,n_,a_=E('SV_PUSH').split(':'); d_=np.load(f_); k_=[str(x) for x in d_['names']].index(n_) - sv=sv+float(a_)*float(d_['sd_orth'][k_])*torch.tensor(d_['u_orth'][k_],dtype=sv.dtype,device=sv.device)[None]; print('style pushed',E('SV_PUSH'),flush=True) # NOREF=1: the old (buggy) way, for the record; 28.10: REFATT/STYLE_TOK nets need the reference tokens too (were never set here) - if DUR=='oracle': - mp=oracle3c_map(); durs=[]; n3=0 - for i,t in zip(ids,toks): - key=(str(va.labels[i]),int(va.off[i+1]-va.off[i])); ci=mp.get(key) - if ci is not None and len(ci)==key[1]: n3+=1 - else: ci=va.al[i] - durs.append(list(np.bincount(np.asarray(ci,np.int64),minlength=len(t))[:len(t)])) - print(f'oracle durations: 3c for {n3}/{len(ids)} lines, ctc3 align_all for the rest',flush=True) - else: durs=head_durs(base,toks) - if E('WDUR_W'): durs=writer_durs(toks,durs,E('WDUR_W')); print('writer durations of',E('WDUR_W'),flush=True) - torch.manual_seed(SEED); seqs=[] - for i in range(0,len(toks),16): - net._tok_base=cat_tok([tk]*len(toks[i:i+16])) if tk is not None else None - seqs+=sample(net,toks[i:i+16],durs[i:i+16],sv) - out=E('SEQS_OUT'); json.dump(dict(labels=labels,ids=[int(i) for i in ids],step=step,durs=[[int(x) for x in d] for d in durs],seqs=[np.round(np.asarray(q,np.float32),4).tolist() for q in seqs]),open(out,'w'),ensure_ascii=False) - print('SEQS SAVED',out,len(seqs),'step',step,'DUR',DUR,'CFG',E('CFG','1.0'),flush=True) - if E('SHEET_OUT'): # quick local eyeball of the same seqs - from PIL import Image, ImageDraw, ImageFont - f=ImageFont.truetype(os.path.join(PROJ,'DejaVuSans.ttf'),14); ims=[render(q) for q in seqs[:int(E('SHEET_N','20'))]]; W=max(im.width for im in ims); H=sum(im.height+24 for im in ims) - sh=Image.new('L',(W,H),255); dr=ImageDraw.Draw(sh); y=0 - for im,l in zip(ims,labels): sh.paste(im,(0,y)); dr.text((4,y+im.height+2),l[:90],fill=0,font=f); y+=im.height+24 - sh.save(E('SHEET_OUT')); print('sheet',E('SHEET_OUT'),flush=True) - -def blind(): - """20 v2-valid texts (not the judge's seed-3 ones), 10 real + 10 denoiser in random order, no labels; key -> OUT.key.json.""" - from PIL import Image, ImageDraw, ImageFont - net,step=load_net(E('CKPT')); base=load_base(); va=T2.Inks('valid'); rng=np.random.RandomState(int(E('SEED','11'))) - pool=[i for i in va.idx if 15<=len(str(va.labels[i]))<=70 and '$' not in str(va.labels[i])]; ids=rng.choice(pool,20,replace=False); is_gen=rng.permutation([1]*10+[0]*10) - z=np.load(os.path.join(PROJ,'data','ink_lines_v2','valid_raw.npz'),allow_pickle=True); off=z['pt_off']; ref=[z['pts'][off[i]:off[i+1]].astype(np.float32) for i in (0,16,17)] - sv=T2.style_from_seqs(base,ref); toks=[list(str(va.labels[i])) for i in ids]; torch.manual_seed(0); gen=sample(net,toks,head_durs(base,toks),sv) - ims=[render(g) if k else render(va.pts[va.off[i]:va.off[i+1]]) for i,g,k in zip(ids,gen,is_gen)]; f=ImageFont.truetype(os.path.join(PROJ,'DejaVuSans.ttf'),18) - W=max(im.width for im in ims)+60; H=sum(im.height+16 for im in ims); sh=Image.new('L',(W,H),255); dr=ImageDraw.Draw(sh); y=0 - for k,im in enumerate(ims): dr.text((4,y+im.height//2-9),f'{k+1}',fill=0,font=f); sh.paste(im,(50,y)); y+=im.height+16 - out=E('OUT',os.path.join(PROJ,'figures',f'blind-denoise-{RUN}.png')); sh.save(out) - json.dump(dict(step=step,ckpt=E('CKPT'),rows=[dict(n=k+1,gen=int(g),label=str(va.labels[i])) for k,(i,g) in enumerate(zip(ids,is_gen))]),open(out+'.key.json','w'),ensure_ascii=False,indent=1) - print('blind sheet',out,'key',out+'.key.json',flush=True) - -if __name__=='__main__': - if MODE=='train': train() - elif MODE=='judge': judge() - elif MODE=='acts': acts() - elif MODE=='steer': steer() - elif MODE=='gradcos': gradcos() - elif MODE=='svdirs': svdirs() - elif MODE=='blind': blind() - elif MODE=='splits': splits() - elif MODE=='pres_eval': pres_eval() - elif MODE=='sheet': - net,step=load_net(E('CKPT')); base=load_base(); va=Inks('valid'); print('sheet',sheet(net,base,va,E('OUT',os.path.join(SAMP,f'sheet_{step}.png')),n=int(E('N','8')),seed=int(E('SEED','1'))),flush=True)