File size: 4,036 Bytes
2bf0f47 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | """Real-tokenizer T1/T3/T9 gates; requires local tokenizer snapshots and archived text."""
import argparse
import hashlib
import json
from pathlib import Path
from kdflow.token_alignment import get_tokenizer_byte_codec,align_byte_span_batch_1to1
from kdflow.algorithms._mp_opd_port._mp_opd_atoms import SimCTAtomizer
from .common import archive_rows,tokenize_record,row_groups
def main(argv=None):
p=argparse.ArgumentParser(description=__doc__)
for key in ('student','teacher','archive','texts','output'): p.add_argument('--'+key,required=True)
p.add_argument('--assume-shared-prompt',action='store_true')
args=p.parse_args(argv)
target=Path(args.output)
if target.exists(): raise FileExistsError(target)
from transformers import AutoTokenizer
toks=[AutoTokenizer.from_pretrained(s,local_files_only=True) for s in (args.student,args.teacher)]
codecs=[get_tokenizer_byte_codec(t) for t in toks]
texts=Path(args.texts).read_text().splitlines()
if len(texts)<10000: raise ValueError('T1 requires at least 10,000 texts')
# User-supplied corpus must include challenging scripts, not just 10k copies
# of one ASCII fixture. This list is supplemental and reported separately.
probes=['Tiếng Việt: một phép thử','中文 日本語 한국어','🙂🚀','\\frac{120}{3} = 40','def f(x):\n return x+1','a\t b\n']
failures=[]
for model,tok,codec in zip(('student','teacher'),toks,codecs):
for i,text in enumerate(texts+probes):
ids=tok(text,add_special_tokens=False)['input_ids']
raw=b''.join(codec.piece_to_bytes(p) for p in tok.convert_ids_to_tokens(ids))
if raw!=text.encode(): failures.append(dict(test='T1',model=model,index=i))
records=list(archive_rows(args.archive,500))
if len(records)!=500: raise ValueError('T3/T9 require 500 real responses')
atomizer=SimCTAtomizer(toks[0],toks[1])
common=0; dropped=0
for ri,record in enumerate(records):
batch=tokenize_record(record,*toks,args.assume_shared_prompt)
grouped=row_groups(batch,*toks)[0]
sid=batch['stu_input_ids'].roll(-1,1)[batch['stu_loss_mask'].bool()].tolist()
tid=batch['tea_input_ids'].roll(-1,1)[batch['tea_loss_mask'].bool()].tolist()
st,tt=toks[0].convert_ids_to_tokens(sid),toks[1].convert_ids_to_tokens(tid)
strict=align_byte_span_batch_1to1([tt],[st],toks[1].eos_token,toks[0].eos_token,
codecs[1].piece_to_bytes,codecs[0].piece_to_bytes)
actual=([g.t_idx[0] for g in grouped.groups if g.kind=='strict'],[g.s_idx[0] for g in grouped.groups if g.kind=='strict'])
if actual!=strict:
failures.append(dict(test='T3',response=ri,reason=grouped.reason or 'strict_set_difference',author_indices=strict,h1b_indices=actual))
atomized=atomizer.atomize(sid,tid,sample_id=str(ri))
if not grouped.valid or not atomized.valid: dropped+=1; continue
common+=1
# EOS is a Strict-only synchronization event; compare content spans.
pairs=[(g.s_idx[0],g.s_idx[-1]+1,g.t_idx[0],g.t_idx[-1]+1,g.kind) for g in grouped.groups if not g.eos]
reference=[(a.student_start,a.student_end,a.teacher_start,a.teacher_end,
'strict' if a.student_end-a.student_start==a.teacher_end-a.teacher_start==1 else 'mis') for a in atomized.atoms]
if pairs!=reference: failures.append(dict(test='T9',response=ri,h1b=pairs,mp_opd=reference))
out=dict(text_count=len(texts),supplemental_probes=len(probes),responses=500,common_accepted=common,
excluded_from_common=dropped,disagreements=failures,passed=not failures,
archive_sha256=hashlib.sha256(Path(args.archive).read_bytes()).hexdigest(),
qualification='Real-input alignment gate only; no GPU training')
target.parent.mkdir(parents=True,exist_ok=True); target.write_text(json.dumps(out,indent=2))
if failures: raise SystemExit('H1b alignment gate has disagreements; see report')
if __name__=='__main__': main()
|