Download kdflow/h1b/groups.py from codemaivanngu/ctopd: direct link, hf CLI and curl.
- Browser
- Download file 2.9 kB
-
https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/h1b/groups.py
- Command line
-
hf download hf://codemaivanngu/ctopd/kdflow/h1b/groups.py
-
curl -L -o groups.py https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/h1b/groups.py
2.9 kB
| """Byte-piece groups using Strict's codecs and EOS segmentation.""" | |
| from dataclasses import dataclass | |
| from itertools import zip_longest | |
| from kdflow.token_alignment import _split_on_eos | |
| class Group: | |
| kind: str | |
| s_idx: tuple | |
| t_idx: tuple | |
| b0: int | |
| b1: int | |
| content: bytes | |
| eos: bool = False | |
| class RowGroups: | |
| groups: tuple | |
| valid: bool | |
| reason: str = '' | |
| student_tokens: int = 0 | |
| teacher_tokens: int = 0 | |
| def align_byte_span_groups_batch(tea_tokens, stu_tokens, teacher_eos_token, | |
| student_eos_token, teacher_piece_to_bytes, | |
| student_piece_to_bytes): | |
| if len(tea_tokens) != len(stu_tokens): | |
| raise ValueError('H1b response counts differ') | |
| rows, so, to = [], 0, 0 | |
| for ts, ss in zip(tea_tokens, stu_tokens): | |
| groups, reason, sc, tc = [], '', 0, 0 | |
| tsegments, ssegments = _split_on_eos(ts, teacher_eos_token), _split_on_eos(ss, student_eos_token) | |
| for titem, sitem in zip_longest(tsegments, ssegments): | |
| if titem is None or sitem is None: | |
| reason = 'eos_segment_mismatch' | |
| break | |
| tseg, teos = titem | |
| sseg, seos = sitem | |
| tp = [teacher_piece_to_bytes(p) for _, p in tseg] | |
| sp = [student_piece_to_bytes(p) for _, p in sseg] | |
| tc += len(tp) | |
| sc += len(sp) | |
| if b''.join(tp) != b''.join(sp): | |
| reason = 'byte_mismatch' | |
| break | |
| if any(not p for p in tp + sp): | |
| reason = 'empty_piece' | |
| break | |
| ti = si = t0 = s0 = b0 = 0 | |
| tend, send = [], [] | |
| for p in tp: tend.append((tend[-1] if tend else 0) + len(p)) | |
| for p in sp: send.append((send[-1] if send else 0) + len(p)) | |
| while ti < len(tp) and si < len(sp): | |
| if tend[ti] < send[si]: ti += 1; continue | |
| if tend[ti] > send[si]: si += 1; continue | |
| tt = tuple(to + tseg[k][0] for k in range(t0, ti + 1)) | |
| st = tuple(so + sseg[k][0] for k in range(s0, si + 1)) | |
| groups.append(Group('strict' if len(tt) == len(st) == 1 else 'mis', | |
| st, tt, b0, tend[ti], b''.join(tp[t0:ti+1]))) | |
| b0 = tend[ti] | |
| ti += 1; si += 1; t0, s0 = ti, si | |
| if teos is not None and seos is not None: | |
| groups.append(Group('strict', (so+seos,), (to+teos,), b0, b0, b'', True)) | |
| # Content denominators count even a rejected response completely. | |
| sc = sum(p != student_eos_token for p in ss) | |
| tc = sum(p != teacher_eos_token for p in ts) | |
| rows.append(RowGroups(tuple(groups) if not reason else (), not reason, reason, sc, tc)) | |
| so += len(ss); to += len(ts) | |
| return rows | |