File size: 5,936 Bytes
6105b30
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
"""
CodeSymb whitespace codec: newlines + indentation without a new vocabulary
==========================================================================
The CodeSymb SentencePiece model (nmt_nfkc, no byte fallback) turns every
"\\n", "\\t" and run of spaces into ONE space, so code loses its lines and
indentation.  This codec gives 7 token ids that NEVER occurred in 5.5B
tokens of the training pool a fixed whitespace meaning:

    NL  = newline           TAB = tab
    SP1, SP2, SP4, SP8, SP16 = 1 / 2 / 4 / 8 / 16 spaces (any width = sum)

Same vocabulary, same model, same embeddings table: training continues.
Text between whitespace is still encoded by SentencePiece itself.

  enc = WSCodec(sp)            # wraps a SentencePieceProcessor
  ids = enc.encode(text)       # list[int]
  txt = enc.decode(ids)        # exact for code (see limits)
Everything else (eos_id, get_piece_size, ...) is passed through to sp.

Turn on everywhere with:  export CODESYMB_WS_CODEC=1
(codesymb_common.load_tokenizer then returns a WSCodec).

Limits: single trailing spaces at line end are dropped, and the
tokenizer's NFKC normalisation still applies inside a line.
"""
import re

# id -> (meaning, piece text that must sit at that id in the tokenizer)
CODEC = {
    46632: ("\n", "CHOPEL"),
    46642: ("\t", "ETHFSx"),
    46648: (" ", "GENlep"),
    42206: ("  ", "FakeNodeGroup"),
    37796: ("    ", "JobTickStatus"),
    42279: (" " * 8, "Repairprogress"),
    42227: (" " * 16, "getyesterdaty"),
}
NL, TAB, SP1, SP2, SP4, SP8, SP16 = (46632, 46642, 46648, 42206, 37796,
                                     42279, 42227)
SPACE_IDS = ((16, SP16), (8, SP8), (4, SP4), (2, SP2), (1, SP1))
WS_SPLIT = re.compile(r"([ \t]{2,}|\t)")
LEAD = re.compile(r"[ \t]*")


def spaces(n, out):
    for w, i in SPACE_IDS:
        while n >= w:
            out.append(i)
            n -= w


def ws_ids(ws, out):
    run = 0
    for ch in ws:
        if ch == " ":
            run += 1
        else:                                   # tab
            spaces(run, out)
            run = 0
            out.append(TAB)
    spaces(run, out)


class WSCodec:
    def __init__(self, sp):
        self.sp = sp
        bad = [(i, p, sp.id_to_piece(i)) for i, (_, p) in CODEC.items()
               if i >= sp.get_piece_size() or sp.id_to_piece(i) != p]
        if bad:
            raise RuntimeError(f"ws_codec does not match this tokenizer: {bad}")
        self.ids = frozenset(CODEC)
        self.text = {i: m for i, (m, _) in CODEC.items()}

    def __getattr__(self, name):                # eos_id, get_piece_size, ...
        return getattr(self.sp, name)

    def encode(self, text, *args, **kwargs):
        if not isinstance(text, str):           # batch call
            return [self.encode(t) for t in text]
        text = text.replace("\r\n", "\n").replace("\r", "\n")
        plan, segs = [], []                     # plan: ints or ("seg", k)
        for li, line in enumerate(text.split("\n")):
            if li:
                plan.append(NL)
            lead = LEAD.match(line).group()
            if lead:
                ws_ids(lead, plan)
            for k, part in enumerate(WS_SPLIT.split(line[len(lead):])):
                if not part:
                    continue
                if k % 2:                       # whitespace run
                    ws_ids(part, plan)
                elif part.strip():
                    plan.append(("seg", len(segs)))
                    segs.append(part)
        enc = self.sp.encode(segs) if segs else []
        out = []
        for x in plan:
            if isinstance(x, tuple):
                out.extend(i for i in enc[x[1]] if i not in self.ids)
            else:
                out.append(x)
        return out

    def decode(self, ids, *args, **kwargs):
        if ids and not isinstance(ids[0], int) and hasattr(ids[0], "__len__"):
            return [self.decode(x) for x in ids]
        out, run = [], []
        for i in ids:
            i = int(i)
            if i in self.ids:
                if run:
                    out.append(self.sp.decode(run))
                    run = []
                out.append(self.text[i])
            else:
                run.append(i)
        if run:
            out.append(self.sp.decode(run))
        return "".join(out)


SAMPLES = [
    "def f(x):\n    if x:\n        return 1\n\treturn  2\n",
    "class A:\n\n    def g(self, a,  b):\n        # comment   here\n"
    "        return {'k': [1, 2,   3]}\n\n\nprint(A().g(1, 2))\n",
    "<div class=\"box\">\n  <p>Hello,  world</p>\n</div>\n",
    "int main() {\n\tfor (int i = 0; i < 3; i++) {\n\t\tputs(\"hi\");\n\t}\n}\n",
    "server:\n  port: 8080\n  hosts:\n    - a.example.com\n    - b.example.com\n",
    "Plain prose line one.\nLine two, after a newline.\n\nNew paragraph.",
]


def selftest():
    """python3 ws_codec.py  -> round-trips code samples with the REAL
    CodeSymb tokenizer and shows the before/after."""
    import os
    import sys
    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
    import codesymb_common as C
    os.environ.pop("CODESYMB_WS_CODEC", None)
    sp = C.load_tokenizer(None)
    enc = WSCodec(sp)
    ok = 0
    for s in SAMPLES:
        back = enc.decode(enc.encode(s))
        flat = sp.decode(sp.encode(s))
        want = "\n".join(re.sub(r"(?<=\S) $", "", line)
                         for line in s.split("\n"))
        good = back == want
        ok += good
        print(f"{'OK ' if good else 'BAD'} {len(enc.encode(s)):4d} tok "
              f"(plain {len(sp.encode(s)):4d}) | old: {flat[:50]!r}")
        if not good:
            print(f"     want {want!r}\n     got  {back!r}")
    print(f"\n{ok}/{len(SAMPLES)} samples round-trip exactly"
          + ("  -> codec READY" if ok == len(SAMPLES) else "  -> NOT ready"))
    return 0 if ok == len(SAMPLES) else 1


if __name__ == "__main__":
    raise SystemExit(selftest())