techprotrade commited on
Commit
4deb34c
·
verified ·
1 Parent(s): 107c15c

Add loan-advisor-agent

Browse files
loan-advisor-agent/.venv/Lib/site-packages/tiktoken_ext/openai_public.py ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from tiktoken.load import data_gym_to_mergeable_bpe_ranks, load_tiktoken_bpe
2
+
3
+ ENDOFTEXT = "<|endoftext|>"
4
+ FIM_PREFIX = "<|fim_prefix|>"
5
+ FIM_MIDDLE = "<|fim_middle|>"
6
+ FIM_SUFFIX = "<|fim_suffix|>"
7
+ ENDOFPROMPT = "<|endofprompt|>"
8
+
9
+ # The pattern in the original GPT-2 release is:
10
+ # r"""'s|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
11
+ # This is equivalent, but executes faster:
12
+ r50k_pat_str = (
13
+ r"""'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\s"""
14
+ )
15
+
16
+
17
+ def gpt2():
18
+ mergeable_ranks = data_gym_to_mergeable_bpe_ranks(
19
+ vocab_bpe_file="https://openaipublic.blob.core.windows.net/gpt-2/encodings/main/vocab.bpe",
20
+ encoder_json_file="https://openaipublic.blob.core.windows.net/gpt-2/encodings/main/encoder.json",
21
+ vocab_bpe_hash="1ce1664773c50f3e0cc8842619a93edc4624525b728b188a9e0be33b7726adc5",
22
+ encoder_json_hash="196139668be63f3b5d6574427317ae82f612a97c5d1cdaf36ed2256dbf636783",
23
+ )
24
+ return {
25
+ "name": "gpt2",
26
+ "explicit_n_vocab": 50257,
27
+ "pat_str": r50k_pat_str,
28
+ "mergeable_ranks": mergeable_ranks,
29
+ "special_tokens": {ENDOFTEXT: 50256},
30
+ }
31
+
32
+
33
+ def r50k_base():
34
+ mergeable_ranks = load_tiktoken_bpe(
35
+ "https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken",
36
+ expected_hash="306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930",
37
+ )
38
+ return {
39
+ "name": "r50k_base",
40
+ "explicit_n_vocab": 50257,
41
+ "pat_str": r50k_pat_str,
42
+ "mergeable_ranks": mergeable_ranks,
43
+ "special_tokens": {ENDOFTEXT: 50256},
44
+ }
45
+
46
+
47
+ def p50k_base():
48
+ mergeable_ranks = load_tiktoken_bpe(
49
+ "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken",
50
+ expected_hash="94b5ca7dff4d00767bc256fdd1b27e5b17361d7b8a5f968547f9f23eb70d2069",
51
+ )
52
+ return {
53
+ "name": "p50k_base",
54
+ "explicit_n_vocab": 50281,
55
+ "pat_str": r50k_pat_str,
56
+ "mergeable_ranks": mergeable_ranks,
57
+ "special_tokens": {ENDOFTEXT: 50256},
58
+ }
59
+
60
+
61
+ def p50k_edit():
62
+ mergeable_ranks = load_tiktoken_bpe(
63
+ "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken",
64
+ expected_hash="94b5ca7dff4d00767bc256fdd1b27e5b17361d7b8a5f968547f9f23eb70d2069",
65
+ )
66
+ special_tokens = {ENDOFTEXT: 50256, FIM_PREFIX: 50281, FIM_MIDDLE: 50282, FIM_SUFFIX: 50283}
67
+ return {
68
+ "name": "p50k_edit",
69
+ "pat_str": r50k_pat_str,
70
+ "mergeable_ranks": mergeable_ranks,
71
+ "special_tokens": special_tokens,
72
+ }
73
+
74
+
75
+ def cl100k_base():
76
+ mergeable_ranks = load_tiktoken_bpe(
77
+ "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
78
+ expected_hash="223921b76ee99bde995b7ff738513eef100fb51d18c93597a113bcffe865b2a7",
79
+ )
80
+ special_tokens = {
81
+ ENDOFTEXT: 100257,
82
+ FIM_PREFIX: 100258,
83
+ FIM_MIDDLE: 100259,
84
+ FIM_SUFFIX: 100260,
85
+ ENDOFPROMPT: 100276,
86
+ }
87
+ return {
88
+ "name": "cl100k_base",
89
+ "pat_str": r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s""",
90
+ "mergeable_ranks": mergeable_ranks,
91
+ "special_tokens": special_tokens,
92
+ }
93
+
94
+
95
+ def o200k_base():
96
+ mergeable_ranks = load_tiktoken_bpe(
97
+ "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken",
98
+ expected_hash="446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2d",
99
+ )
100
+ special_tokens = {ENDOFTEXT: 199999, ENDOFPROMPT: 200018}
101
+ # This regex could be made more efficient. If I was the one working on this encoding, I would
102
+ # have done a few other things differently too, e.g. I think you can allocate tokens more
103
+ # efficiently across languages.
104
+ pat_str = "|".join(
105
+ [
106
+ r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?""",
107
+ r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?""",
108
+ r"""\p{N}{1,3}""",
109
+ r""" ?[^\s\p{L}\p{N}]+[\r\n/]*""",
110
+ r"""\s*[\r\n]+""",
111
+ r"""\s+(?!\S)""",
112
+ r"""\s+""",
113
+ ]
114
+ )
115
+ return {
116
+ "name": "o200k_base",
117
+ "pat_str": pat_str,
118
+ "mergeable_ranks": mergeable_ranks,
119
+ "special_tokens": special_tokens,
120
+ }
121
+
122
+
123
+ def o200k_harmony():
124
+ base_enc = o200k_base()
125
+ name = "o200k_harmony"
126
+ pat_str = base_enc["pat_str"]
127
+ mergeable_ranks = base_enc["mergeable_ranks"]
128
+ special_tokens = {
129
+ **base_enc["special_tokens"],
130
+ "<|startoftext|>": 199998,
131
+ "<|endoftext|>": 199999,
132
+ "<|reserved_200000|>": 200000,
133
+ "<|reserved_200001|>": 200001,
134
+ "<|return|>": 200002,
135
+ "<|constrain|>": 200003,
136
+ "<|reserved_200004|>": 200004,
137
+ "<|channel|>": 200005,
138
+ "<|start|>": 200006,
139
+ "<|end|>": 200007,
140
+ "<|message|>": 200008,
141
+ "<|reserved_200009|>": 200009,
142
+ "<|reserved_200010|>": 200010,
143
+ "<|reserved_200011|>": 200011,
144
+ "<|call|>": 200012,
145
+ } | {f"<|reserved_{i}|>": i for i in range(200013, 201088)}
146
+ return {
147
+ "name": name,
148
+ "pat_str": pat_str,
149
+ "mergeable_ranks": mergeable_ranks,
150
+ "special_tokens": special_tokens,
151
+ }
152
+
153
+
154
+ ENCODING_CONSTRUCTORS = {
155
+ "gpt2": gpt2,
156
+ "r50k_base": r50k_base,
157
+ "p50k_base": p50k_base,
158
+ "p50k_edit": p50k_edit,
159
+ "cl100k_base": cl100k_base,
160
+ "o200k_base": o200k_base,
161
+ "o200k_harmony": o200k_harmony,
162
+ }