Hugging Face's logo

SlayerLab
/
TokenizerTest-32m-A

Text Generation
PyTorch
Safetensors
Polish
polish
language-model
research
tokenizer-comparison
Model card Files
xet
 Community
TokenizerTest-32m-A
File size: 2,215 Bytes
9e59882
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
{
 "definition": "bits/byte = sum of NLL over text tokens / ln 2 / UTF-8 bytes; non-overlapping 1024-token windows; EOS targets excluded",
 "trainer_sha256": "0223c0830ed4ae323f6424038a2de7bfb6b425653f3591dd55844f95706f5d17",
 "tokenizer_sha256": "60e23148ae8ee2f8b5e8e2a42d23cedbc550fdccb598e0262c15b8c6af60d9c5",
 "sets": {
  "W": {
   "sha256": "234e7ec21bf2236bc74425591531f2c4772f8c49ff853d57f6fa16f33ee90ac7",
   "docs": 463,
   "text_bytes": 1430217,
   "tokens_text": 341025
  },
  "val": {
   "sha256": "748d13898e736666d96f1c9307222c52414a9b66b06bf212e7a07e765babc668",
   "docs": 2723,
   "text_bytes": 7232820,
   "tokens_text": 1639802
  }
 },
 "device": "cpu",
 "ckpts": [
  {
   "step": 10000,
   "ckpt_sha256": "985246af7a264adb3ccacb1968c6424c485153d0ed2d6afe1fe942a8f3854bf8",
   "W": 1.2327165523642996,
   "val": 1.1909538985721075
  },
  {
   "step": 20000,
   "ckpt_sha256": "089dbd5b141657416caab8d29c93139d0a993ecb443810bbe426e9e1e50039fe",
   "W": 1.179908492896963,
   "val": 1.1447952006986657
  },
  {
   "step": 30000,
   "ckpt_sha256": "930dbc5dd583cf6c489bea0f334805879c1e3cfb4eaa0161f5969362bea1f52a",
   "W": 1.1499359005482026,
   "val": 1.119994574275409
  },
  {
   "step": 40000,
   "ckpt_sha256": "fb54576a3504f496dd31a47043bfa4a13c38f375fbdafe1dbb15640bda9740a5",
   "W": 1.1303579741310747,
   "val": 1.1032037185642838
  },
  {
   "step": 50000,
   "ckpt_sha256": "a4b60f59dc85578435f37f53fdd5513d7907fa627b13f71330d387667c627750",
   "W": 1.114065574242555,
   "val": 1.0888208051528139
  },
  {
   "step": 60000,
   "ckpt_sha256": "2ee24f6e1d09faea31f17f472c72dce9fbc6f29b6f3906cb40eacc36bd8bc93c",
   "W": 1.0999765100751717,
   "val": 1.0772723645560955
  },
  {
   "step": 70000,
   "ckpt_sha256": "95150f77acc6ab16d0e877e0bf4cf91ee21b54225d5cc2e31989617164e20e59",
   "W": 1.0885261369122678,
   "val": 1.0674803839610683
  },
  {
   "step": 80000,
   "ckpt_sha256": "066adbf14e74e7cf6deb887c9b8f5536a9dac896b7fa0f40da16cae816f17cf7",
   "W": 1.0817551480434846,
   "val": 1.061001320817973
  },
  {
   "step": 89000,
   "ckpt_sha256": "19040910cb8bd11136364da88d01080f0fcb3f281383b3bfb27bf347f78cc92d",
   "W": 1.078115063691179,
   "val": 1.0582625876804834
  }
 ]
}