Hugging Face's logo

fffoivos
/
apertus-tokenizer-extension

Greek
tokenizers
tokenizer
greek
apertus
continued-pretraining
Model card Files
xet
 Community
apertus-tokenizer-extension
File size: 1,968 Bytes
f46b92a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
{
  "repo": "https://github.com/fffoivos/glossapi-tokenizer-extension",
  "branch": "main",
  "source_commit": "7c67cf11eb3d31f0ddd006f7d709709037db93a1",
  "subproject": "subprojects/03_apertus_extension_and_embedding_adaptation",
  "script_families": [
    {
      "family": "tokenizer_shipping",
      "paths": [
        "03_3_cscs_experiments_kickoff/ship/apertus_greek_modern_only_148480",
        "03_3_cscs_experiments_kickoff/ship/apertus_greek_extended_153600"
      ],
      "artifacts": [
        "tokenizer.ModernGreek-148k",
        "tokenizer.ModernGreek-Polytonic-154k"
      ]
    },
    {
      "family": "corpus_build",
      "paths": [
        "03_2_apertus_c3_dedup_audit/CPT_DATASET_BUILD_RUNBOOK.md",
        "03_4_implementation_experiments/init_bakeoff/corpus_build"
      ],
      "artifacts": [
        "dataset.CPT-7B-mix"
      ]
    },
    {
      "family": "init_and_conversion",
      "paths": [
        "03_4_implementation_experiments/init_bakeoff/arms",
        "03_4_implementation_experiments/init_bakeoff/megatron_patches"
      ],
      "artifacts": [
        "checkpoint.TokenDistil-Init",
        "checkpoint.Vanilla-3.5B",
        "checkpoint.ReTok-3.5B"
      ]
    },
    {
      "family": "token_distillation",
      "paths": [
        "TOKEN_DISTILLATION_PLAN.md",
        "03_4_implementation_experiments/init_bakeoff/token_distillation"
      ],
      "artifacts": [
        "checkpoint.TokenDistil-Init"
      ]
    },
    {
      "family": "training",
      "paths": [
        "TRAINING_RECIPE.md",
        "03_4_implementation_experiments/init_bakeoff/bakeoff_training"
      ],
      "artifacts": [
        "checkpoint.TokenDistil-3.5B",
        "checkpoint.Vanilla-3.5B",
        "checkpoint.ReTok-3.5B"
      ]
    },
    {
      "family": "evaluation",
      "paths": [
        "03_4_implementation_experiments/init_bakeoff/eval"
      ],
      "artifacts": [
        "results.3.5B-comparison"
      ]
    }
  ]
}