lkmill commited on
Commit
ff51fd5
Β·
1 Parent(s): 2191a1b

pairCount

Browse files
Files changed (2) hide show
  1. src/bpe.js +53 -0
  2. test/bpe.js +19 -1
src/bpe.js CHANGED
@@ -18,3 +18,56 @@ export function preprocess(corpus) {
18
  export function initialVocabulary(corpus) {
19
  return Array.from(new Set(corpus)).sort()
20
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
  export function initialVocabulary(corpus) {
19
  return Array.from(new Set(corpus)).sort()
20
  }
21
+
22
+ /**
23
+ * def pair_count(corpus_l):
24
+ * pairs = {}
25
+ *
26
+ * for i in range(0, len(corpus_l) - 1):
27
+ * if corpus_l[i + 1].startswith('▁'):
28
+ * continue
29
+ *
30
+ * bigram = (corpus_l[i], corpus_l[i + 1])
31
+ * if bigram in pairs:
32
+ * pairs[bigram] += 1
33
+ * else:
34
+ * pairs[bigram] = 1
35
+ *
36
+ * return dict(sorted(pairs.items(), reverse=True, key=lambda pair: pair[1]))
37
+ *
38
+ * @param {string} corpus_l
39
+ * @returns {Record<string,number>}
40
+ */
41
+ export function pairCount(corpus_l) {
42
+ const pairs = {}
43
+
44
+ for (let i = 0; i < corpus_l.length - 1; i++) {
45
+ const b = corpus_l[i + 1]
46
+
47
+ if (b.startsWith('▁')) continue
48
+
49
+ const a = corpus_l[i]
50
+
51
+ const bigram = `${a}|${b}`
52
+
53
+ if (pairs[bigram]) {
54
+ pairs[bigram] = pairs[bigram] + 1
55
+ } else {
56
+ pairs[bigram] = 1
57
+ }
58
+ }
59
+
60
+ return Object.fromEntries(
61
+ Object.entries(pairs).sort(([, a], [, b]) => {
62
+ if (a > b) {
63
+ return -1
64
+ }
65
+
66
+ if (b > a) {
67
+ return 1
68
+ }
69
+
70
+ return 0
71
+ }),
72
+ )
73
+ }
test/bpe.js CHANGED
@@ -1,7 +1,7 @@
1
  import { test } from 'node:test'
2
  import herrgard from '../corpus/herrgard.txt' with { type: 'text' }
3
 
4
- import { preprocess, initialVocabulary } from '../src/bpe.js'
5
 
6
  test('preprocess', (t) => {
7
  const expected =
@@ -83,3 +83,21 @@ test('initialVocabulary', (t) => {
83
 
84
  t.assert.deepEqual(initialVocabulary(preprocess(herrgard)), expected)
85
  })
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import { test } from 'node:test'
2
  import herrgard from '../corpus/herrgard.txt' with { type: 'text' }
3
 
4
+ import { preprocess, initialVocabulary, pairCount } from '../src/bpe.js'
5
 
6
  test('preprocess', (t) => {
7
  const expected =
 
83
 
84
  t.assert.deepEqual(initialVocabulary(preprocess(herrgard)), expected)
85
  })
86
+
87
+ test('pairCount', (t) => {
88
+ const expected = [
89
+ ['▁|s', 4147],
90
+ ['d|e', 4016],
91
+ ['▁|h', 3974],
92
+ ['e|n', 3439],
93
+ ['▁|d', 2469],
94
+ ['a|n', 2441],
95
+ ['t|t', 2178],
96
+ ['a|r', 1970],
97
+ ['e|t', 1954],
98
+ ['▁|a', 1940],
99
+ ]
100
+ const result = Object.entries(pairCount(preprocess(herrgard))).slice(0, 10)
101
+
102
+ t.assert.deepEqual(result, expected)
103
+ })