lkmill's picture
reorganise files and fix import bugs
aec018b
Raw History Blame Contribute Delete
447 Bytes
/**
* corpus_raw = f.read().strip()
* corpus = '▁' + re.sub(r'\s+', '▁', corpus_raw)
*
* @param {string} corpus
*/
export function preprocess(corpus) {
return '▁' + corpus.trim().replaceAll(/\s+/g, '▁')
}
/**
* def initial_vocabulary(corpus_l):
* return sorted(set(corpus_l))
*
* @param {string} corpus
* @returns {string[]}
*/
export function getInitialVocabulary(corpus) {
return Array.from(new Set(corpus)).sort()
}