en_generic / custom_functions.py

Update spaCy pipeline

758877c verified almost 2 years ago

671 Bytes

import spacy
from spacy.tokenizer import Tokenizer


@spacy.registry.tokenizers("custom_tokenizer")
def create_custom_tokenizer():
    def create_tokenizer(nlp):
        infixes = nlp.Defaults.infixes + [
            r"/",
            r"-",
            r",",
            r":",
        ]
        prefixes = nlp.Defaults.prefixes + [
            r"-",
        ]
        prefix_regex = spacy.util.compile_prefix_regex(prefixes)
        infix_regex = spacy.util.compile_infix_regex(infixes)
        return Tokenizer(
            nlp.vocab,
            prefix_search=prefix_regex.search,
            infix_finditer=infix_regex.finditer,
        )

    return create_tokenizer

	import spacy
	from spacy.tokenizer import Tokenizer


	@spacy.registry.tokenizers("custom_tokenizer")
	def create_custom_tokenizer():
	def create_tokenizer(nlp):
	infixes = nlp.Defaults.infixes + [
	r"/",
	r"-",
	r",",
	r":",
	]
	prefixes = nlp.Defaults.prefixes + [
	r"-",
	]
	prefix_regex = spacy.util.compile_prefix_regex(prefixes)
	infix_regex = spacy.util.compile_infix_regex(infixes)
	return Tokenizer(
	nlp.vocab,
	prefix_search=prefix_regex.search,
	infix_finditer=infix_regex.finditer,
	)

	return create_tokenizer