youssefreda9 commited on
Commit
bcf3607
·
1 Parent(s): 53f433b

feat: Enable ContextualCorrector (BERT MLM) for spelling - Pre-download AutoModelForMaskedLM in Dockerfile

Browse files
Dockerfile CHANGED
@@ -33,14 +33,16 @@ print('Summarization model cached!'); \
33
  # 2. Spelling model (AraSpell — AraBERT encoder-decoder + checkpoint)
34
  RUN python -c "\
35
  from huggingface_hub import hf_hub_download; \
36
- from transformers import AutoTokenizer, EncoderDecoderModel; \
37
  print('Downloading AraSpell checkpoint...'); \
38
  hf_hub_download(repo_id='bayan10/AraSpell-Model', filename='last_model.pt'); \
39
  print('Downloading AraBERT tokenizer...'); \
40
  AutoTokenizer.from_pretrained('aubmindlab/bert-base-arabertv02'); \
41
  print('Downloading AraBERT encoder-decoder...'); \
42
  EncoderDecoderModel.from_encoder_decoder_pretrained('aubmindlab/bert-base-arabertv02', 'aubmindlab/bert-base-arabertv02'); \
43
- print('Spelling model cached!'); \
 
 
44
  "
45
 
46
  # Copy application code
 
33
  # 2. Spelling model (AraSpell — AraBERT encoder-decoder + checkpoint)
34
  RUN python -c "\
35
  from huggingface_hub import hf_hub_download; \
36
+ from transformers import AutoTokenizer, EncoderDecoderModel, AutoModelForMaskedLM; \
37
  print('Downloading AraSpell checkpoint...'); \
38
  hf_hub_download(repo_id='bayan10/AraSpell-Model', filename='last_model.pt'); \
39
  print('Downloading AraBERT tokenizer...'); \
40
  AutoTokenizer.from_pretrained('aubmindlab/bert-base-arabertv02'); \
41
  print('Downloading AraBERT encoder-decoder...'); \
42
  EncoderDecoderModel.from_encoder_decoder_pretrained('aubmindlab/bert-base-arabertv02', 'aubmindlab/bert-base-arabertv02'); \
43
+ print('Downloading AraBERT MLM (for ContextualCorrector)...'); \
44
+ AutoModelForMaskedLM.from_pretrained('aubmindlab/bert-base-arabertv02'); \
45
+ print('Spelling model + MLM cached!'); \
46
  "
47
 
48
  # Copy application code
src/nlp/spelling/araspell_service.py CHANGED
@@ -80,7 +80,7 @@ def get_spelling_model():
80
  # 6. Initialize the spell checker pipeline (contextual=False to save RAM)
81
  from nlp.spelling.araspell_rules import ArabicSpellChecker
82
  _spell_checker = ArabicSpellChecker(
83
- model, tokenizer, device, use_contextual=False
84
  )
85
 
86
  elapsed = time.time() - t0
 
80
  # 6. Initialize the spell checker pipeline (contextual=False to save RAM)
81
  from nlp.spelling.araspell_rules import ArabicSpellChecker
82
  _spell_checker = ArabicSpellChecker(
83
+ model, tokenizer, device, use_contextual=True
84
  )
85
 
86
  elapsed = time.time() - t0