Commit ·
bcf3607
1
Parent(s): 53f433b
feat: Enable ContextualCorrector (BERT MLM) for spelling - Pre-download AutoModelForMaskedLM in Dockerfile
Browse files- Dockerfile +4 -2
- src/nlp/spelling/araspell_service.py +1 -1
Dockerfile
CHANGED
|
@@ -33,14 +33,16 @@ print('Summarization model cached!'); \
|
|
| 33 |
# 2. Spelling model (AraSpell — AraBERT encoder-decoder + checkpoint)
|
| 34 |
RUN python -c "\
|
| 35 |
from huggingface_hub import hf_hub_download; \
|
| 36 |
-
from transformers import AutoTokenizer, EncoderDecoderModel; \
|
| 37 |
print('Downloading AraSpell checkpoint...'); \
|
| 38 |
hf_hub_download(repo_id='bayan10/AraSpell-Model', filename='last_model.pt'); \
|
| 39 |
print('Downloading AraBERT tokenizer...'); \
|
| 40 |
AutoTokenizer.from_pretrained('aubmindlab/bert-base-arabertv02'); \
|
| 41 |
print('Downloading AraBERT encoder-decoder...'); \
|
| 42 |
EncoderDecoderModel.from_encoder_decoder_pretrained('aubmindlab/bert-base-arabertv02', 'aubmindlab/bert-base-arabertv02'); \
|
| 43 |
-
print('
|
|
|
|
|
|
|
| 44 |
"
|
| 45 |
|
| 46 |
# Copy application code
|
|
|
|
| 33 |
# 2. Spelling model (AraSpell — AraBERT encoder-decoder + checkpoint)
|
| 34 |
RUN python -c "\
|
| 35 |
from huggingface_hub import hf_hub_download; \
|
| 36 |
+
from transformers import AutoTokenizer, EncoderDecoderModel, AutoModelForMaskedLM; \
|
| 37 |
print('Downloading AraSpell checkpoint...'); \
|
| 38 |
hf_hub_download(repo_id='bayan10/AraSpell-Model', filename='last_model.pt'); \
|
| 39 |
print('Downloading AraBERT tokenizer...'); \
|
| 40 |
AutoTokenizer.from_pretrained('aubmindlab/bert-base-arabertv02'); \
|
| 41 |
print('Downloading AraBERT encoder-decoder...'); \
|
| 42 |
EncoderDecoderModel.from_encoder_decoder_pretrained('aubmindlab/bert-base-arabertv02', 'aubmindlab/bert-base-arabertv02'); \
|
| 43 |
+
print('Downloading AraBERT MLM (for ContextualCorrector)...'); \
|
| 44 |
+
AutoModelForMaskedLM.from_pretrained('aubmindlab/bert-base-arabertv02'); \
|
| 45 |
+
print('Spelling model + MLM cached!'); \
|
| 46 |
"
|
| 47 |
|
| 48 |
# Copy application code
|
src/nlp/spelling/araspell_service.py
CHANGED
|
@@ -80,7 +80,7 @@ def get_spelling_model():
|
|
| 80 |
# 6. Initialize the spell checker pipeline (contextual=False to save RAM)
|
| 81 |
from nlp.spelling.araspell_rules import ArabicSpellChecker
|
| 82 |
_spell_checker = ArabicSpellChecker(
|
| 83 |
-
model, tokenizer, device, use_contextual=
|
| 84 |
)
|
| 85 |
|
| 86 |
elapsed = time.time() - t0
|
|
|
|
| 80 |
# 6. Initialize the spell checker pipeline (contextual=False to save RAM)
|
| 81 |
from nlp.spelling.araspell_rules import ArabicSpellChecker
|
| 82 |
_spell_checker = ArabicSpellChecker(
|
| 83 |
+
model, tokenizer, device, use_contextual=True
|
| 84 |
)
|
| 85 |
|
| 86 |
elapsed = time.time() - t0
|