File size: 1,349 Bytes
ca0bb3f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | PYTHON = python3
SP_PREFIX = amaia
VOCAB_SIZE = 32768
CKPT = amaia.pt
SP_MODEL = amaia.model
GUTENBERG_IDS = 84 1342 11 98 2701 1661 345 844 1952 174 5200 76 1400 2600 4300
.PHONY: all vocab train stream run clean help
all: vocab
vocab: train.txt
$(PYTHON) vocab.py train.txt --prefix $(SP_PREFIX) --vocab-size $(VOCAB_SIZE)
train: $(SP_MODEL)
$(PYTHON) train.py train.txt --sp-model $(SP_MODEL) --ckpt $(CKPT)
stream: $(SP_MODEL)
@echo "Streaming $(words $(GUTENBERG_IDS)) books from Project Gutenberg..."
@for id in $(GUTENBERG_IDS); do \
echo " book $$id" >&2; \
curl -s --max-time 30 \
"https://www.gutenberg.org/cache/epub/$$id/pg$$id.txt" \
|| curl -s --max-time 30 \
"https://gutenberg.org/files/$$id/$$id-0.txt"; \
done > /tmp/amaia_stream.txt && \
$(PYTHON) train.py /tmp/amaia_stream.txt --sp-model $(SP_MODEL) --ckpt $(CKPT)
run: $(CKPT)
@read -p "Prompt: " p && $(PYTHON) generate.py "$$p" --sp-model $(SP_MODEL) --ckpt $(CKPT)
clean:
rm -f $(SP_PREFIX).model $(SP_PREFIX).vocab $(CKPT) /tmp/amaia_stream.txt
help:
@echo ""
@echo " make vocab train tokenizer from train.txt"
@echo " make train train on local train.txt"
@echo " make stream download Gutenberg books and train"
@echo " make run interactive prompt"
@echo " make clean remove model files"
@echo ""
|