PYTHON = python3 SP_PREFIX = amaia VOCAB_SIZE = 32768 CKPT = amaia.pt SP_MODEL = amaia.model GUTENBERG_IDS = 84 1342 11 98 2701 1661 345 844 1952 174 5200 76 1400 2600 4300 .PHONY: all vocab train stream run clean help all: vocab vocab: train.txt $(PYTHON) vocab.py train.txt --prefix $(SP_PREFIX) --vocab-size $(VOCAB_SIZE) train: $(SP_MODEL) $(PYTHON) train.py train.txt --sp-model $(SP_MODEL) --ckpt $(CKPT) stream: $(SP_MODEL) @echo "Streaming $(words $(GUTENBERG_IDS)) books from Project Gutenberg..." @for id in $(GUTENBERG_IDS); do \ echo " book $$id" >&2; \ curl -s --max-time 30 \ "https://www.gutenberg.org/cache/epub/$$id/pg$$id.txt" \ || curl -s --max-time 30 \ "https://gutenberg.org/files/$$id/$$id-0.txt"; \ done > /tmp/amaia_stream.txt && \ $(PYTHON) train.py /tmp/amaia_stream.txt --sp-model $(SP_MODEL) --ckpt $(CKPT) run: $(CKPT) @read -p "Prompt: " p && $(PYTHON) generate.py "$$p" --sp-model $(SP_MODEL) --ckpt $(CKPT) clean: rm -f $(SP_PREFIX).model $(SP_PREFIX).vocab $(CKPT) /tmp/amaia_stream.txt help: @echo "" @echo " make vocab train tokenizer from train.txt" @echo " make train train on local train.txt" @echo " make stream download Gutenberg books and train" @echo " make run interactive prompt" @echo " make clean remove model files" @echo ""