File size: 1,349 Bytes
ca0bb3f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
PYTHON     = python3
SP_PREFIX  = amaia
VOCAB_SIZE = 32768
CKPT       = amaia.pt
SP_MODEL   = amaia.model

GUTENBERG_IDS = 84 1342 11 98 2701 1661 345 844 1952 174 5200 76 1400 2600 4300

.PHONY: all vocab train stream run clean help

all: vocab

vocab: train.txt
	$(PYTHON) vocab.py train.txt --prefix $(SP_PREFIX) --vocab-size $(VOCAB_SIZE)

train: $(SP_MODEL)
	$(PYTHON) train.py train.txt --sp-model $(SP_MODEL) --ckpt $(CKPT)

stream: $(SP_MODEL)
	@echo "Streaming $(words $(GUTENBERG_IDS)) books from Project Gutenberg..."
	@for id in $(GUTENBERG_IDS); do \
		echo "  book $$id" >&2; \
		curl -s --max-time 30 \
			"https://www.gutenberg.org/cache/epub/$$id/pg$$id.txt" \
		|| curl -s --max-time 30 \
			"https://gutenberg.org/files/$$id/$$id-0.txt"; \
	done > /tmp/amaia_stream.txt && \
	$(PYTHON) train.py /tmp/amaia_stream.txt --sp-model $(SP_MODEL) --ckpt $(CKPT)

run: $(CKPT)
	@read -p "Prompt: " p && $(PYTHON) generate.py "$$p" --sp-model $(SP_MODEL) --ckpt $(CKPT)

clean:
	rm -f $(SP_PREFIX).model $(SP_PREFIX).vocab $(CKPT) /tmp/amaia_stream.txt

help:
	@echo ""
	@echo "  make vocab     train tokenizer from train.txt"
	@echo "  make train     train on local train.txt"
	@echo "  make stream    download Gutenberg books and train"
	@echo "  make run       interactive prompt"
	@echo "  make clean     remove model files"
	@echo ""