Rename to Smaug-Toxicity-Perspective; add onnx/model_quantized.onnx and TEI serving notes

7c413b7 verified 1 day ago

907 Bytes

# CPU server image: 8-bit ONNX (block-wise; about 0.001 from float32 on average, on any CPU), no PyTorch. Build it from a download of this repository:
#   hf download abacusai/Smaug-Toxicity-Perspective --local-dir smaug-toxicity-perspective --include "*.json" --include server.py \
#     --include Dockerfile --include .dockerignore --include onnx/model_int8_block32.onnx
#   docker build -t smaug-toxicity-perspective smaug-toxicity-perspective && docker run -p 8080:8080 smaug-toxicity-perspective
FROM python:3.12-slim
RUN pip install --no-cache-dir fastapi==0.138.1 uvicorn==0.49.0 onnxruntime==1.31.1 tokenizers==0.22.2 "numpy<3"
WORKDIR /app
COPY server.py config.json tokenizer.json tokenizer_config.json ./
COPY onnx/model_int8_block32.onnx onnx/
ENV MODEL_DIR=/app BACKEND=onnx MODEL_NAME=smaug-toxicity-perspective
EXPOSE 8080
CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8080"]

	# CPU server image: 8-bit ONNX (block-wise; about 0.001 from float32 on average, on any CPU), no PyTorch. Build it from a download of this repository:
	# hf download abacusai/Smaug-Toxicity-Perspective --local-dir smaug-toxicity-perspective --include "*.json" --include server.py \
	# --include Dockerfile --include .dockerignore --include onnx/model_int8_block32.onnx
	# docker build -t smaug-toxicity-perspective smaug-toxicity-perspective && docker run -p 8080:8080 smaug-toxicity-perspective
	FROM python:3.12-slim
	RUN pip install --no-cache-dir fastapi==0.138.1 uvicorn==0.49.0 onnxruntime==1.31.1 tokenizers==0.22.2 "numpy<3"
	WORKDIR /app
	COPY server.py config.json tokenizer.json tokenizer_config.json ./
	COPY onnx/model_int8_block32.onnx onnx/
	ENV MODEL_DIR=/app BACKEND=onnx MODEL_NAME=smaug-toxicity-perspective
	EXPOSE 8080
	CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8080"]