test new nsfw model
Browse files
main.py
CHANGED
|
@@ -3,8 +3,15 @@ from fastapi import FastAPI, Form
|
|
| 3 |
from pydantic import BaseModel
|
| 4 |
|
| 5 |
#Model loading
|
| 6 |
-
llm = AutoModelForCausalLM.from_pretrained("TheBloke/Toppy-M-7B-GGUF",
|
| 7 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
model_type="mistral",
|
| 9 |
context_length=4096,
|
| 10 |
temperature=0.6,
|
|
|
|
| 3 |
from pydantic import BaseModel
|
| 4 |
|
| 5 |
#Model loading
|
| 6 |
+
# llm = AutoModelForCausalLM.from_pretrained("TheBloke/Toppy-M-7B-GGUF",
|
| 7 |
+
# model_file="toppy-m-7b.Q5_K_M.gguf",
|
| 8 |
+
# model_type="mistral",
|
| 9 |
+
# context_length=4096,
|
| 10 |
+
# temperature=0.6,
|
| 11 |
+
# gpu_layers=50)
|
| 12 |
+
|
| 13 |
+
llm = AutoModelForCausalLM.from_pretrained("TheDrummer/Moistral-11B-v3-GGUF",
|
| 14 |
+
model_file="Moistral-11B-v3-Q6_K.gguf",
|
| 15 |
model_type="mistral",
|
| 16 |
context_length=4096,
|
| 17 |
temperature=0.6,
|