Spaces:
Runtime error
Runtime error
Download app.py from Subodh358/CoderSayu-8B-4bit: direct link, hf CLI and curl.
- Browser
- Download file 744 Bytes
-
https://huggingface.co/spaces/Subodh358/CoderSayu-8B-4bit/resolve/main/app.py
- Command line
-
hf download hf://spaces/Subodh358/CoderSayu-8B-4bit/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Subodh358/CoderSayu-8B-4bit/resolve/main/app.py
744 Bytes
| from fastapi import FastAPI | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| import torch | |
| import os | |
| app = FastAPI() | |
| # Set the cache directory environment variable | |
| os.environ["TRANSFORMERS_CACHE"] = "/tmp/cache" | |
| # Load the model and tokenizer | |
| model_name = "unsloth/llama-3-8b-bnb-4bit" | |
| tokenizer = AutoTokenizer.from_pretrained(model_name) | |
| model = AutoModelForCausalLM.from_pretrained(model_name) | |
| def read_root(): | |
| return {"Hello": "World"} | |
| def generate(prompt: str): | |
| inputs = tokenizer(prompt, return_tensors="pt") | |
| outputs = model.generate(inputs["input_ids"], max_length=50) | |
| text = tokenizer.decode(outputs[0], skip_special_tokens=True) | |
| return {"generated_text": text} | |