Download app.py from Idhammz/Spaces: direct link, hf CLI and curl.
- Browser
- Download file 1.85 kB
-
https://huggingface.co/spaces/Idhammz/Spaces/resolve/main/app.py
- Command line
-
hf download hf://spaces/Idhammz/Spaces/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Idhammz/Spaces/resolve/main/app.py
1.85 kB
| from fastapi import FastAPI, HTTPException | |
| from pydantic import BaseModel | |
| import transformers | |
| import torch | |
| import os | |
| from dotenv import load_dotenv | |
| # Load environment variables from .env file | |
| load_dotenv() | |
| API_TOKEN = os.getenv("HF_API_TOKEN") | |
| # Initialize the text-generation pipeline with authentication | |
| model_id = "mistralai/Mistral-7B-Instruct-v0.3" | |
| pipeline = transformers.pipeline( | |
| "text-generation", | |
| model=model_id, | |
| model_kwargs={"torch_dtype": torch.bfloat16}, | |
| device_map="auto", | |
| use_auth_token=API_TOKEN | |
| ) | |
| # Define the FastAPI app | |
| app = FastAPI() | |
| # Define the request model | |
| class Message(BaseModel): | |
| role: str | |
| content: str | |
| class ChatMessages(BaseModel): | |
| messages: list[Message] | |
| def greet_json(): | |
| return {"Hello": "World!"} | |
| async def chat(request: ChatMessages): | |
| # Prepare the input messages for the pipeline | |
| messages = [{"role": msg.role, "content": msg.content} for msg in request.messages] | |
| # Set the terminators | |
| terminators = [ | |
| pipeline.tokenizer.eos_token_id, | |
| pipeline.tokenizer.convert_tokens_to_ids("") | |
| ] | |
| # Generate response using the model | |
| try: | |
| # Mistral-7B-Instruct-v0.3 might require a slightly different format | |
| # Adjust this if needed according to the model's documentation | |
| outputs = pipeline( | |
| messages, | |
| max_new_tokens=256, | |
| eos_token_id=terminators, | |
| do_sample=True, | |
| temperature=0.6, | |
| top_p=0.9, | |
| ) | |
| response_text = outputs[0]["generated_text"] | |
| return {"response": response_text} | |
| except Exception as e: | |
| raise HTTPException(status_code=500, detail=str(e)) | |
| # Start the FastAPI app | |
| if __name__ == "__main__": | |
| import uvicorn | |
| uvicorn.run(app, host="0.0.0.0", port=8000) | |