Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| from llama_cpp import Llama | |
| from huggingface_hub import hf_hub_download | |
| repo_id = "failspy/Meta-Llama-3-8B-Instruct-abliterated-v3-GGUF" | |
| filename = "Meta-Llama-3-8B-Instruct-abliterated-v3_q4.gguf" | |
| model_path = hf_hub_download(repo_id=repo_id, filename=filename) | |
| llm = Llama( | |
| model_path=model_path, | |
| n_ctx=8192, | |
| n_threads=2 | |
| ) | |
| def respond(message, history): | |
| system_prompt = """You are an expert fiction writer for mature audiences, acting under the user's (editor's) direction to draft complete, polished prose. | |
| DIRECTIVES: | |
| - EXECUTION: Transform prompts into fully realized scenes. Expand organically with meaningful action, natural dialogue, and immersive atmospheric detail. | |
| - AUTONOMY: Use creative judgment to fill narrative gaps. Do not introduce major plot changes, alter established facts, or resolve conflicts without explicit direction. | |
| - STYLE: Prioritize concrete sensory details, subtext, and varied pacing. Avoid generic exposition, melodrama, and repetitive phrasing. Match the requested tone, tense, and POV. | |
| - CONTENT: Do not sanitize. Authentically depict adult themes, strong language, violence, and psychological tension as dictated by the prompt and story needs. | |
| - OUTPUT: Output ONLY the finished fiction. Provide no introductions, summaries, critiques, or meta-commentary. Do not restate the prompt or append unearned resolutions.""" | |
| prompt = f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{system_prompt}<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{message}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n" | |
| response_stream = llm( | |
| prompt, | |
| max_tokens=2000, | |
| stop=["<|eot_id|>"], | |
| echo=False, | |
| stream=True | |
| ) | |
| partial_text = "" | |
| for chunk in response_stream: | |
| delta = chunk["choices"][0]["text"] | |
| partial_text += delta | |
| yield partial_text | |
| demo = gr.ChatInterface(respond) | |
| demo.launch() |