Spaces:
Sleeping
Sleeping
Download app.py from afnotes/YeI-Tweet-Generator: direct link, hf CLI and curl.
- Browser
- Download file 5.72 kB
-
https://huggingface.co/spaces/afnotes/YeI-Tweet-Generator/resolve/main/app.py
- Command line
-
hf download hf://spaces/afnotes/YeI-Tweet-Generator/app.py
-
curl -L -o app.py https://huggingface.co/spaces/afnotes/YeI-Tweet-Generator/resolve/main/app.py
5.72 kB
| import gradio as gr | |
| import torch | |
| import spaces | |
| from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig | |
| from peft import PeftModel | |
| import os | |
| # --- Configuration --- | |
| BASE_MODEL_ID = "Qwen/Qwen3-4B" | |
| ADAPTER_MODEL_ID = "afnotes/Qwen3-4B-YeI-LoRA" | |
| # --- Model Loading (Optimized for ZeroGPU) --- | |
| def load_model(): | |
| print(f"Loading tokenizer from: {ADAPTER_MODEL_ID}") | |
| # Load tokenizer without triggering CUDA | |
| tokenizer = AutoTokenizer.from_pretrained(ADAPTER_MODEL_ID, use_fast=False) | |
| if tokenizer.eos_token is None: | |
| tokenizer.eos_token = "<|endoftext|>" | |
| print(f"Setting tokenizer.eos_token to: {tokenizer.eos_token}") | |
| if tokenizer.pad_token is None: | |
| tokenizer.pad_token = tokenizer.eos_token | |
| print(f"Setting tokenizer.pad_token to: {tokenizer.pad_token}") | |
| print(f"Loading base model: {BASE_MODEL_ID}") | |
| # Check for CUDA availability inside the @spaces.GPU context | |
| device_to_load = "cuda" if torch.cuda.is_available() else "cpu" | |
| print(f"Using device: {device_to_load}") | |
| # Quantization configuration is no longer needed for 16-bit | |
| bnb_config = None | |
| try: | |
| # Load base model in float16 if on CUDA, otherwise float32 on CPU | |
| base_model = AutoModelForCausalLM.from_pretrained( | |
| BASE_MODEL_ID, | |
| # Use torch.float16 for CUDA, torch.float32 for CPU | |
| torch_dtype=torch.float16 if device_to_load == "cuda" else torch.float32, | |
| device_map="auto" if device_to_load == "cuda" else "cpu", | |
| trust_remote_code=True | |
| ) | |
| print(f"Loading LoRA adapter from {ADAPTER_MODEL_ID} on top of base model...") | |
| model = PeftModel.from_pretrained(base_model, ADAPTER_MODEL_ID) | |
| model = model.merge_and_unload() | |
| print(f"Successfully loaded LoRA adapter from {ADAPTER_MODEL_ID} on top of {BASE_MODEL_ID}") | |
| except Exception as e: | |
| print(f"Error loading LoRA adapter with 16-bit/GPU setup ({e}). Attempting to load without specific device_map or in full precision on CPU...") | |
| try: | |
| model = AutoModelForCausalLM.from_pretrained( | |
| ADAPTER_MODEL_ID, | |
| torch_dtype=torch.float32, | |
| device_map="cpu", | |
| trust_remote_code=True | |
| ) | |
| print(f"Successfully loaded full merged model directly from {ADAPTER_MODEL_ID} on CPU.") | |
| except Exception as e_full: | |
| print(f"Failed to load model in fallback: {e_full}") | |
| raise RuntimeError("Model loading failed completely.") | |
| model.eval() | |
| if device_to_load == "cuda": | |
| model = model.to("cuda") | |
| print(f"Model loaded and moved to CUDA device: {model.device}") | |
| print(f"CUDA memory allocated: {torch.cuda.memory_allocated() / (1024**3):.2f} GB") | |
| print(f"CUDA memory cached: {torch.cuda.memory_reserved() / (1024**3):.2f} GB") | |
| else: | |
| print(f"Model loaded and running on CPU device: {model.device}") | |
| print("Model loaded successfully and set to evaluation mode.") | |
| return tokenizer, model | |
| # --- Generation Function for Gradio --- | |
| def generate_kanye_tweet(prompt_text, max_new_tokens, temperature, top_p, repetition_penalty): | |
| print(f"--- Generating for prompt ---\n'{prompt_text}'") | |
| # Load model and tokenizer within the @spaces.GPU context | |
| tokenizer, model = load_model() | |
| inputs = tokenizer(prompt_text, return_tensors="pt").to(model.device) | |
| input_token_len = inputs.input_ids.shape[1] | |
| with torch.no_grad(): | |
| outputs = model.generate( | |
| **inputs, | |
| max_new_tokens=max_new_tokens, | |
| temperature=temperature, | |
| top_p=top_p, | |
| do_sample=True, | |
| repetition_penalty=repetition_penalty, | |
| pad_token_id=tokenizer.eos_token_id, | |
| eos_token_id=tokenizer.eos_token_id, | |
| ) | |
| full_generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True) | |
| if full_generated_text.startswith(prompt_text): | |
| completion_text = full_generated_text[len(prompt_text):].strip() | |
| else: | |
| print("Warning: Decoded text did not start exactly with the prompt. Showing full output.") | |
| completion_text = full_generated_text | |
| print(f"\n--- Generated Completion (Ye-style) ---") | |
| print(completion_text) | |
| print("------------------------------------------") | |
| # Clear model from memory to comply with ZeroGPU | |
| del model | |
| del tokenizer | |
| if torch.cuda.is_available(): | |
| torch.cuda.empty_cache() | |
| return completion_text | |
| # --- Gradio Interface --- | |
| iface = gr.Interface( | |
| fn=generate_kanye_tweet, | |
| inputs=[ | |
| gr.Textbox(lines=2, label="Enter your prompt for Ye-style tweet:", placeholder="My next album will be called..."), | |
| gr.Slider(minimum=10, maximum=250, value=100, step=5, label="Max New Tokens"), | |
| gr.Slider(minimum=0.1, maximum=1.0, value=0.9, step=0.05, label="Temperature"), | |
| gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top P"), | |
| gr.Slider(minimum=1.0, maximum=2.0, value=1.0, step=0.05, label="Repetition Penalty") | |
| ], | |
| outputs=gr.Textbox(label="Generated Ye Tweet"), | |
| title="🎙️ Ye-I: Ye Tweet Generator (4B Model) 🎤", | |
| description="Generate Ye-style tweet snippets using a fine-tuned Qwen3-4B model. Optimized for ZeroGPU.", | |
| examples=[ | |
| ["I HATE", 50, 0.9, 0.95, 1.0], | |
| ["My next album will be called", 50, 0.9, 0.95, 1.0], | |
| ["Fashion is dead without", 50, 0.9, 0.95, 1.0], | |
| ], | |
| allow_flagging="never" | |
| ) | |
| if __name__ == "__main__": | |
| # Disable SSR mode to avoid potential main process issues | |
| iface.launch(ssr_mode=False) | |