Instructions to use muditbaid/llama3.1-8b-Instruct-qlora-hatexplain with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use muditbaid/llama3.1-8b-Instruct-qlora-hatexplain with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
Download scripts/qlora_inference.py from muditbaid/llama3.1-8b-Instruct-qlora-hatexplain: direct link, hf CLI and curl.
- Browser
- Download file 5.83 kB
-
https://huggingface.co/muditbaid/llama3.1-8b-Instruct-qlora-hatexplain/resolve/main/scripts/qlora_inference.py
- Command line
-
hf download hf://muditbaid/llama3.1-8b-Instruct-qlora-hatexplain/scripts/qlora_inference.py
-
curl -L -o qlora_inference.py https://huggingface.co/muditbaid/llama3.1-8b-Instruct-qlora-hatexplain/resolve/main/scripts/qlora_inference.py
5.83 kB
| #!/usr/bin/env python3 | |
| """ | |
| Run inference with a base model and a saved QLoRA adapter. | |
| Example: | |
| python qlora_inference.py \ | |
| --base-model meta-llama/Meta-Llama-3.1-8B-Instruct \ | |
| --adapter-path saves/llama31-8b/kaggle_cyberbullying/qlora \ | |
| --system-prompt "You are a helpful assistant." \ | |
| --user-input "How should we moderate this post?" | |
| """ | |
| import argparse | |
| from typing import Optional, Tuple | |
| import torch | |
| from peft import PeftModel | |
| from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser( | |
| description="Generate text with a base model and LoRA/QLoRA adapter." | |
| ) | |
| parser.add_argument( | |
| "--base-model", | |
| default="meta-llama/Meta-Llama-3.1-8B-Instruct", | |
| help="Base model name or local path (default: %(default)s).", | |
| ) | |
| parser.add_argument( | |
| "--adapter-path", | |
| required=True, | |
| help="Path to the trained adapter directory (containing adapter_model.safetensors).", | |
| ) | |
| parser.add_argument( | |
| "--system-prompt", | |
| default="You are a helpful assistant.", | |
| help="System prompt to steer the assistant (set empty string to skip).", | |
| ) | |
| parser.add_argument( | |
| "--user-input", | |
| required=True, | |
| help="User instruction or text the model should respond to.", | |
| ) | |
| parser.add_argument( | |
| "--max-new-tokens", | |
| type=int, | |
| default=512, | |
| help="Maximum number of new tokens to generate.", | |
| ) | |
| parser.add_argument( | |
| "--temperature", | |
| type=float, | |
| default=0.7, | |
| help="Sampling temperature; set <= 0 for deterministic decoding.", | |
| ) | |
| parser.add_argument( | |
| "--top-p", | |
| type=float, | |
| default=0.9, | |
| help="Top-p value for nucleus sampling (ignored if temperature <= 0).", | |
| ) | |
| parser.add_argument( | |
| "--no-quantization", | |
| action="store_true", | |
| help="Disable 4-bit loading; use this if you have a full-precision base model locally.", | |
| ) | |
| return parser.parse_args() | |
| def build_prompt( | |
| tokenizer: AutoTokenizer, system_prompt: str, user_input: str | |
| ) -> str: | |
| """Create a conversation prompt using chat templates when available.""" | |
| messages = [] | |
| if system_prompt: | |
| messages.append({"role": "system", "content": system_prompt}) | |
| messages.append({"role": "user", "content": user_input}) | |
| if hasattr(tokenizer, "apply_chat_template") and tokenizer.chat_template: | |
| return tokenizer.apply_chat_template( | |
| messages, tokenize=False, add_generation_prompt=True | |
| ) | |
| # Fallback for tokenizers without chat templates. | |
| if system_prompt: | |
| return f"{system_prompt}\n\nUser: {user_input}\nAssistant:" | |
| return f"User: {user_input}\nAssistant:" | |
| def load_model_and_tokenizer( | |
| base_model: str, adapter_path: str, use_quantization: bool | |
| ) -> Tuple[AutoModelForCausalLM, AutoTokenizer]: | |
| """Load the base model, adapter, and tokenizer with sensible defaults.""" | |
| tokenizer = AutoTokenizer.from_pretrained( | |
| adapter_path, use_fast=True, trust_remote_code=True | |
| ) | |
| if tokenizer.pad_token is None and tokenizer.eos_token is not None: | |
| tokenizer.pad_token = tokenizer.eos_token | |
| torch_dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32 | |
| quantization_config: Optional[BitsAndBytesConfig] = None | |
| if use_quantization: | |
| quantization_config = BitsAndBytesConfig( | |
| load_in_4bit=True, | |
| bnb_4bit_compute_dtype=torch.bfloat16, | |
| bnb_4bit_use_double_quant=True, | |
| bnb_4bit_quant_type="nf4", | |
| ) | |
| model = AutoModelForCausalLM.from_pretrained( | |
| base_model, | |
| device_map="auto", | |
| torch_dtype=torch_dtype, | |
| trust_remote_code=True, | |
| quantization_config=quantization_config, | |
| ) | |
| model = PeftModel.from_pretrained(model, adapter_path, is_trainable=False) | |
| model.eval() | |
| return model, tokenizer | |
| def generate_response( | |
| model: AutoModelForCausalLM, | |
| tokenizer: AutoTokenizer, | |
| prompt: str, | |
| max_new_tokens: int, | |
| temperature: float, | |
| top_p: float, | |
| ) -> str: | |
| """You are a helpful Assistant.""" | |
| inputs = tokenizer(prompt, return_tensors="pt") | |
| input_ids = inputs["input_ids"].to(model.device) | |
| attention_mask = inputs.get("attention_mask") | |
| if attention_mask is not None: | |
| attention_mask = attention_mask.to(model.device) | |
| generation_kwargs = { | |
| "input_ids": input_ids, | |
| "attention_mask": attention_mask, | |
| "max_new_tokens": max_new_tokens, | |
| "pad_token_id": tokenizer.pad_token_id, | |
| } | |
| if temperature > 0: | |
| generation_kwargs.update( | |
| { | |
| "do_sample": True, | |
| "temperature": temperature, | |
| "top_p": top_p, | |
| } | |
| ) | |
| else: | |
| generation_kwargs.update({"do_sample": False}) | |
| with torch.inference_mode(): | |
| outputs = model.generate(**generation_kwargs) | |
| generated_ids = outputs[0, input_ids.shape[-1] :] | |
| return tokenizer.decode(generated_ids, skip_special_tokens=True).strip() | |
| def main() -> None: | |
| args = parse_args() | |
| model, tokenizer = load_model_and_tokenizer( | |
| base_model=args.base_model, | |
| adapter_path=args.adapter_path, | |
| use_quantization=not args.no_quantization, | |
| ) | |
| prompt = build_prompt(tokenizer, args.system_prompt, args.user_input) | |
| response = generate_response( | |
| model=model, | |
| tokenizer=tokenizer, | |
| prompt=prompt, | |
| max_new_tokens=args.max_new_tokens, | |
| temperature=args.temperature, | |
| top_p=args.top_p, | |
| ) | |
| print("\n=== Model Response ===") | |
| print(response) | |
| if __name__ == "__main__": | |
| main() | |