Update handler.py

handler.py

@@ -6,6 +6,9 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
 from huggingface_inference_toolkit.logging import logger
 
 model_dir = 'Ephraimmm/PIDGIN_gemma-3'
+
+
+
 class EndpointHandler:
     """
     Custom handler for Hugging Face Inference Endpoints.
@@ -19,16 +22,34 @@ class EndpointHandler:
         logger.info(f"Initializing EndpointHandler with model_dir = {model_dir}")
 
         # load tokenizer and model from the model_dir or hub
+        base_model_name = "unsloth/gemma-3-4b-it-unsloth-bnb-4bit" 
+        base_model, tokenizer = FastLanguageModel.from_pretrained(
+        model_name=base_model_name,
+        max_seq_length=2048,
+        dtype=torch.float16,
+        load_in_4bit=True,
+        r = 8,           # Larger = higher accuracy, but might overfit
+        lora_alpha = 8,  # Recommended alpha == r at least
+        lora_dropout = 0,
+        )
+        
+        # -----------------------------
+        # 3️⃣ Load LoRA
+        # -----------------------------
+        lora_repo =  "Ephraimmm/PIDGIN_gemma-3" #"Ephraimmm/pigin-gemma-3-0.2" #"Ephraimmm/Pidgin_llamma_model" 
+        lora_model = PeftModel.from_pretrained(base_model, lora_repo, adapter_name="adapter_model")
+        FastLanguageModel.for_inference(lora_model)
+
         # we assume model files are present in model_dir
-        try:
-            self.tokenizer = AutoTokenizer.from_pretrained(model_dir, use_fast=False)
-            self.model = AutoModelForCausalLM.from_pretrained(model_dir)
-        except Exception as e:
-            # fallback: try loading from hub name
-            hf_model_id = "Ephraimmm/PIDGIN_gemma-3"
-            logger.warn(f"Loading model from hub id {hf_model_id} because fallback. Exception: {e}")
-            self.tokenizer = AutoTokenizer.from_pretrained(hf_model_id, use_fast=False)
-            self.model = AutoModelForCausalLM.from_pretrained(hf_model_id)
+        # try:
+        #     self.tokenizer = AutoTokenizer.from_pretrained(model_dir, use_fast=False)
+        #     self.model = AutoModelForCausalLM.from_pretrained(model_dir)
+        # except Exception as e:
+        #     # fallback: try loading from hub name
+        #     hf_model_id = "Ephraimmm/PIDGIN_gemma-3"
+        #     logger.warn(f"Loading model from hub id {hf_model_id} because fallback. Exception: {e}")
+        #     self.tokenizer = AutoTokenizer.from_pretrained(hf_model_id, use_fast=False)
+        #     self.model = AutoModelForCausalLM.from_pretrained(hf_model_id)
 
         # put the model on GPU if available
         self.device = "cuda" if torch.cuda.is_available() else "cpu"