File size: 1,666 Bytes
170da0e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
# model_loader.py
from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer
import logging

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def load_model_pipeline(model_name: str = "distilgpt2", device: int = -1, max_new_tokens: int = 128, 
                        do_sample: bool = True, temperature: float = 0.7, top_k: int = 50, top_p: float = 0.95):
    """
    Load and return a text-generation pipeline.
    - model_name: HF model id
    - device: -1 for CPU, >=0 for CUDA device
    Returns a callable pipeline that accepts a prompt string and returns generated text.
    """
    logger.info(f"Loading model '{model_name}' (device={device})...")
    # Use Auto classes to ensure tokenizer/model compatibility
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForCausalLM.from_pretrained(model_name)
    gen_pipeline = pipeline(
        "text-generation",
        model=model,
        tokenizer=tokenizer,
        device=device,
    )

    def generate(prompt: str, max_new_tokens_local: int = None, **kwargs):
        mnt = max_new_tokens_local if max_new_tokens_local is not None else max_new_tokens
        outputs = gen_pipeline(
            prompt,
            max_new_tokens=mnt,
            do_sample=do_sample,
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            pad_token_id=tokenizer.eos_token_id,  # avoid warnings / truncation issues
            **kwargs
        )
        # pipeline returns a list of dicts with 'generated_text'
        return outputs[0]["generated_text"]

    logger.info("Model loaded.")
    return generate