..
Browse files- app.py +12 -9
- requirements.txt +1 -0
app.py
CHANGED
|
@@ -21,7 +21,6 @@ from commitlens import run_pipeline
|
|
| 21 |
# Model config
|
| 22 |
# ---------------------------------------------------------------------------
|
| 23 |
|
| 24 |
-
# Switched to the base model repository for native PyTorch compatibility
|
| 25 |
MODEL_REPO_ID = "JetBrains/Mellum2-12B-A2.5B-Thinking"
|
| 26 |
|
| 27 |
SUMMARY_SYSTEM_PROMPT = (
|
|
@@ -51,16 +50,20 @@ _tokenizer = None
|
|
| 51 |
def _get_llm():
|
| 52 |
global _model, _tokenizer
|
| 53 |
if _model is None:
|
| 54 |
-
#
|
| 55 |
quantization_config = BitsAndBytesConfig(
|
| 56 |
load_in_8bit=True,
|
| 57 |
)
|
| 58 |
|
| 59 |
_tokenizer = AutoTokenizer.from_pretrained(MODEL_REPO_ID)
|
|
|
|
|
|
|
| 60 |
_model = AutoModelForCausalLM.from_pretrained(
|
| 61 |
MODEL_REPO_ID,
|
| 62 |
quantization_config=quantization_config,
|
| 63 |
-
device_map="auto"
|
|
|
|
|
|
|
| 64 |
)
|
| 65 |
return _model, _tokenizer
|
| 66 |
|
|
@@ -87,10 +90,13 @@ def _generate_response(system_prompt: str, user_prompt: str, max_tokens: int) ->
|
|
| 87 |
|
| 88 |
inputs = tokenizer(formatted_prompt, return_tensors="pt").to("cuda")
|
| 89 |
|
|
|
|
| 90 |
outputs = model.generate(
|
| 91 |
**inputs,
|
| 92 |
max_new_tokens=max_tokens,
|
| 93 |
-
|
|
|
|
|
|
|
| 94 |
)
|
| 95 |
|
| 96 |
# Decode and return just the generated response
|
|
@@ -98,12 +104,10 @@ def _generate_response(system_prompt: str, user_prompt: str, max_tokens: int) ->
|
|
| 98 |
return response.strip()
|
| 99 |
|
| 100 |
|
| 101 |
-
# Removed @spaces.GPU from individual helper functions
|
| 102 |
def _summarize(prompt: str) -> str:
|
| 103 |
return _generate_response(SUMMARY_SYSTEM_PROMPT, prompt, max_tokens=1024)
|
| 104 |
|
| 105 |
|
| 106 |
-
# Removed @spaces.GPU from individual helper functions
|
| 107 |
def _final_md(combined: str) -> str:
|
| 108 |
return _generate_response(FINAL_SYSTEM_PROMPT, combined, max_tokens=2048)
|
| 109 |
|
|
@@ -112,9 +116,8 @@ def _final_md(combined: str) -> str:
|
|
| 112 |
# Pipeline
|
| 113 |
# ---------------------------------------------------------------------------
|
| 114 |
|
| 115 |
-
#
|
| 116 |
-
|
| 117 |
-
@spaces.GPU(duration=120)
|
| 118 |
def process_repo(repo_url: str, token: str, progress: gr.Progress = gr.Progress()):
|
| 119 |
try:
|
| 120 |
progress(0, desc="Running CommitLens pipeline...")
|
|
|
|
| 21 |
# Model config
|
| 22 |
# ---------------------------------------------------------------------------
|
| 23 |
|
|
|
|
| 24 |
MODEL_REPO_ID = "JetBrains/Mellum2-12B-A2.5B-Thinking"
|
| 25 |
|
| 26 |
SUMMARY_SYSTEM_PROMPT = (
|
|
|
|
| 50 |
def _get_llm():
|
| 51 |
global _model, _tokenizer
|
| 52 |
if _model is None:
|
| 53 |
+
# 8-bit quantization to fit in the 16GB System RAM bottleneck
|
| 54 |
quantization_config = BitsAndBytesConfig(
|
| 55 |
load_in_8bit=True,
|
| 56 |
)
|
| 57 |
|
| 58 |
_tokenizer = AutoTokenizer.from_pretrained(MODEL_REPO_ID)
|
| 59 |
+
|
| 60 |
+
# Load model with hardware optimizations (Flash Attention 2 & bfloat16)
|
| 61 |
_model = AutoModelForCausalLM.from_pretrained(
|
| 62 |
MODEL_REPO_ID,
|
| 63 |
quantization_config=quantization_config,
|
| 64 |
+
device_map="auto",
|
| 65 |
+
torch_dtype=torch.bfloat16,
|
| 66 |
+
attn_implementation="flash_attention_2"
|
| 67 |
)
|
| 68 |
return _model, _tokenizer
|
| 69 |
|
|
|
|
| 90 |
|
| 91 |
inputs = tokenizer(formatted_prompt, return_tensors="pt").to("cuda")
|
| 92 |
|
| 93 |
+
# Generate with speed optimizations (KV Cache on, Greedy Decoding)
|
| 94 |
outputs = model.generate(
|
| 95 |
**inputs,
|
| 96 |
max_new_tokens=max_tokens,
|
| 97 |
+
use_cache=True, # Speeds up sequential token generation
|
| 98 |
+
do_sample=False, # Greedy decoding for max speed (ignores temperature)
|
| 99 |
+
pad_token_id=tokenizer.eos_token_id # Silences the padding warning
|
| 100 |
)
|
| 101 |
|
| 102 |
# Decode and return just the generated response
|
|
|
|
| 104 |
return response.strip()
|
| 105 |
|
| 106 |
|
|
|
|
| 107 |
def _summarize(prompt: str) -> str:
|
| 108 |
return _generate_response(SUMMARY_SYSTEM_PROMPT, prompt, max_tokens=1024)
|
| 109 |
|
| 110 |
|
|
|
|
| 111 |
def _final_md(combined: str) -> str:
|
| 112 |
return _generate_response(FINAL_SYSTEM_PROMPT, combined, max_tokens=2048)
|
| 113 |
|
|
|
|
| 116 |
# Pipeline
|
| 117 |
# ---------------------------------------------------------------------------
|
| 118 |
|
| 119 |
+
# Single @spaces.GPU decorator with 300s timeout to survive the cold start
|
| 120 |
+
@spaces.GPU(duration=300)
|
|
|
|
| 121 |
def process_repo(repo_url: str, token: str, progress: gr.Progress = gr.Progress()):
|
| 122 |
try:
|
| 123 |
progress(0, desc="Running CommitLens pipeline...")
|
requirements.txt
CHANGED
|
@@ -2,4 +2,5 @@ transformers
|
|
| 2 |
accelerate
|
| 3 |
bitsandbytes
|
| 4 |
torch
|
|
|
|
| 5 |
gradio
|
|
|
|
| 2 |
accelerate
|
| 3 |
bitsandbytes
|
| 4 |
torch
|
| 5 |
+
flash-attn
|
| 6 |
gradio
|