pkheria commited on
Commit
ef80c0c
·
1 Parent(s): 2d5d362
Files changed (2) hide show
  1. app.py +12 -9
  2. requirements.txt +1 -0
app.py CHANGED
@@ -21,7 +21,6 @@ from commitlens import run_pipeline
21
  # Model config
22
  # ---------------------------------------------------------------------------
23
 
24
- # Switched to the base model repository for native PyTorch compatibility
25
  MODEL_REPO_ID = "JetBrains/Mellum2-12B-A2.5B-Thinking"
26
 
27
  SUMMARY_SYSTEM_PROMPT = (
@@ -51,16 +50,20 @@ _tokenizer = None
51
  def _get_llm():
52
  global _model, _tokenizer
53
  if _model is None:
54
- # Configured for 8-bit quantization via bitsandbytes
55
  quantization_config = BitsAndBytesConfig(
56
  load_in_8bit=True,
57
  )
58
 
59
  _tokenizer = AutoTokenizer.from_pretrained(MODEL_REPO_ID)
 
 
60
  _model = AutoModelForCausalLM.from_pretrained(
61
  MODEL_REPO_ID,
62
  quantization_config=quantization_config,
63
- device_map="auto" # Automatically handles GPU placement
 
 
64
  )
65
  return _model, _tokenizer
66
 
@@ -87,10 +90,13 @@ def _generate_response(system_prompt: str, user_prompt: str, max_tokens: int) ->
87
 
88
  inputs = tokenizer(formatted_prompt, return_tensors="pt").to("cuda")
89
 
 
90
  outputs = model.generate(
91
  **inputs,
92
  max_new_tokens=max_tokens,
93
- temperature=0.3
 
 
94
  )
95
 
96
  # Decode and return just the generated response
@@ -98,12 +104,10 @@ def _generate_response(system_prompt: str, user_prompt: str, max_tokens: int) ->
98
  return response.strip()
99
 
100
 
101
- # Removed @spaces.GPU from individual helper functions
102
  def _summarize(prompt: str) -> str:
103
  return _generate_response(SUMMARY_SYSTEM_PROMPT, prompt, max_tokens=1024)
104
 
105
 
106
- # Removed @spaces.GPU from individual helper functions
107
  def _final_md(combined: str) -> str:
108
  return _generate_response(FINAL_SYSTEM_PROMPT, combined, max_tokens=2048)
109
 
@@ -112,9 +116,8 @@ def _final_md(combined: str) -> str:
112
  # Pipeline
113
  # ---------------------------------------------------------------------------
114
 
115
- # Placed a single @spaces.GPU decorator here to lock the GPU for the entire pipeline.
116
- # duration=120 ensures the GPU isn't preempted too early for large commits.
117
- @spaces.GPU(duration=120)
118
  def process_repo(repo_url: str, token: str, progress: gr.Progress = gr.Progress()):
119
  try:
120
  progress(0, desc="Running CommitLens pipeline...")
 
21
  # Model config
22
  # ---------------------------------------------------------------------------
23
 
 
24
  MODEL_REPO_ID = "JetBrains/Mellum2-12B-A2.5B-Thinking"
25
 
26
  SUMMARY_SYSTEM_PROMPT = (
 
50
  def _get_llm():
51
  global _model, _tokenizer
52
  if _model is None:
53
+ # 8-bit quantization to fit in the 16GB System RAM bottleneck
54
  quantization_config = BitsAndBytesConfig(
55
  load_in_8bit=True,
56
  )
57
 
58
  _tokenizer = AutoTokenizer.from_pretrained(MODEL_REPO_ID)
59
+
60
+ # Load model with hardware optimizations (Flash Attention 2 & bfloat16)
61
  _model = AutoModelForCausalLM.from_pretrained(
62
  MODEL_REPO_ID,
63
  quantization_config=quantization_config,
64
+ device_map="auto",
65
+ torch_dtype=torch.bfloat16,
66
+ attn_implementation="flash_attention_2"
67
  )
68
  return _model, _tokenizer
69
 
 
90
 
91
  inputs = tokenizer(formatted_prompt, return_tensors="pt").to("cuda")
92
 
93
+ # Generate with speed optimizations (KV Cache on, Greedy Decoding)
94
  outputs = model.generate(
95
  **inputs,
96
  max_new_tokens=max_tokens,
97
+ use_cache=True, # Speeds up sequential token generation
98
+ do_sample=False, # Greedy decoding for max speed (ignores temperature)
99
+ pad_token_id=tokenizer.eos_token_id # Silences the padding warning
100
  )
101
 
102
  # Decode and return just the generated response
 
104
  return response.strip()
105
 
106
 
 
107
  def _summarize(prompt: str) -> str:
108
  return _generate_response(SUMMARY_SYSTEM_PROMPT, prompt, max_tokens=1024)
109
 
110
 
 
111
  def _final_md(combined: str) -> str:
112
  return _generate_response(FINAL_SYSTEM_PROMPT, combined, max_tokens=2048)
113
 
 
116
  # Pipeline
117
  # ---------------------------------------------------------------------------
118
 
119
+ # Single @spaces.GPU decorator with 300s timeout to survive the cold start
120
+ @spaces.GPU(duration=300)
 
121
  def process_repo(repo_url: str, token: str, progress: gr.Progress = gr.Progress()):
122
  try:
123
  progress(0, desc="Running CommitLens pipeline...")
requirements.txt CHANGED
@@ -2,4 +2,5 @@ transformers
2
  accelerate
3
  bitsandbytes
4
  torch
 
5
  gradio
 
2
  accelerate
3
  bitsandbytes
4
  torch
5
+ flash-attn
6
  gradio