pkheria commited on
Commit
4b6c43f
·
1 Parent(s): ef80c0c

fixed the issue

Browse files
Files changed (3) hide show
  1. .DS_Store +0 -0
  2. app.py +9 -27
  3. requirements.txt +0 -1
.DS_Store CHANGED
Binary files a/.DS_Store and b/.DS_Store differ
 
app.py CHANGED
@@ -1,17 +1,10 @@
1
  """
2
- CommitLens — Gradio UI
3
- =======================
4
- - Asks for a GitHub repo URL (and optional PAT)
5
- - Runs commitlens.run_pipeline() to get per-file prompts
6
- - Feeds each prompt to Mellum 2 for a per-file summary
7
- - Combines all summaries and asks the model for a final .md report
8
- - Displays everything in the browser
9
  """
10
 
11
  from __future__ import annotations
12
 
13
  import gradio as gr
14
- import spaces
15
  import torch
16
  from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
17
 
@@ -46,35 +39,32 @@ FINAL_SYSTEM_PROMPT = (
46
  _model = None
47
  _tokenizer = None
48
 
49
-
50
  def _get_llm():
51
  global _model, _tokenizer
52
  if _model is None:
53
- # 8-bit quantization to fit in the 16GB System RAM bottleneck
54
  quantization_config = BitsAndBytesConfig(
55
  load_in_8bit=True,
56
  )
57
 
58
  _tokenizer = AutoTokenizer.from_pretrained(MODEL_REPO_ID)
59
 
60
- # Load model with hardware optimizations (Flash Attention 2 & bfloat16)
 
61
  _model = AutoModelForCausalLM.from_pretrained(
62
  MODEL_REPO_ID,
63
  quantization_config=quantization_config,
64
  device_map="auto",
65
- torch_dtype=torch.bfloat16,
66
- attn_implementation="flash_attention_2"
67
  )
68
  return _model, _tokenizer
69
 
70
-
71
  def _extract_filename(prompt: str) -> str:
72
  for line in prompt.splitlines():
73
  if line.startswith("Filename :"):
74
  return line.split(":", 1)[1].strip()
75
  return "unknown"
76
 
77
-
78
  def _generate_response(system_prompt: str, user_prompt: str, max_tokens: int) -> str:
79
  model, tokenizer = _get_llm()
80
 
@@ -83,41 +73,34 @@ def _generate_response(system_prompt: str, user_prompt: str, max_tokens: int) ->
83
  {"role": "user", "content": user_prompt},
84
  ]
85
 
86
- # Format the prompt using the model's chat template
87
  formatted_prompt = tokenizer.apply_chat_template(
88
  messages, tokenize=False, add_generation_prompt=True
89
  )
90
 
91
  inputs = tokenizer(formatted_prompt, return_tensors="pt").to("cuda")
92
 
93
- # Generate with speed optimizations (KV Cache on, Greedy Decoding)
94
  outputs = model.generate(
95
  **inputs,
96
  max_new_tokens=max_tokens,
97
- use_cache=True, # Speeds up sequential token generation
98
- do_sample=False, # Greedy decoding for max speed (ignores temperature)
99
- pad_token_id=tokenizer.eos_token_id # Silences the padding warning
100
  )
101
 
102
- # Decode and return just the generated response
103
  response = tokenizer.decode(outputs[0][inputs.input_ids.shape[-1]:], skip_special_tokens=True)
104
  return response.strip()
105
 
106
-
107
  def _summarize(prompt: str) -> str:
108
  return _generate_response(SUMMARY_SYSTEM_PROMPT, prompt, max_tokens=1024)
109
 
110
-
111
  def _final_md(combined: str) -> str:
112
  return _generate_response(FINAL_SYSTEM_PROMPT, combined, max_tokens=2048)
113
 
114
-
115
  # ---------------------------------------------------------------------------
116
  # Pipeline
117
  # ---------------------------------------------------------------------------
118
 
119
- # Single @spaces.GPU decorator with 300s timeout to survive the cold start
120
- @spaces.GPU(duration=300)
121
  def process_repo(repo_url: str, token: str, progress: gr.Progress = gr.Progress()):
122
  try:
123
  progress(0, desc="Running CommitLens pipeline...")
@@ -146,7 +129,6 @@ def process_repo(repo_url: str, token: str, progress: gr.Progress = gr.Progress(
146
  except Exception as e:
147
  raise gr.Error(str(e))
148
 
149
-
150
  # ---------------------------------------------------------------------------
151
  # Gradio app
152
  # ---------------------------------------------------------------------------
 
1
  """
2
+ CommitLens — Gradio UI (Dedicated T4 Medium Version)
 
 
 
 
 
 
3
  """
4
 
5
  from __future__ import annotations
6
 
7
  import gradio as gr
 
8
  import torch
9
  from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
10
 
 
39
  _model = None
40
  _tokenizer = None
41
 
 
42
  def _get_llm():
43
  global _model, _tokenizer
44
  if _model is None:
45
+ # 8-bit quantization is mandatory to fit inside the T4's 16GB VRAM
46
  quantization_config = BitsAndBytesConfig(
47
  load_in_8bit=True,
48
  )
49
 
50
  _tokenizer = AutoTokenizer.from_pretrained(MODEL_REPO_ID)
51
 
52
+ # CHANGED: torch_dtype set to float16 (T4 does not support bfloat16)
53
+ # CHANGED: Removed Flash Attention (T4 relies on native SDPA instead)
54
  _model = AutoModelForCausalLM.from_pretrained(
55
  MODEL_REPO_ID,
56
  quantization_config=quantization_config,
57
  device_map="auto",
58
+ torch_dtype=torch.float16,
 
59
  )
60
  return _model, _tokenizer
61
 
 
62
  def _extract_filename(prompt: str) -> str:
63
  for line in prompt.splitlines():
64
  if line.startswith("Filename :"):
65
  return line.split(":", 1)[1].strip()
66
  return "unknown"
67
 
 
68
  def _generate_response(system_prompt: str, user_prompt: str, max_tokens: int) -> str:
69
  model, tokenizer = _get_llm()
70
 
 
73
  {"role": "user", "content": user_prompt},
74
  ]
75
 
 
76
  formatted_prompt = tokenizer.apply_chat_template(
77
  messages, tokenize=False, add_generation_prompt=True
78
  )
79
 
80
  inputs = tokenizer(formatted_prompt, return_tensors="pt").to("cuda")
81
 
 
82
  outputs = model.generate(
83
  **inputs,
84
  max_new_tokens=max_tokens,
85
+ use_cache=True,
86
+ do_sample=False,
87
+ pad_token_id=tokenizer.eos_token_id
88
  )
89
 
 
90
  response = tokenizer.decode(outputs[0][inputs.input_ids.shape[-1]:], skip_special_tokens=True)
91
  return response.strip()
92
 
 
93
  def _summarize(prompt: str) -> str:
94
  return _generate_response(SUMMARY_SYSTEM_PROMPT, prompt, max_tokens=1024)
95
 
 
96
  def _final_md(combined: str) -> str:
97
  return _generate_response(FINAL_SYSTEM_PROMPT, combined, max_tokens=2048)
98
 
 
99
  # ---------------------------------------------------------------------------
100
  # Pipeline
101
  # ---------------------------------------------------------------------------
102
 
103
+ # CHANGED: Removed @spaces.GPU entirely. The T4 is dedicated and always active.
 
104
  def process_repo(repo_url: str, token: str, progress: gr.Progress = gr.Progress()):
105
  try:
106
  progress(0, desc="Running CommitLens pipeline...")
 
129
  except Exception as e:
130
  raise gr.Error(str(e))
131
 
 
132
  # ---------------------------------------------------------------------------
133
  # Gradio app
134
  # ---------------------------------------------------------------------------
requirements.txt CHANGED
@@ -2,5 +2,4 @@ transformers
2
  accelerate
3
  bitsandbytes
4
  torch
5
- flash-attn
6
  gradio
 
2
  accelerate
3
  bitsandbytes
4
  torch
 
5
  gradio