Spaces:
Paused
Paused
Darksting4906 commited on
Commit ·
50087b1
1
Parent(s): 38702fe
Initial commit
Browse files- Dockerfile +12 -0
- app.py +154 -0
- requirements.txt +8 -0
Dockerfile
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM pytorch/pytorch:2.1.0-cuda12.1-cudnn8-runtime
|
| 2 |
+
|
| 3 |
+
WORKDIR /app
|
| 4 |
+
|
| 5 |
+
COPY requirements.txt .
|
| 6 |
+
|
| 7 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 8 |
+
|
| 9 |
+
COPY app.py .
|
| 10 |
+
|
| 11 |
+
EXPOSE 7860
|
| 12 |
+
CMD ["python", "app.py"]
|
app.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from flask import Flask, request, jsonify
|
| 2 |
+
from transformers import AutoTokenizer, AutoModelForCausalLM,BitsAndBytesConfig
|
| 3 |
+
from peft import PeftModel
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
|
| 7 |
+
BASE_MODEL = "Qwen/Qwen2.5-Math-1.5B-Instruct"
|
| 8 |
+
ADAPTER_ID = "DarkSting/teachers_guide_and_matching_score"
|
| 9 |
+
NUM_SECTIONS = 8
|
| 10 |
+
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
| 11 |
+
|
| 12 |
+
def build_prompt(sample):
|
| 13 |
+
if sample["task"] == "task_a":
|
| 14 |
+
return f"""### TASK: ANSWER EVALUATION
|
| 15 |
+
Topic: {sample['topic']}
|
| 16 |
+
Question: {sample['question']}
|
| 17 |
+
Student Answer:
|
| 18 |
+
{sample['answer']}
|
| 19 |
+
|
| 20 |
+
Return:
|
| 21 |
+
"""
|
| 22 |
+
else:
|
| 23 |
+
time_lines = "\n".join(
|
| 24 |
+
f"[{k.upper()}_TIME={v}]" for k, v in sample["time_spent"].items()
|
| 25 |
+
)
|
| 26 |
+
return f"""### TASK: TEACHING ANALYSIS
|
| 27 |
+
Teacher Guide:
|
| 28 |
+
{sample['teacher_guide']}
|
| 29 |
+
|
| 30 |
+
Student Feedback:
|
| 31 |
+
{sample['student_feedback']}
|
| 32 |
+
|
| 33 |
+
Time Signals:
|
| 34 |
+
{time_lines}
|
| 35 |
+
|
| 36 |
+
Return:
|
| 37 |
+
"""
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class MultiTaskModel(nn.Module):
|
| 41 |
+
def __init__(self, base_model):
|
| 42 |
+
super().__init__()
|
| 43 |
+
self.base = base_model
|
| 44 |
+
hidden = base_model.config.hidden_size
|
| 45 |
+
|
| 46 |
+
self.score_head = nn.Sequential(
|
| 47 |
+
nn.Linear(hidden, 1),
|
| 48 |
+
nn.Sigmoid()
|
| 49 |
+
)
|
| 50 |
+
self.section_head = nn.Linear(hidden, NUM_SECTIONS)
|
| 51 |
+
|
| 52 |
+
def forward(self, input_ids, attention_mask, labels=None,
|
| 53 |
+
score=None, section_labels=None):
|
| 54 |
+
|
| 55 |
+
out = self.base(
|
| 56 |
+
input_ids=input_ids,
|
| 57 |
+
attention_mask=attention_mask,
|
| 58 |
+
labels=labels,
|
| 59 |
+
output_hidden_states=True
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
# last real token
|
| 63 |
+
seq_len = attention_mask.sum(dim=1) - 1
|
| 64 |
+
hidden = out.hidden_states[-1][
|
| 65 |
+
torch.arange(input_ids.size(0)), seq_len
|
| 66 |
+
]
|
| 67 |
+
# Ensure hidden state is bfloat16 to match head dtypes
|
| 68 |
+
hidden = hidden.to(torch.bfloat16)
|
| 69 |
+
|
| 70 |
+
# Ensure base model's loss is also bfloat16 for consistent arithmetic
|
| 71 |
+
loss = None
|
| 72 |
+
if out.loss is not None:
|
| 73 |
+
loss = out.loss.to(torch.bfloat16)
|
| 74 |
+
|
| 75 |
+
score_pred = self.score_head(hidden).squeeze(-1)
|
| 76 |
+
section_logits = self.section_head(hidden)
|
| 77 |
+
|
| 78 |
+
if score is not None and score.min() >= 0:
|
| 79 |
+
loss += nn.functional.mse_loss(score_pred, score)
|
| 80 |
+
|
| 81 |
+
if section_labels is not None:
|
| 82 |
+
loss += nn.functional.binary_cross_entropy_with_logits(
|
| 83 |
+
section_logits, section_labels
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
return {
|
| 87 |
+
"loss": loss,
|
| 88 |
+
"score": score_pred,
|
| 89 |
+
"sections": section_logits
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
loaded_tokenizer = AutoTokenizer.from_pretrained(ADAPTER_ID)
|
| 95 |
+
|
| 96 |
+
loaded_bnb_config = BitsAndBytesConfig(
|
| 97 |
+
load_in_4bit=True,
|
| 98 |
+
bnb_4bit_compute_dtype=torch.bfloat16
|
| 99 |
+
)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
loaded_base_model = AutoModelForCausalLM.from_pretrained(
|
| 103 |
+
BASE_MODEL,
|
| 104 |
+
quantization_config=loaded_bnb_config,
|
| 105 |
+
device_map="auto"
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
loaded_peft_model = PeftModel.from_pretrained(loaded_base_model, ADAPTER_ID)
|
| 109 |
+
|
| 110 |
+
loaded_multi_task_model = MultiTaskModel(loaded_peft_model).to(DEVICE)
|
| 111 |
+
# Convert the loaded model's custom heads to bfloat16 to match the base model's output dtype
|
| 112 |
+
loaded_multi_task_model.score_head = loaded_multi_task_model.score_head.to(torch.bfloat16)
|
| 113 |
+
loaded_multi_task_model.section_head = loaded_multi_task_model.section_head.to(torch.bfloat16)
|
| 114 |
+
|
| 115 |
+
app = Flask(__name__)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
@torch.no_grad()
|
| 119 |
+
def infer(sample):
|
| 120 |
+
prompt = build_prompt(sample)
|
| 121 |
+
ids = loaded_tokenizer(prompt, return_tensors="pt").to(DEVICE)
|
| 122 |
+
|
| 123 |
+
gen = loaded_multi_task_model.base.generate(
|
| 124 |
+
**ids,
|
| 125 |
+
max_new_tokens=200,
|
| 126 |
+
temperature=0.8,
|
| 127 |
+
top_p=0.95
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
outputs = loaded_multi_task_model(
|
| 131 |
+
input_ids=ids["input_ids"],
|
| 132 |
+
attention_mask=ids["attention_mask"]
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
return {
|
| 136 |
+
"generated_output": loaded_tokenizer.decode(gen[0], skip_special_tokens=True),
|
| 137 |
+
"matching_score": outputs["score"].item(),
|
| 138 |
+
"section_probs": torch.sigmoid(outputs["sections"]).cpu().tolist()
|
| 139 |
+
}
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
@app.route("/generate", methods=["POST"])
|
| 143 |
+
def generate():
|
| 144 |
+
data = request.get_json(force=True)
|
| 145 |
+
|
| 146 |
+
if not data:
|
| 147 |
+
return jsonify({"error": "Missing 'prompt'"}), 400
|
| 148 |
+
|
| 149 |
+
prompt = infer(data)
|
| 150 |
+
|
| 151 |
+
return jsonify({"output": prompt})
|
| 152 |
+
|
| 153 |
+
if __name__ == "__main__":
|
| 154 |
+
app.run(host="0.0.0.0", port=7860)
|
requirements.txt
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch
|
| 2 |
+
transformers
|
| 3 |
+
accelerate
|
| 4 |
+
peft
|
| 5 |
+
safetensors
|
| 6 |
+
huggingface_hub
|
| 7 |
+
flask
|
| 8 |
+
bitsandbytes
|