Darksting4906 commited on
Commit
50087b1
·
1 Parent(s): 38702fe

Initial commit

Browse files
Files changed (3) hide show
  1. Dockerfile +12 -0
  2. app.py +154 -0
  3. requirements.txt +8 -0
Dockerfile ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM pytorch/pytorch:2.1.0-cuda12.1-cudnn8-runtime
2
+
3
+ WORKDIR /app
4
+
5
+ COPY requirements.txt .
6
+
7
+ RUN pip install --no-cache-dir -r requirements.txt
8
+
9
+ COPY app.py .
10
+
11
+ EXPOSE 7860
12
+ CMD ["python", "app.py"]
app.py ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from flask import Flask, request, jsonify
2
+ from transformers import AutoTokenizer, AutoModelForCausalLM,BitsAndBytesConfig
3
+ from peft import PeftModel
4
+ import torch
5
+ import torch.nn as nn
6
+
7
+ BASE_MODEL = "Qwen/Qwen2.5-Math-1.5B-Instruct"
8
+ ADAPTER_ID = "DarkSting/teachers_guide_and_matching_score"
9
+ NUM_SECTIONS = 8
10
+ DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
11
+
12
+ def build_prompt(sample):
13
+ if sample["task"] == "task_a":
14
+ return f"""### TASK: ANSWER EVALUATION
15
+ Topic: {sample['topic']}
16
+ Question: {sample['question']}
17
+ Student Answer:
18
+ {sample['answer']}
19
+
20
+ Return:
21
+ """
22
+ else:
23
+ time_lines = "\n".join(
24
+ f"[{k.upper()}_TIME={v}]" for k, v in sample["time_spent"].items()
25
+ )
26
+ return f"""### TASK: TEACHING ANALYSIS
27
+ Teacher Guide:
28
+ {sample['teacher_guide']}
29
+
30
+ Student Feedback:
31
+ {sample['student_feedback']}
32
+
33
+ Time Signals:
34
+ {time_lines}
35
+
36
+ Return:
37
+ """
38
+
39
+
40
+ class MultiTaskModel(nn.Module):
41
+ def __init__(self, base_model):
42
+ super().__init__()
43
+ self.base = base_model
44
+ hidden = base_model.config.hidden_size
45
+
46
+ self.score_head = nn.Sequential(
47
+ nn.Linear(hidden, 1),
48
+ nn.Sigmoid()
49
+ )
50
+ self.section_head = nn.Linear(hidden, NUM_SECTIONS)
51
+
52
+ def forward(self, input_ids, attention_mask, labels=None,
53
+ score=None, section_labels=None):
54
+
55
+ out = self.base(
56
+ input_ids=input_ids,
57
+ attention_mask=attention_mask,
58
+ labels=labels,
59
+ output_hidden_states=True
60
+ )
61
+
62
+ # last real token
63
+ seq_len = attention_mask.sum(dim=1) - 1
64
+ hidden = out.hidden_states[-1][
65
+ torch.arange(input_ids.size(0)), seq_len
66
+ ]
67
+ # Ensure hidden state is bfloat16 to match head dtypes
68
+ hidden = hidden.to(torch.bfloat16)
69
+
70
+ # Ensure base model's loss is also bfloat16 for consistent arithmetic
71
+ loss = None
72
+ if out.loss is not None:
73
+ loss = out.loss.to(torch.bfloat16)
74
+
75
+ score_pred = self.score_head(hidden).squeeze(-1)
76
+ section_logits = self.section_head(hidden)
77
+
78
+ if score is not None and score.min() >= 0:
79
+ loss += nn.functional.mse_loss(score_pred, score)
80
+
81
+ if section_labels is not None:
82
+ loss += nn.functional.binary_cross_entropy_with_logits(
83
+ section_logits, section_labels
84
+ )
85
+
86
+ return {
87
+ "loss": loss,
88
+ "score": score_pred,
89
+ "sections": section_logits
90
+ }
91
+
92
+
93
+
94
+ loaded_tokenizer = AutoTokenizer.from_pretrained(ADAPTER_ID)
95
+
96
+ loaded_bnb_config = BitsAndBytesConfig(
97
+ load_in_4bit=True,
98
+ bnb_4bit_compute_dtype=torch.bfloat16
99
+ )
100
+
101
+
102
+ loaded_base_model = AutoModelForCausalLM.from_pretrained(
103
+ BASE_MODEL,
104
+ quantization_config=loaded_bnb_config,
105
+ device_map="auto"
106
+ )
107
+
108
+ loaded_peft_model = PeftModel.from_pretrained(loaded_base_model, ADAPTER_ID)
109
+
110
+ loaded_multi_task_model = MultiTaskModel(loaded_peft_model).to(DEVICE)
111
+ # Convert the loaded model's custom heads to bfloat16 to match the base model's output dtype
112
+ loaded_multi_task_model.score_head = loaded_multi_task_model.score_head.to(torch.bfloat16)
113
+ loaded_multi_task_model.section_head = loaded_multi_task_model.section_head.to(torch.bfloat16)
114
+
115
+ app = Flask(__name__)
116
+
117
+
118
+ @torch.no_grad()
119
+ def infer(sample):
120
+ prompt = build_prompt(sample)
121
+ ids = loaded_tokenizer(prompt, return_tensors="pt").to(DEVICE)
122
+
123
+ gen = loaded_multi_task_model.base.generate(
124
+ **ids,
125
+ max_new_tokens=200,
126
+ temperature=0.8,
127
+ top_p=0.95
128
+ )
129
+
130
+ outputs = loaded_multi_task_model(
131
+ input_ids=ids["input_ids"],
132
+ attention_mask=ids["attention_mask"]
133
+ )
134
+
135
+ return {
136
+ "generated_output": loaded_tokenizer.decode(gen[0], skip_special_tokens=True),
137
+ "matching_score": outputs["score"].item(),
138
+ "section_probs": torch.sigmoid(outputs["sections"]).cpu().tolist()
139
+ }
140
+
141
+
142
+ @app.route("/generate", methods=["POST"])
143
+ def generate():
144
+ data = request.get_json(force=True)
145
+
146
+ if not data:
147
+ return jsonify({"error": "Missing 'prompt'"}), 400
148
+
149
+ prompt = infer(data)
150
+
151
+ return jsonify({"output": prompt})
152
+
153
+ if __name__ == "__main__":
154
+ app.run(host="0.0.0.0", port=7860)
requirements.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ torch
2
+ transformers
3
+ accelerate
4
+ peft
5
+ safetensors
6
+ huggingface_hub
7
+ flask
8
+ bitsandbytes