Improve skill classifier training for contacts, Gmail, and Slack.
Browse filesAdd a shared system prompt, contrastive search examples, deduped skills,
and stronger LoRA settings so the model distinguishes similar app actions.
- data/skills.jsonl +0 -2
- data/train.jsonl +0 -0
- modal_apps/evaluate_modal.py +15 -1
- modal_apps/infer_modal.py +15 -1
- modal_apps/predict_api.py +15 -1
- modal_apps/train_modal.py +3 -3
- scripts/generate_training_data.py +127 -7
- src/classifier_prompt.py +17 -0
- src/evaluate.py +2 -1
data/skills.jsonl
CHANGED
|
@@ -12,6 +12,4 @@
|
|
| 12 |
{"skill": "youtube_search", "task": "search Ghibli food on youtube"}
|
| 13 |
{"skill": "contacts_search", "task": "search parag shah in contacts"}
|
| 14 |
{"skill": "gmail_send_email", "task": "send mail using Gmail to kriyanshishah06@gmail.com saying hello-world"}
|
| 15 |
-
{"skill": "whatsapp_send_message", "task": "text hi to biraj on WhatsApp"}
|
| 16 |
-
{"skill": "bluetooth_enable", "task": "turn on bluetooth"}
|
| 17 |
{"skill": "bluetooth_enable", "task": "turn on bluetooth"}
|
|
|
|
| 12 |
{"skill": "youtube_search", "task": "search Ghibli food on youtube"}
|
| 13 |
{"skill": "contacts_search", "task": "search parag shah in contacts"}
|
| 14 |
{"skill": "gmail_send_email", "task": "send mail using Gmail to kriyanshishah06@gmail.com saying hello-world"}
|
|
|
|
|
|
|
| 15 |
{"skill": "bluetooth_enable", "task": "turn on bluetooth"}
|
data/train.jsonl
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
modal_apps/evaluate_modal.py
CHANGED
|
@@ -35,6 +35,20 @@ MODEL_DIR = pathlib.Path("/model")
|
|
| 35 |
ADAPTER_DIR = MODEL_DIR / "adapter"
|
| 36 |
MAX_SEQ_LENGTH = 2048
|
| 37 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
GPU_TYPE = "A10G"
|
| 39 |
TIMEOUT_SECONDS = 30 * 60
|
| 40 |
|
|
@@ -168,7 +182,7 @@ def evaluate() -> None:
|
|
| 168 |
prompt = case["prompt"]
|
| 169 |
expected = case["expected"]
|
| 170 |
|
| 171 |
-
messages =
|
| 172 |
inputs = tokenizer.apply_chat_template(
|
| 173 |
messages,
|
| 174 |
tokenize=True,
|
|
|
|
| 35 |
ADAPTER_DIR = MODEL_DIR / "adapter"
|
| 36 |
MAX_SEQ_LENGTH = 2048
|
| 37 |
|
| 38 |
+
SYSTEM_PROMPT = (
|
| 39 |
+
"You classify Android automation requests into exactly one skill. "
|
| 40 |
+
'Reply with JSON only: {"skill": "<skill_name>"}. '
|
| 41 |
+
"Use the app or action named in the request (contacts, Gmail, Slack, YouTube, etc.) "
|
| 42 |
+
"to pick the correct skill."
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def build_classifier_messages(user_content: str) -> list[dict[str, str]]:
|
| 47 |
+
return [
|
| 48 |
+
{"role": "system", "content": SYSTEM_PROMPT},
|
| 49 |
+
{"role": "user", "content": user_content},
|
| 50 |
+
]
|
| 51 |
+
|
| 52 |
GPU_TYPE = "A10G"
|
| 53 |
TIMEOUT_SECONDS = 30 * 60
|
| 54 |
|
|
|
|
| 182 |
prompt = case["prompt"]
|
| 183 |
expected = case["expected"]
|
| 184 |
|
| 185 |
+
messages = build_classifier_messages(prompt)
|
| 186 |
inputs = tokenizer.apply_chat_template(
|
| 187 |
messages,
|
| 188 |
tokenize=True,
|
modal_apps/infer_modal.py
CHANGED
|
@@ -29,6 +29,20 @@ MODEL_DIR = pathlib.Path("/model")
|
|
| 29 |
ADAPTER_DIR = MODEL_DIR / "adapter"
|
| 30 |
MAX_SEQ_LENGTH = 2048
|
| 31 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
GPU_TYPE = "A10G"
|
| 33 |
TIMEOUT_SECONDS = 10 * 60
|
| 34 |
|
|
@@ -135,7 +149,7 @@ def infer(prompt: str) -> str:
|
|
| 135 |
)
|
| 136 |
FastLanguageModel.for_inference(model)
|
| 137 |
|
| 138 |
-
messages =
|
| 139 |
inputs = tokenizer.apply_chat_template(
|
| 140 |
messages,
|
| 141 |
tokenize=True,
|
|
|
|
| 29 |
ADAPTER_DIR = MODEL_DIR / "adapter"
|
| 30 |
MAX_SEQ_LENGTH = 2048
|
| 31 |
|
| 32 |
+
SYSTEM_PROMPT = (
|
| 33 |
+
"You classify Android automation requests into exactly one skill. "
|
| 34 |
+
'Reply with JSON only: {"skill": "<skill_name>"}. '
|
| 35 |
+
"Use the app or action named in the request (contacts, Gmail, Slack, YouTube, etc.) "
|
| 36 |
+
"to pick the correct skill."
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def build_classifier_messages(user_content: str) -> list[dict[str, str]]:
|
| 41 |
+
return [
|
| 42 |
+
{"role": "system", "content": SYSTEM_PROMPT},
|
| 43 |
+
{"role": "user", "content": user_content},
|
| 44 |
+
]
|
| 45 |
+
|
| 46 |
GPU_TYPE = "A10G"
|
| 47 |
TIMEOUT_SECONDS = 10 * 60
|
| 48 |
|
|
|
|
| 149 |
)
|
| 150 |
FastLanguageModel.for_inference(model)
|
| 151 |
|
| 152 |
+
messages = build_classifier_messages(prompt)
|
| 153 |
inputs = tokenizer.apply_chat_template(
|
| 154 |
messages,
|
| 155 |
tokenize=True,
|
modal_apps/predict_api.py
CHANGED
|
@@ -46,6 +46,20 @@ MODEL_DIR = pathlib.Path("/model")
|
|
| 46 |
ADAPTER_DIR = MODEL_DIR / "adapter"
|
| 47 |
MAX_SEQ_LENGTH = 2048
|
| 48 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
GPU_TYPE = "A10G"
|
| 50 |
TIMEOUT_SECONDS = 10 * 60
|
| 51 |
SCALEDOWN_WINDOW = 5 * 60
|
|
@@ -153,7 +167,7 @@ class SkillPredictor:
|
|
| 153 |
print("Model ready.")
|
| 154 |
|
| 155 |
def _generate_skill(self, prompt: str) -> str | None:
|
| 156 |
-
messages =
|
| 157 |
inputs = self.tokenizer.apply_chat_template(
|
| 158 |
messages,
|
| 159 |
tokenize=True,
|
|
|
|
| 46 |
ADAPTER_DIR = MODEL_DIR / "adapter"
|
| 47 |
MAX_SEQ_LENGTH = 2048
|
| 48 |
|
| 49 |
+
SYSTEM_PROMPT = (
|
| 50 |
+
"You classify Android automation requests into exactly one skill. "
|
| 51 |
+
'Reply with JSON only: {"skill": "<skill_name>"}. '
|
| 52 |
+
"Use the app or action named in the request (contacts, Gmail, Slack, YouTube, etc.) "
|
| 53 |
+
"to pick the correct skill."
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def build_classifier_messages(user_content: str) -> list[dict[str, str]]:
|
| 58 |
+
return [
|
| 59 |
+
{"role": "system", "content": SYSTEM_PROMPT},
|
| 60 |
+
{"role": "user", "content": user_content},
|
| 61 |
+
]
|
| 62 |
+
|
| 63 |
GPU_TYPE = "A10G"
|
| 64 |
TIMEOUT_SECONDS = 10 * 60
|
| 65 |
SCALEDOWN_WINDOW = 5 * 60
|
|
|
|
| 167 |
print("Model ready.")
|
| 168 |
|
| 169 |
def _generate_skill(self, prompt: str) -> str | None:
|
| 170 |
+
messages = build_classifier_messages(prompt)
|
| 171 |
inputs = self.tokenizer.apply_chat_template(
|
| 172 |
messages,
|
| 173 |
tokenize=True,
|
modal_apps/train_modal.py
CHANGED
|
@@ -39,9 +39,9 @@ MERGED_DIR = MODEL_DIR / "merged"
|
|
| 39 |
CHECKPOINT_DIR = MODEL_DIR / "checkpoints"
|
| 40 |
MAX_SEQ_LENGTH = 2048
|
| 41 |
|
| 42 |
-
LORA_R =
|
| 43 |
-
LORA_ALPHA =
|
| 44 |
-
NUM_EPOCHS =
|
| 45 |
BATCH_SIZE = 8
|
| 46 |
|
| 47 |
GPU_TYPE = "A10G"
|
|
|
|
| 39 |
CHECKPOINT_DIR = MODEL_DIR / "checkpoints"
|
| 40 |
MAX_SEQ_LENGTH = 2048
|
| 41 |
|
| 42 |
+
LORA_R = 32
|
| 43 |
+
LORA_ALPHA = 32
|
| 44 |
+
NUM_EPOCHS = 5
|
| 45 |
BATCH_SIZE = 8
|
| 46 |
|
| 47 |
GPU_TYPE = "A10G"
|
scripts/generate_training_data.py
CHANGED
|
@@ -3,13 +3,40 @@
|
|
| 3 |
|
| 4 |
import json
|
| 5 |
import re
|
|
|
|
| 6 |
from pathlib import Path
|
| 7 |
|
| 8 |
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
INPUT_FILE = PROJECT_ROOT / "data" / "skills.jsonl"
|
| 10 |
OUTPUT_FILE = PROJECT_ROOT / "data" / "train.jsonl"
|
| 11 |
VARIATIONS_PER_SKILL = 30
|
| 12 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
def _unique_lower(prompts: list[str], limit: int) -> list[str]:
|
| 15 |
seen: set[str] = set()
|
|
@@ -32,6 +59,7 @@ def _assistant_content(skill: str) -> str:
|
|
| 32 |
def _record(skill: str, user_content: str) -> dict:
|
| 33 |
return {
|
| 34 |
"messages": [
|
|
|
|
| 35 |
{"role": "user", "content": user_content},
|
| 36 |
{"role": "assistant", "content": _assistant_content(skill)},
|
| 37 |
]
|
|
@@ -332,6 +360,14 @@ def generate_slack_open_channel(task: str) -> list[str]:
|
|
| 332 |
return _unique_lower(
|
| 333 |
[
|
| 334 |
task,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 335 |
f"open slack channel {channel}",
|
| 336 |
f"go to {channel} channel on slack",
|
| 337 |
f"open the {channel} slack channel",
|
|
@@ -649,6 +685,11 @@ def generate_contacts_search(task: str) -> list[str]:
|
|
| 649 |
return _unique_lower(
|
| 650 |
[
|
| 651 |
task,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 652 |
f"find {contact} in contacts",
|
| 653 |
f"look up {contact} in my contacts",
|
| 654 |
f"contacts search {contact}",
|
|
@@ -700,6 +741,13 @@ def generate_gmail_send_email(task: str) -> list[str]:
|
|
| 700 |
return _unique_lower(
|
| 701 |
[
|
| 702 |
task,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 703 |
f"email {email} {body} via gmail",
|
| 704 |
f"compose gmail to {email} saying {body}",
|
| 705 |
f"send email to {email} {body}",
|
|
@@ -767,6 +815,57 @@ GENERATORS = {
|
|
| 767 |
}
|
| 768 |
|
| 769 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 770 |
def generate_variations(skill: str, task: str) -> list[str]:
|
| 771 |
generator = GENERATORS.get(skill, lambda t: generate_generic(skill, t))
|
| 772 |
return generator(task)
|
|
@@ -783,23 +882,44 @@ def load_skills(path: Path) -> list[dict]:
|
|
| 783 |
|
| 784 |
|
| 785 |
def main() -> None:
|
| 786 |
-
skills = load_skills(INPUT_FILE)
|
| 787 |
-
records = []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 788 |
|
| 789 |
for entry in skills:
|
| 790 |
skill = entry["skill"]
|
| 791 |
task = entry["task"]
|
| 792 |
for prompt in generate_variations(skill, task):
|
| 793 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 794 |
|
| 795 |
with OUTPUT_FILE.open("w", encoding="utf-8") as f:
|
| 796 |
for record in records:
|
| 797 |
f.write(json.dumps(record) + "\n")
|
| 798 |
|
| 799 |
-
|
| 800 |
-
|
| 801 |
-
|
| 802 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 803 |
|
| 804 |
|
| 805 |
if __name__ == "__main__":
|
|
|
|
| 3 |
|
| 4 |
import json
|
| 5 |
import re
|
| 6 |
+
import sys
|
| 7 |
from pathlib import Path
|
| 8 |
|
| 9 |
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
| 10 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 11 |
+
|
| 12 |
+
from src.classifier_prompt import SYSTEM_PROMPT # noqa: E402
|
| 13 |
+
|
| 14 |
INPUT_FILE = PROJECT_ROOT / "data" / "skills.jsonl"
|
| 15 |
OUTPUT_FILE = PROJECT_ROOT / "data" / "train.jsonl"
|
| 16 |
VARIATIONS_PER_SKILL = 30
|
| 17 |
|
| 18 |
+
# Guaranteed in training — matches Gradio demo prompts and target weak skills.
|
| 19 |
+
MUST_INCLUDE_PROMPTS: dict[str, str] = {
|
| 20 |
+
"play my workout playlist": "spotify_play_playlist",
|
| 21 |
+
"turn bluetooth on": "bluetooth_enable",
|
| 22 |
+
"wake me up tomorrow morning": "create_alarm",
|
| 23 |
+
"send ri a message on whatsapp": "whatsapp_send_message",
|
| 24 |
+
"open the engineering channel in slack": "slack_open_channel",
|
| 25 |
+
"pause spotify": "spotify_pause",
|
| 26 |
+
"book an uber to the airport": "uber_request_ride",
|
| 27 |
+
"search pasta recipes on youtube": "youtube_search",
|
| 28 |
+
"email my team saying project update": "gmail_send_email",
|
| 29 |
+
"send gmail to boss saying i'll be late": "gmail_send_email",
|
| 30 |
+
"write mail to john@company.com saying hello": "gmail_send_email",
|
| 31 |
+
"compose an email to sarah@gmail.com thanks for your help": "gmail_send_email",
|
| 32 |
+
"find parag shah in contacts": "contacts_search",
|
| 33 |
+
"look up john smith in my contacts": "contacts_search",
|
| 34 |
+
"search contact parag shah": "contacts_search",
|
| 35 |
+
"open slack channel data contributors": "slack_open_channel",
|
| 36 |
+
"go to general channel in slack": "slack_open_channel",
|
| 37 |
+
"navigate to data contributors on slack": "slack_open_channel",
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
|
| 41 |
def _unique_lower(prompts: list[str], limit: int) -> list[str]:
|
| 42 |
seen: set[str] = set()
|
|
|
|
| 59 |
def _record(skill: str, user_content: str) -> dict:
|
| 60 |
return {
|
| 61 |
"messages": [
|
| 62 |
+
{"role": "system", "content": SYSTEM_PROMPT},
|
| 63 |
{"role": "user", "content": user_content},
|
| 64 |
{"role": "assistant", "content": _assistant_content(skill)},
|
| 65 |
]
|
|
|
|
| 360 |
return _unique_lower(
|
| 361 |
[
|
| 362 |
task,
|
| 363 |
+
"open the engineering channel in slack",
|
| 364 |
+
"open slack channel data contributors",
|
| 365 |
+
"go to general channel in slack",
|
| 366 |
+
"navigate to data contributors on slack",
|
| 367 |
+
"switch to #general in slack",
|
| 368 |
+
"open the data contributors slack channel",
|
| 369 |
+
"take me to engineering on slack",
|
| 370 |
+
"show me the random channel on slack",
|
| 371 |
f"open slack channel {channel}",
|
| 372 |
f"go to {channel} channel on slack",
|
| 373 |
f"open the {channel} slack channel",
|
|
|
|
| 685 |
return _unique_lower(
|
| 686 |
[
|
| 687 |
task,
|
| 688 |
+
"find parag shah in contacts",
|
| 689 |
+
"look up john smith in my contacts",
|
| 690 |
+
"search contact parag shah",
|
| 691 |
+
"find mom in my contacts",
|
| 692 |
+
"search contacts for dad",
|
| 693 |
f"find {contact} in contacts",
|
| 694 |
f"look up {contact} in my contacts",
|
| 695 |
f"contacts search {contact}",
|
|
|
|
| 741 |
return _unique_lower(
|
| 742 |
[
|
| 743 |
task,
|
| 744 |
+
"email my team saying project update",
|
| 745 |
+
"send gmail to boss saying i'll be late",
|
| 746 |
+
"write mail to john@company.com saying hello",
|
| 747 |
+
"compose an email to sarah@gmail.com thanks for your help",
|
| 748 |
+
"send mail using gmail",
|
| 749 |
+
"write an email to the team",
|
| 750 |
+
"compose mail saying hello world",
|
| 751 |
f"email {email} {body} via gmail",
|
| 752 |
f"compose gmail to {email} saying {body}",
|
| 753 |
f"send email to {email} {body}",
|
|
|
|
| 815 |
}
|
| 816 |
|
| 817 |
|
| 818 |
+
def generate_contrastive_examples() -> list[dict]:
|
| 819 |
+
"""Hard negatives: same phrasing, different app → different skill."""
|
| 820 |
+
records: list[dict] = []
|
| 821 |
+
|
| 822 |
+
search_names = ["parag shah", "arya sheth", "john smith", "mom", "sarah"]
|
| 823 |
+
for name in search_names:
|
| 824 |
+
records.extend(
|
| 825 |
+
[
|
| 826 |
+
_record("contacts_search", f"search {name} in contacts"),
|
| 827 |
+
_record("contacts_search", f"find {name} in my contacts"),
|
| 828 |
+
_record("linkedin_search_person", f"search {name} on linkedin"),
|
| 829 |
+
_record("linkedin_search_person", f"find {name} on linkedin"),
|
| 830 |
+
_record("youtube_search", f"search {name} on youtube"),
|
| 831 |
+
_record("youtube_search", f"look up {name} videos on youtube"),
|
| 832 |
+
_record("spotify_search_play", f"search {name} on spotify and play"),
|
| 833 |
+
_record("spotify_search_play", f"find {name} music on spotify and play it"),
|
| 834 |
+
]
|
| 835 |
+
)
|
| 836 |
+
|
| 837 |
+
records.extend(
|
| 838 |
+
[
|
| 839 |
+
_record("gmail_send_email", "email boss saying i'll be late"),
|
| 840 |
+
_record("gmail_send_email", "send mail to team@work.com weekly report"),
|
| 841 |
+
_record("gmail_send_email", "compose gmail to client proposal attached"),
|
| 842 |
+
_record("whatsapp_send_message", "message boss on whatsapp running late"),
|
| 843 |
+
_record("whatsapp_send_message", "text team on whatsapp meeting moved"),
|
| 844 |
+
_record("whatsapp_send_message", "send ri a message on whatsapp"),
|
| 845 |
+
_record("slack_open_channel", "open the engineering channel in slack"),
|
| 846 |
+
_record("slack_open_channel", "go to data contributors channel on slack"),
|
| 847 |
+
_record("slack_open_channel", "switch to announcements in slack"),
|
| 848 |
+
_record("youtube_search", "search pasta recipes on youtube"),
|
| 849 |
+
_record("contacts_search", "find parag shah in contacts"),
|
| 850 |
+
_record("gmail_send_email", "write mail to john@company.com saying hello"),
|
| 851 |
+
]
|
| 852 |
+
)
|
| 853 |
+
|
| 854 |
+
return records
|
| 855 |
+
|
| 856 |
+
|
| 857 |
+
def dedupe_skills(skills: list[dict]) -> list[dict]:
|
| 858 |
+
seen: set[str] = set()
|
| 859 |
+
result: list[dict] = []
|
| 860 |
+
for entry in skills:
|
| 861 |
+
skill = entry["skill"]
|
| 862 |
+
if skill in seen:
|
| 863 |
+
continue
|
| 864 |
+
seen.add(skill)
|
| 865 |
+
result.append(entry)
|
| 866 |
+
return result
|
| 867 |
+
|
| 868 |
+
|
| 869 |
def generate_variations(skill: str, task: str) -> list[str]:
|
| 870 |
generator = GENERATORS.get(skill, lambda t: generate_generic(skill, t))
|
| 871 |
return generator(task)
|
|
|
|
| 882 |
|
| 883 |
|
| 884 |
def main() -> None:
|
| 885 |
+
skills = dedupe_skills(load_skills(INPUT_FILE))
|
| 886 |
+
records: list[dict] = []
|
| 887 |
+
seen_prompts: set[str] = set()
|
| 888 |
+
|
| 889 |
+
def add_record(skill: str, prompt: str) -> None:
|
| 890 |
+
key = prompt.lower().strip()
|
| 891 |
+
if not key or key in seen_prompts:
|
| 892 |
+
return
|
| 893 |
+
seen_prompts.add(key)
|
| 894 |
+
records.append(_record(skill, prompt))
|
| 895 |
|
| 896 |
for entry in skills:
|
| 897 |
skill = entry["skill"]
|
| 898 |
task = entry["task"]
|
| 899 |
for prompt in generate_variations(skill, task):
|
| 900 |
+
add_record(skill, prompt)
|
| 901 |
+
|
| 902 |
+
for prompt, skill in MUST_INCLUDE_PROMPTS.items():
|
| 903 |
+
add_record(skill, prompt)
|
| 904 |
+
|
| 905 |
+
for record in generate_contrastive_examples():
|
| 906 |
+
prompt = record["messages"][1]["content"]
|
| 907 |
+
skill = json.loads(record["messages"][2]["content"])["skill"]
|
| 908 |
+
add_record(skill, prompt)
|
| 909 |
|
| 910 |
with OUTPUT_FILE.open("w", encoding="utf-8") as f:
|
| 911 |
for record in records:
|
| 912 |
f.write(json.dumps(record) + "\n")
|
| 913 |
|
| 914 |
+
skill_counts: dict[str, int] = {}
|
| 915 |
+
for record in records:
|
| 916 |
+
skill = json.loads(record["messages"][2]["content"])["skill"]
|
| 917 |
+
skill_counts[skill] = skill_counts.get(skill, 0) + 1
|
| 918 |
+
|
| 919 |
+
print(f"Wrote {len(records)} training examples to {OUTPUT_FILE}")
|
| 920 |
+
print(f"Skills: {len(skills)} unique, contrastive + must-include prompts merged")
|
| 921 |
+
for skill, count in sorted(skill_counts.items()):
|
| 922 |
+
print(f" {skill}: {count}")
|
| 923 |
|
| 924 |
|
| 925 |
if __name__ == "__main__":
|
src/classifier_prompt.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Shared system prompt for skill classification (training + inference)."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
SYSTEM_PROMPT = (
|
| 6 |
+
"You classify Android automation requests into exactly one skill. "
|
| 7 |
+
'Reply with JSON only: {"skill": "<skill_name>"}. '
|
| 8 |
+
"Use the app or action named in the request (contacts, Gmail, Slack, YouTube, etc.) "
|
| 9 |
+
"to pick the correct skill."
|
| 10 |
+
)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def build_classifier_messages(user_content: str) -> list[dict[str, str]]:
|
| 14 |
+
return [
|
| 15 |
+
{"role": "system", "content": SYSTEM_PROMPT},
|
| 16 |
+
{"role": "user", "content": user_content},
|
| 17 |
+
]
|
src/evaluate.py
CHANGED
|
@@ -23,6 +23,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
| 23 |
|
| 24 |
from src.paths import DATA_DIR, TRAINED_MODEL_DIR
|
| 25 |
from src.skill_utils import extract_skill
|
|
|
|
| 26 |
|
| 27 |
BASE_MODEL = "Qwen/Qwen2.5-3B-Instruct"
|
| 28 |
MAX_SEQ_LENGTH = 2048
|
|
@@ -121,7 +122,7 @@ def load_model(model_path: Path, device: str):
|
|
| 121 |
|
| 122 |
|
| 123 |
def generate_skill(model, tokenizer, prompt: str, device: str) -> str:
|
| 124 |
-
messages =
|
| 125 |
inputs = tokenizer.apply_chat_template(
|
| 126 |
messages,
|
| 127 |
tokenize=True,
|
|
|
|
| 23 |
|
| 24 |
from src.paths import DATA_DIR, TRAINED_MODEL_DIR
|
| 25 |
from src.skill_utils import extract_skill
|
| 26 |
+
from src.classifier_prompt import build_classifier_messages
|
| 27 |
|
| 28 |
BASE_MODEL = "Qwen/Qwen2.5-3B-Instruct"
|
| 29 |
MAX_SEQ_LENGTH = 2048
|
|
|
|
| 122 |
|
| 123 |
|
| 124 |
def generate_skill(model, tokenizer, prompt: str, device: str) -> str:
|
| 125 |
+
messages = build_classifier_messages(prompt)
|
| 126 |
inputs = tokenizer.apply_chat_template(
|
| 127 |
messages,
|
| 128 |
tokenize=True,
|