kriyanshi commited on
Commit
24492a8
·
1 Parent(s): a4bdc35

Improve skill classifier training for contacts, Gmail, and Slack.

Browse files

Add a shared system prompt, contrastive search examples, deduped skills,
and stronger LoRA settings so the model distinguishes similar app actions.

data/skills.jsonl CHANGED
@@ -12,6 +12,4 @@
12
  {"skill": "youtube_search", "task": "search Ghibli food on youtube"}
13
  {"skill": "contacts_search", "task": "search parag shah in contacts"}
14
  {"skill": "gmail_send_email", "task": "send mail using Gmail to kriyanshishah06@gmail.com saying hello-world"}
15
- {"skill": "whatsapp_send_message", "task": "text hi to biraj on WhatsApp"}
16
- {"skill": "bluetooth_enable", "task": "turn on bluetooth"}
17
  {"skill": "bluetooth_enable", "task": "turn on bluetooth"}
 
12
  {"skill": "youtube_search", "task": "search Ghibli food on youtube"}
13
  {"skill": "contacts_search", "task": "search parag shah in contacts"}
14
  {"skill": "gmail_send_email", "task": "send mail using Gmail to kriyanshishah06@gmail.com saying hello-world"}
 
 
15
  {"skill": "bluetooth_enable", "task": "turn on bluetooth"}
data/train.jsonl CHANGED
The diff for this file is too large to render. See raw diff
 
modal_apps/evaluate_modal.py CHANGED
@@ -35,6 +35,20 @@ MODEL_DIR = pathlib.Path("/model")
35
  ADAPTER_DIR = MODEL_DIR / "adapter"
36
  MAX_SEQ_LENGTH = 2048
37
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
38
  GPU_TYPE = "A10G"
39
  TIMEOUT_SECONDS = 30 * 60
40
 
@@ -168,7 +182,7 @@ def evaluate() -> None:
168
  prompt = case["prompt"]
169
  expected = case["expected"]
170
 
171
- messages = [{"role": "user", "content": prompt}]
172
  inputs = tokenizer.apply_chat_template(
173
  messages,
174
  tokenize=True,
 
35
  ADAPTER_DIR = MODEL_DIR / "adapter"
36
  MAX_SEQ_LENGTH = 2048
37
 
38
+ SYSTEM_PROMPT = (
39
+ "You classify Android automation requests into exactly one skill. "
40
+ 'Reply with JSON only: {"skill": "<skill_name>"}. '
41
+ "Use the app or action named in the request (contacts, Gmail, Slack, YouTube, etc.) "
42
+ "to pick the correct skill."
43
+ )
44
+
45
+
46
+ def build_classifier_messages(user_content: str) -> list[dict[str, str]]:
47
+ return [
48
+ {"role": "system", "content": SYSTEM_PROMPT},
49
+ {"role": "user", "content": user_content},
50
+ ]
51
+
52
  GPU_TYPE = "A10G"
53
  TIMEOUT_SECONDS = 30 * 60
54
 
 
182
  prompt = case["prompt"]
183
  expected = case["expected"]
184
 
185
+ messages = build_classifier_messages(prompt)
186
  inputs = tokenizer.apply_chat_template(
187
  messages,
188
  tokenize=True,
modal_apps/infer_modal.py CHANGED
@@ -29,6 +29,20 @@ MODEL_DIR = pathlib.Path("/model")
29
  ADAPTER_DIR = MODEL_DIR / "adapter"
30
  MAX_SEQ_LENGTH = 2048
31
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  GPU_TYPE = "A10G"
33
  TIMEOUT_SECONDS = 10 * 60
34
 
@@ -135,7 +149,7 @@ def infer(prompt: str) -> str:
135
  )
136
  FastLanguageModel.for_inference(model)
137
 
138
- messages = [{"role": "user", "content": prompt}]
139
  inputs = tokenizer.apply_chat_template(
140
  messages,
141
  tokenize=True,
 
29
  ADAPTER_DIR = MODEL_DIR / "adapter"
30
  MAX_SEQ_LENGTH = 2048
31
 
32
+ SYSTEM_PROMPT = (
33
+ "You classify Android automation requests into exactly one skill. "
34
+ 'Reply with JSON only: {"skill": "<skill_name>"}. '
35
+ "Use the app or action named in the request (contacts, Gmail, Slack, YouTube, etc.) "
36
+ "to pick the correct skill."
37
+ )
38
+
39
+
40
+ def build_classifier_messages(user_content: str) -> list[dict[str, str]]:
41
+ return [
42
+ {"role": "system", "content": SYSTEM_PROMPT},
43
+ {"role": "user", "content": user_content},
44
+ ]
45
+
46
  GPU_TYPE = "A10G"
47
  TIMEOUT_SECONDS = 10 * 60
48
 
 
149
  )
150
  FastLanguageModel.for_inference(model)
151
 
152
+ messages = build_classifier_messages(prompt)
153
  inputs = tokenizer.apply_chat_template(
154
  messages,
155
  tokenize=True,
modal_apps/predict_api.py CHANGED
@@ -46,6 +46,20 @@ MODEL_DIR = pathlib.Path("/model")
46
  ADAPTER_DIR = MODEL_DIR / "adapter"
47
  MAX_SEQ_LENGTH = 2048
48
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
49
  GPU_TYPE = "A10G"
50
  TIMEOUT_SECONDS = 10 * 60
51
  SCALEDOWN_WINDOW = 5 * 60
@@ -153,7 +167,7 @@ class SkillPredictor:
153
  print("Model ready.")
154
 
155
  def _generate_skill(self, prompt: str) -> str | None:
156
- messages = [{"role": "user", "content": prompt}]
157
  inputs = self.tokenizer.apply_chat_template(
158
  messages,
159
  tokenize=True,
 
46
  ADAPTER_DIR = MODEL_DIR / "adapter"
47
  MAX_SEQ_LENGTH = 2048
48
 
49
+ SYSTEM_PROMPT = (
50
+ "You classify Android automation requests into exactly one skill. "
51
+ 'Reply with JSON only: {"skill": "<skill_name>"}. '
52
+ "Use the app or action named in the request (contacts, Gmail, Slack, YouTube, etc.) "
53
+ "to pick the correct skill."
54
+ )
55
+
56
+
57
+ def build_classifier_messages(user_content: str) -> list[dict[str, str]]:
58
+ return [
59
+ {"role": "system", "content": SYSTEM_PROMPT},
60
+ {"role": "user", "content": user_content},
61
+ ]
62
+
63
  GPU_TYPE = "A10G"
64
  TIMEOUT_SECONDS = 10 * 60
65
  SCALEDOWN_WINDOW = 5 * 60
 
167
  print("Model ready.")
168
 
169
  def _generate_skill(self, prompt: str) -> str | None:
170
+ messages = build_classifier_messages(prompt)
171
  inputs = self.tokenizer.apply_chat_template(
172
  messages,
173
  tokenize=True,
modal_apps/train_modal.py CHANGED
@@ -39,9 +39,9 @@ MERGED_DIR = MODEL_DIR / "merged"
39
  CHECKPOINT_DIR = MODEL_DIR / "checkpoints"
40
  MAX_SEQ_LENGTH = 2048
41
 
42
- LORA_R = 16
43
- LORA_ALPHA = 16
44
- NUM_EPOCHS = 3
45
  BATCH_SIZE = 8
46
 
47
  GPU_TYPE = "A10G"
 
39
  CHECKPOINT_DIR = MODEL_DIR / "checkpoints"
40
  MAX_SEQ_LENGTH = 2048
41
 
42
+ LORA_R = 32
43
+ LORA_ALPHA = 32
44
+ NUM_EPOCHS = 5
45
  BATCH_SIZE = 8
46
 
47
  GPU_TYPE = "A10G"
scripts/generate_training_data.py CHANGED
@@ -3,13 +3,40 @@
3
 
4
  import json
5
  import re
 
6
  from pathlib import Path
7
 
8
  PROJECT_ROOT = Path(__file__).resolve().parent.parent
 
 
 
 
9
  INPUT_FILE = PROJECT_ROOT / "data" / "skills.jsonl"
10
  OUTPUT_FILE = PROJECT_ROOT / "data" / "train.jsonl"
11
  VARIATIONS_PER_SKILL = 30
12
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
 
14
  def _unique_lower(prompts: list[str], limit: int) -> list[str]:
15
  seen: set[str] = set()
@@ -32,6 +59,7 @@ def _assistant_content(skill: str) -> str:
32
  def _record(skill: str, user_content: str) -> dict:
33
  return {
34
  "messages": [
 
35
  {"role": "user", "content": user_content},
36
  {"role": "assistant", "content": _assistant_content(skill)},
37
  ]
@@ -332,6 +360,14 @@ def generate_slack_open_channel(task: str) -> list[str]:
332
  return _unique_lower(
333
  [
334
  task,
 
 
 
 
 
 
 
 
335
  f"open slack channel {channel}",
336
  f"go to {channel} channel on slack",
337
  f"open the {channel} slack channel",
@@ -649,6 +685,11 @@ def generate_contacts_search(task: str) -> list[str]:
649
  return _unique_lower(
650
  [
651
  task,
 
 
 
 
 
652
  f"find {contact} in contacts",
653
  f"look up {contact} in my contacts",
654
  f"contacts search {contact}",
@@ -700,6 +741,13 @@ def generate_gmail_send_email(task: str) -> list[str]:
700
  return _unique_lower(
701
  [
702
  task,
 
 
 
 
 
 
 
703
  f"email {email} {body} via gmail",
704
  f"compose gmail to {email} saying {body}",
705
  f"send email to {email} {body}",
@@ -767,6 +815,57 @@ GENERATORS = {
767
  }
768
 
769
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
770
  def generate_variations(skill: str, task: str) -> list[str]:
771
  generator = GENERATORS.get(skill, lambda t: generate_generic(skill, t))
772
  return generator(task)
@@ -783,23 +882,44 @@ def load_skills(path: Path) -> list[dict]:
783
 
784
 
785
  def main() -> None:
786
- skills = load_skills(INPUT_FILE)
787
- records = []
 
 
 
 
 
 
 
 
788
 
789
  for entry in skills:
790
  skill = entry["skill"]
791
  task = entry["task"]
792
  for prompt in generate_variations(skill, task):
793
- records.append(_record(skill, prompt))
 
 
 
 
 
 
 
 
794
 
795
  with OUTPUT_FILE.open("w", encoding="utf-8") as f:
796
  for record in records:
797
  f.write(json.dumps(record) + "\n")
798
 
799
- print(
800
- f"Wrote {len(records)} training examples "
801
- f"({len(skills)} skills x {VARIATIONS_PER_SKILL} variations) to {OUTPUT_FILE}"
802
- )
 
 
 
 
 
803
 
804
 
805
  if __name__ == "__main__":
 
3
 
4
  import json
5
  import re
6
+ import sys
7
  from pathlib import Path
8
 
9
  PROJECT_ROOT = Path(__file__).resolve().parent.parent
10
+ sys.path.insert(0, str(PROJECT_ROOT))
11
+
12
+ from src.classifier_prompt import SYSTEM_PROMPT # noqa: E402
13
+
14
  INPUT_FILE = PROJECT_ROOT / "data" / "skills.jsonl"
15
  OUTPUT_FILE = PROJECT_ROOT / "data" / "train.jsonl"
16
  VARIATIONS_PER_SKILL = 30
17
 
18
+ # Guaranteed in training — matches Gradio demo prompts and target weak skills.
19
+ MUST_INCLUDE_PROMPTS: dict[str, str] = {
20
+ "play my workout playlist": "spotify_play_playlist",
21
+ "turn bluetooth on": "bluetooth_enable",
22
+ "wake me up tomorrow morning": "create_alarm",
23
+ "send ri a message on whatsapp": "whatsapp_send_message",
24
+ "open the engineering channel in slack": "slack_open_channel",
25
+ "pause spotify": "spotify_pause",
26
+ "book an uber to the airport": "uber_request_ride",
27
+ "search pasta recipes on youtube": "youtube_search",
28
+ "email my team saying project update": "gmail_send_email",
29
+ "send gmail to boss saying i'll be late": "gmail_send_email",
30
+ "write mail to john@company.com saying hello": "gmail_send_email",
31
+ "compose an email to sarah@gmail.com thanks for your help": "gmail_send_email",
32
+ "find parag shah in contacts": "contacts_search",
33
+ "look up john smith in my contacts": "contacts_search",
34
+ "search contact parag shah": "contacts_search",
35
+ "open slack channel data contributors": "slack_open_channel",
36
+ "go to general channel in slack": "slack_open_channel",
37
+ "navigate to data contributors on slack": "slack_open_channel",
38
+ }
39
+
40
 
41
  def _unique_lower(prompts: list[str], limit: int) -> list[str]:
42
  seen: set[str] = set()
 
59
  def _record(skill: str, user_content: str) -> dict:
60
  return {
61
  "messages": [
62
+ {"role": "system", "content": SYSTEM_PROMPT},
63
  {"role": "user", "content": user_content},
64
  {"role": "assistant", "content": _assistant_content(skill)},
65
  ]
 
360
  return _unique_lower(
361
  [
362
  task,
363
+ "open the engineering channel in slack",
364
+ "open slack channel data contributors",
365
+ "go to general channel in slack",
366
+ "navigate to data contributors on slack",
367
+ "switch to #general in slack",
368
+ "open the data contributors slack channel",
369
+ "take me to engineering on slack",
370
+ "show me the random channel on slack",
371
  f"open slack channel {channel}",
372
  f"go to {channel} channel on slack",
373
  f"open the {channel} slack channel",
 
685
  return _unique_lower(
686
  [
687
  task,
688
+ "find parag shah in contacts",
689
+ "look up john smith in my contacts",
690
+ "search contact parag shah",
691
+ "find mom in my contacts",
692
+ "search contacts for dad",
693
  f"find {contact} in contacts",
694
  f"look up {contact} in my contacts",
695
  f"contacts search {contact}",
 
741
  return _unique_lower(
742
  [
743
  task,
744
+ "email my team saying project update",
745
+ "send gmail to boss saying i'll be late",
746
+ "write mail to john@company.com saying hello",
747
+ "compose an email to sarah@gmail.com thanks for your help",
748
+ "send mail using gmail",
749
+ "write an email to the team",
750
+ "compose mail saying hello world",
751
  f"email {email} {body} via gmail",
752
  f"compose gmail to {email} saying {body}",
753
  f"send email to {email} {body}",
 
815
  }
816
 
817
 
818
+ def generate_contrastive_examples() -> list[dict]:
819
+ """Hard negatives: same phrasing, different app → different skill."""
820
+ records: list[dict] = []
821
+
822
+ search_names = ["parag shah", "arya sheth", "john smith", "mom", "sarah"]
823
+ for name in search_names:
824
+ records.extend(
825
+ [
826
+ _record("contacts_search", f"search {name} in contacts"),
827
+ _record("contacts_search", f"find {name} in my contacts"),
828
+ _record("linkedin_search_person", f"search {name} on linkedin"),
829
+ _record("linkedin_search_person", f"find {name} on linkedin"),
830
+ _record("youtube_search", f"search {name} on youtube"),
831
+ _record("youtube_search", f"look up {name} videos on youtube"),
832
+ _record("spotify_search_play", f"search {name} on spotify and play"),
833
+ _record("spotify_search_play", f"find {name} music on spotify and play it"),
834
+ ]
835
+ )
836
+
837
+ records.extend(
838
+ [
839
+ _record("gmail_send_email", "email boss saying i'll be late"),
840
+ _record("gmail_send_email", "send mail to team@work.com weekly report"),
841
+ _record("gmail_send_email", "compose gmail to client proposal attached"),
842
+ _record("whatsapp_send_message", "message boss on whatsapp running late"),
843
+ _record("whatsapp_send_message", "text team on whatsapp meeting moved"),
844
+ _record("whatsapp_send_message", "send ri a message on whatsapp"),
845
+ _record("slack_open_channel", "open the engineering channel in slack"),
846
+ _record("slack_open_channel", "go to data contributors channel on slack"),
847
+ _record("slack_open_channel", "switch to announcements in slack"),
848
+ _record("youtube_search", "search pasta recipes on youtube"),
849
+ _record("contacts_search", "find parag shah in contacts"),
850
+ _record("gmail_send_email", "write mail to john@company.com saying hello"),
851
+ ]
852
+ )
853
+
854
+ return records
855
+
856
+
857
+ def dedupe_skills(skills: list[dict]) -> list[dict]:
858
+ seen: set[str] = set()
859
+ result: list[dict] = []
860
+ for entry in skills:
861
+ skill = entry["skill"]
862
+ if skill in seen:
863
+ continue
864
+ seen.add(skill)
865
+ result.append(entry)
866
+ return result
867
+
868
+
869
  def generate_variations(skill: str, task: str) -> list[str]:
870
  generator = GENERATORS.get(skill, lambda t: generate_generic(skill, t))
871
  return generator(task)
 
882
 
883
 
884
  def main() -> None:
885
+ skills = dedupe_skills(load_skills(INPUT_FILE))
886
+ records: list[dict] = []
887
+ seen_prompts: set[str] = set()
888
+
889
+ def add_record(skill: str, prompt: str) -> None:
890
+ key = prompt.lower().strip()
891
+ if not key or key in seen_prompts:
892
+ return
893
+ seen_prompts.add(key)
894
+ records.append(_record(skill, prompt))
895
 
896
  for entry in skills:
897
  skill = entry["skill"]
898
  task = entry["task"]
899
  for prompt in generate_variations(skill, task):
900
+ add_record(skill, prompt)
901
+
902
+ for prompt, skill in MUST_INCLUDE_PROMPTS.items():
903
+ add_record(skill, prompt)
904
+
905
+ for record in generate_contrastive_examples():
906
+ prompt = record["messages"][1]["content"]
907
+ skill = json.loads(record["messages"][2]["content"])["skill"]
908
+ add_record(skill, prompt)
909
 
910
  with OUTPUT_FILE.open("w", encoding="utf-8") as f:
911
  for record in records:
912
  f.write(json.dumps(record) + "\n")
913
 
914
+ skill_counts: dict[str, int] = {}
915
+ for record in records:
916
+ skill = json.loads(record["messages"][2]["content"])["skill"]
917
+ skill_counts[skill] = skill_counts.get(skill, 0) + 1
918
+
919
+ print(f"Wrote {len(records)} training examples to {OUTPUT_FILE}")
920
+ print(f"Skills: {len(skills)} unique, contrastive + must-include prompts merged")
921
+ for skill, count in sorted(skill_counts.items()):
922
+ print(f" {skill}: {count}")
923
 
924
 
925
  if __name__ == "__main__":
src/classifier_prompt.py ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Shared system prompt for skill classification (training + inference)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ SYSTEM_PROMPT = (
6
+ "You classify Android automation requests into exactly one skill. "
7
+ 'Reply with JSON only: {"skill": "<skill_name>"}. '
8
+ "Use the app or action named in the request (contacts, Gmail, Slack, YouTube, etc.) "
9
+ "to pick the correct skill."
10
+ )
11
+
12
+
13
+ def build_classifier_messages(user_content: str) -> list[dict[str, str]]:
14
+ return [
15
+ {"role": "system", "content": SYSTEM_PROMPT},
16
+ {"role": "user", "content": user_content},
17
+ ]
src/evaluate.py CHANGED
@@ -23,6 +23,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
23
 
24
  from src.paths import DATA_DIR, TRAINED_MODEL_DIR
25
  from src.skill_utils import extract_skill
 
26
 
27
  BASE_MODEL = "Qwen/Qwen2.5-3B-Instruct"
28
  MAX_SEQ_LENGTH = 2048
@@ -121,7 +122,7 @@ def load_model(model_path: Path, device: str):
121
 
122
 
123
  def generate_skill(model, tokenizer, prompt: str, device: str) -> str:
124
- messages = [{"role": "user", "content": prompt}]
125
  inputs = tokenizer.apply_chat_template(
126
  messages,
127
  tokenize=True,
 
23
 
24
  from src.paths import DATA_DIR, TRAINED_MODEL_DIR
25
  from src.skill_utils import extract_skill
26
+ from src.classifier_prompt import build_classifier_messages
27
 
28
  BASE_MODEL = "Qwen/Qwen2.5-3B-Instruct"
29
  MAX_SEQ_LENGTH = 2048
 
122
 
123
 
124
  def generate_skill(model, tokenizer, prompt: str, device: str) -> str:
125
+ messages = build_classifier_messages(prompt)
126
  inputs = tokenizer.apply_chat_template(
127
  messages,
128
  tokenize=True,