piyush-mk commited on
Commit
c5cfe6d
·
verified ·
1 Parent(s): d1e9e92

Sync InvoiceGuard code for GRPO training job

Browse files
training/README.md CHANGED
@@ -1,6 +1,6 @@
1
  # InvoiceGuard Round 2 — Trajectory-level GRPO
2
 
3
- This package trains a small instruction-tuned LM (default `Qwen/Qwen2.5-3B-Instruct`)
4
  on the InvoiceGuard OpenEnv with a hand-written multi-step GRPO loop:
5
 
6
  - Sample **G trajectories** per training task with the current (stochastic) policy.
@@ -41,7 +41,7 @@ benchmark for the README plots is produced separately by
41
  ```powershell
42
  cd invoice_guard
43
  ..\.venv\Scripts\python -m training.train_grpo `
44
- --model-name Qwen/Qwen2.5-0.5B-Instruct `
45
  --num-iterations 1 --group-size 2 --max-train-tasks 2 --no-push
46
  ```
47
 
@@ -60,7 +60,7 @@ cd invoice_guard
60
  --hf-username <your-username> `
61
  --flavor a10g-large `
62
  --timeout 4h `
63
- --base-model Qwen/Qwen2.5-3B-Instruct `
64
  --num-iterations 3 --group-size 4
65
  ```
66
 
@@ -74,19 +74,19 @@ What happens:
74
  4. Trackio dashboard appears at `https://huggingface.co/spaces/<user>/trackio`
75
  (project `invoiceguard-round2`, run `qwen3b-grpo`).
76
  5. On completion the LoRA adapter is pushed to
77
- `{your-username}/invoiceguard-qwen3b-grpo`.
78
 
79
  ## After training
80
 
81
  Run the Round 2 benchmark against the trained model:
82
 
83
  ```powershell
84
- $env:API_BASE_URL = "<endpoint serving your-username/invoiceguard-qwen3b-grpo>"
85
- $env:MODEL_NAME = "<your-username>/invoiceguard-qwen3b-grpo"
86
- ..\.venv\Scripts\python eval_round2.py --slice all --model-tag trained_qwen3b_grpo
87
  ..\.venv\Scripts\python eval_round2.py --compare `
88
- outputs\round2\hard__baseline_qwen3b.json `
89
- outputs\round2\hard__trained_qwen3b_grpo.json
90
  ```
91
 
92
  The compare output is what the README before/after plot is built from in
@@ -96,6 +96,6 @@ Stage H.
96
 
97
  | Setting | Approx | Notes |
98
  | --- | --- | --- |
99
- | `a10g-large` × 3-4h | $15-20 | Default — Qwen2.5-3B-Instruct, 3 iter × 16 tasks × G=4 = 192 trajectories per epoch. |
100
- | `a10g-small` × 4h | $14 | If you drop to Qwen2.5-1.5B-Instruct. |
101
  | Smoke test (CPU) | $0 | Uses in-tree env, no Hub push. |
 
1
  # InvoiceGuard Round 2 — Trajectory-level GRPO
2
 
3
+ This package trains a small instruction-tuned LM (default `Qwen/Qwen3-4B-Instruct-2507`)
4
  on the InvoiceGuard OpenEnv with a hand-written multi-step GRPO loop:
5
 
6
  - Sample **G trajectories** per training task with the current (stochastic) policy.
 
41
  ```powershell
42
  cd invoice_guard
43
  ..\.venv\Scripts\python -m training.train_grpo `
44
+ --model-name Qwen/Qwen3-4B-Instruct-2507 `
45
  --num-iterations 1 --group-size 2 --max-train-tasks 2 --no-push
46
  ```
47
 
 
60
  --hf-username <your-username> `
61
  --flavor a10g-large `
62
  --timeout 4h `
63
+ --base-model Qwen/Qwen3-4B-Instruct-2507 `
64
  --num-iterations 3 --group-size 4
65
  ```
66
 
 
74
  4. Trackio dashboard appears at `https://huggingface.co/spaces/<user>/trackio`
75
  (project `invoiceguard-round2`, run `qwen3b-grpo`).
76
  5. On completion the LoRA adapter is pushed to
77
+ `{your-username}/invoiceguard-qwen3-4b-grpo`.
78
 
79
  ## After training
80
 
81
  Run the Round 2 benchmark against the trained model:
82
 
83
  ```powershell
84
+ $env:API_BASE_URL = "<endpoint serving your-username/invoiceguard-qwen3-4b-grpo>"
85
+ $env:MODEL_NAME = "<your-username>/invoiceguard-qwen3-4b-grpo"
86
+ ..\.venv\Scripts\python eval_round2.py --slice all --model-tag trained_qwen3_4b_grpo
87
  ..\.venv\Scripts\python eval_round2.py --compare `
88
+ outputs\baseline_scores\hard__clean_qwen3_4b.json `
89
+ outputs\round2\hard__trained_qwen3_4b_grpo.json
90
  ```
91
 
92
  The compare output is what the README before/after plot is built from in
 
96
 
97
  | Setting | Approx | Notes |
98
  | --- | --- | --- |
99
+ | `l40s` / `a100-large` × 2-4h | varies | Default — Qwen3-4B-Instruct, 3 iter × 16 tasks × G=4 = 192 trajectories per epoch. |
100
+ | `a10g-large` × 3-5h | lower cost | May require shorter prompts/generation and 4-bit LoRA. |
101
  | Smoke test (CPU) | $0 | Uses in-tree env, no Hub push. |
training/launch_hf_job.py CHANGED
@@ -16,7 +16,7 @@ Usage:
16
  --hf-username <your-username> \
17
  --flavor a10g-large \
18
  --timeout 4h \
19
- --base-model Qwen/Qwen2.5-7B-Instruct \
20
  --num-iterations 3 --group-size 4
21
 
22
  Requires:
@@ -114,8 +114,8 @@ def main() -> None:
114
  p = argparse.ArgumentParser()
115
  p.add_argument("--hf-username", required=True)
116
  p.add_argument("--code-repo-name", default="invoiceguard-code")
117
- p.add_argument("--hub-model-id", default="invoiceguard-qwen25-7b-grpo")
118
- p.add_argument("--base-model", default="Qwen/Qwen2.5-7B-Instruct")
119
  p.add_argument("--flavor", default="a10g-large")
120
  p.add_argument("--timeout", default="4h")
121
  p.add_argument("--num-iterations", type=int, default=3)
@@ -126,7 +126,7 @@ def main() -> None:
126
  p.add_argument("--max-new-tokens", type=int, default=96)
127
  p.add_argument("--max-prompt-tokens", type=int, default=2048)
128
  p.add_argument("--trackio-project", default="invoiceguard-round2")
129
- p.add_argument("--run-name", default="qwen25-7b-grpo")
130
  p.add_argument("--skip-upload", action="store_true",
131
  help="Reuse the existing code repo (no re-upload).")
132
  args = p.parse_args()
 
16
  --hf-username <your-username> \
17
  --flavor a10g-large \
18
  --timeout 4h \
19
+ --base-model Qwen/Qwen3-4B-Instruct-2507 \
20
  --num-iterations 3 --group-size 4
21
 
22
  Requires:
 
114
  p = argparse.ArgumentParser()
115
  p.add_argument("--hf-username", required=True)
116
  p.add_argument("--code-repo-name", default="invoiceguard-code")
117
+ p.add_argument("--hub-model-id", default="invoiceguard-qwen3-4b-grpo")
118
+ p.add_argument("--base-model", default="Qwen/Qwen3-4B-Instruct-2507")
119
  p.add_argument("--flavor", default="a10g-large")
120
  p.add_argument("--timeout", default="4h")
121
  p.add_argument("--num-iterations", type=int, default=3)
 
126
  p.add_argument("--max-new-tokens", type=int, default=96)
127
  p.add_argument("--max-prompt-tokens", type=int, default=2048)
128
  p.add_argument("--trackio-project", default="invoiceguard-round2")
129
+ p.add_argument("--run-name", default="qwen3-4b-grpo")
130
  p.add_argument("--skip-upload", action="store_true",
131
  help="Reuse the existing code repo (no re-upload).")
132
  args = p.parse_args()
training/train_grpo.py CHANGED
@@ -143,11 +143,11 @@ from training.rollout import Trajectory, TrajectoryStep, rollout_episode # type
143
 
144
  @dataclass
145
  class TrainConfig:
146
- base_model: str = os.environ.get("BASE_MODEL", "Qwen/Qwen2.5-7B-Instruct")
147
  hub_username: Optional[str] = os.environ.get("HF_USERNAME")
148
- hub_model_id: str = os.environ.get("HUB_MODEL_ID", "invoiceguard-qwen25-7b-grpo")
149
  trackio_project: str = os.environ.get("TRACKIO_PROJECT", "invoiceguard-round2")
150
- trackio_run_name: str = os.environ.get("TRACKIO_RUN_NAME", "qwen25-7b-grpo")
151
  artifact_dir: str = os.environ.get("ARTIFACT_DIR", "/tmp/invoiceguard-training-artifacts")
152
 
153
  seed: int = 42
 
143
 
144
  @dataclass
145
  class TrainConfig:
146
+ base_model: str = os.environ.get("BASE_MODEL", "Qwen/Qwen3-4B-Instruct-2507")
147
  hub_username: Optional[str] = os.environ.get("HF_USERNAME")
148
+ hub_model_id: str = os.environ.get("HUB_MODEL_ID", "invoiceguard-qwen3-4b-grpo")
149
  trackio_project: str = os.environ.get("TRACKIO_PROJECT", "invoiceguard-round2")
150
+ trackio_run_name: str = os.environ.get("TRACKIO_RUN_NAME", "qwen3-4b-grpo")
151
  artifact_dir: str = os.environ.get("ARTIFACT_DIR", "/tmp/invoiceguard-training-artifacts")
152
 
153
  seed: int = 42