Sync InvoiceGuard code for GRPO training job
Browse files- training/README.md +11 -11
- training/launch_hf_job.py +4 -4
- training/train_grpo.py +3 -3
training/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
# InvoiceGuard Round 2 — Trajectory-level GRPO
|
| 2 |
|
| 3 |
-
This package trains a small instruction-tuned LM (default `Qwen/
|
| 4 |
on the InvoiceGuard OpenEnv with a hand-written multi-step GRPO loop:
|
| 5 |
|
| 6 |
- Sample **G trajectories** per training task with the current (stochastic) policy.
|
|
@@ -41,7 +41,7 @@ benchmark for the README plots is produced separately by
|
|
| 41 |
```powershell
|
| 42 |
cd invoice_guard
|
| 43 |
..\.venv\Scripts\python -m training.train_grpo `
|
| 44 |
-
--model-name Qwen/
|
| 45 |
--num-iterations 1 --group-size 2 --max-train-tasks 2 --no-push
|
| 46 |
```
|
| 47 |
|
|
@@ -60,7 +60,7 @@ cd invoice_guard
|
|
| 60 |
--hf-username <your-username> `
|
| 61 |
--flavor a10g-large `
|
| 62 |
--timeout 4h `
|
| 63 |
-
--base-model Qwen/
|
| 64 |
--num-iterations 3 --group-size 4
|
| 65 |
```
|
| 66 |
|
|
@@ -74,19 +74,19 @@ What happens:
|
|
| 74 |
4. Trackio dashboard appears at `https://huggingface.co/spaces/<user>/trackio`
|
| 75 |
(project `invoiceguard-round2`, run `qwen3b-grpo`).
|
| 76 |
5. On completion the LoRA adapter is pushed to
|
| 77 |
-
`{your-username}/invoiceguard-
|
| 78 |
|
| 79 |
## After training
|
| 80 |
|
| 81 |
Run the Round 2 benchmark against the trained model:
|
| 82 |
|
| 83 |
```powershell
|
| 84 |
-
$env:API_BASE_URL = "<endpoint serving your-username/invoiceguard-
|
| 85 |
-
$env:MODEL_NAME = "<your-username>/invoiceguard-
|
| 86 |
-
..\.venv\Scripts\python eval_round2.py --slice all --model-tag
|
| 87 |
..\.venv\Scripts\python eval_round2.py --compare `
|
| 88 |
-
outputs\
|
| 89 |
-
outputs\round2\
|
| 90 |
```
|
| 91 |
|
| 92 |
The compare output is what the README before/after plot is built from in
|
|
@@ -96,6 +96,6 @@ Stage H.
|
|
| 96 |
|
| 97 |
| Setting | Approx | Notes |
|
| 98 |
| --- | --- | --- |
|
| 99 |
-
| `
|
| 100 |
-
| `a10g-
|
| 101 |
| Smoke test (CPU) | $0 | Uses in-tree env, no Hub push. |
|
|
|
|
| 1 |
# InvoiceGuard Round 2 — Trajectory-level GRPO
|
| 2 |
|
| 3 |
+
This package trains a small instruction-tuned LM (default `Qwen/Qwen3-4B-Instruct-2507`)
|
| 4 |
on the InvoiceGuard OpenEnv with a hand-written multi-step GRPO loop:
|
| 5 |
|
| 6 |
- Sample **G trajectories** per training task with the current (stochastic) policy.
|
|
|
|
| 41 |
```powershell
|
| 42 |
cd invoice_guard
|
| 43 |
..\.venv\Scripts\python -m training.train_grpo `
|
| 44 |
+
--model-name Qwen/Qwen3-4B-Instruct-2507 `
|
| 45 |
--num-iterations 1 --group-size 2 --max-train-tasks 2 --no-push
|
| 46 |
```
|
| 47 |
|
|
|
|
| 60 |
--hf-username <your-username> `
|
| 61 |
--flavor a10g-large `
|
| 62 |
--timeout 4h `
|
| 63 |
+
--base-model Qwen/Qwen3-4B-Instruct-2507 `
|
| 64 |
--num-iterations 3 --group-size 4
|
| 65 |
```
|
| 66 |
|
|
|
|
| 74 |
4. Trackio dashboard appears at `https://huggingface.co/spaces/<user>/trackio`
|
| 75 |
(project `invoiceguard-round2`, run `qwen3b-grpo`).
|
| 76 |
5. On completion the LoRA adapter is pushed to
|
| 77 |
+
`{your-username}/invoiceguard-qwen3-4b-grpo`.
|
| 78 |
|
| 79 |
## After training
|
| 80 |
|
| 81 |
Run the Round 2 benchmark against the trained model:
|
| 82 |
|
| 83 |
```powershell
|
| 84 |
+
$env:API_BASE_URL = "<endpoint serving your-username/invoiceguard-qwen3-4b-grpo>"
|
| 85 |
+
$env:MODEL_NAME = "<your-username>/invoiceguard-qwen3-4b-grpo"
|
| 86 |
+
..\.venv\Scripts\python eval_round2.py --slice all --model-tag trained_qwen3_4b_grpo
|
| 87 |
..\.venv\Scripts\python eval_round2.py --compare `
|
| 88 |
+
outputs\baseline_scores\hard__clean_qwen3_4b.json `
|
| 89 |
+
outputs\round2\hard__trained_qwen3_4b_grpo.json
|
| 90 |
```
|
| 91 |
|
| 92 |
The compare output is what the README before/after plot is built from in
|
|
|
|
| 96 |
|
| 97 |
| Setting | Approx | Notes |
|
| 98 |
| --- | --- | --- |
|
| 99 |
+
| `l40s` / `a100-large` × 2-4h | varies | Default — Qwen3-4B-Instruct, 3 iter × 16 tasks × G=4 = 192 trajectories per epoch. |
|
| 100 |
+
| `a10g-large` × 3-5h | lower cost | May require shorter prompts/generation and 4-bit LoRA. |
|
| 101 |
| Smoke test (CPU) | $0 | Uses in-tree env, no Hub push. |
|
training/launch_hf_job.py
CHANGED
|
@@ -16,7 +16,7 @@ Usage:
|
|
| 16 |
--hf-username <your-username> \
|
| 17 |
--flavor a10g-large \
|
| 18 |
--timeout 4h \
|
| 19 |
-
--base-model Qwen/
|
| 20 |
--num-iterations 3 --group-size 4
|
| 21 |
|
| 22 |
Requires:
|
|
@@ -114,8 +114,8 @@ def main() -> None:
|
|
| 114 |
p = argparse.ArgumentParser()
|
| 115 |
p.add_argument("--hf-username", required=True)
|
| 116 |
p.add_argument("--code-repo-name", default="invoiceguard-code")
|
| 117 |
-
p.add_argument("--hub-model-id", default="invoiceguard-
|
| 118 |
-
p.add_argument("--base-model", default="Qwen/
|
| 119 |
p.add_argument("--flavor", default="a10g-large")
|
| 120 |
p.add_argument("--timeout", default="4h")
|
| 121 |
p.add_argument("--num-iterations", type=int, default=3)
|
|
@@ -126,7 +126,7 @@ def main() -> None:
|
|
| 126 |
p.add_argument("--max-new-tokens", type=int, default=96)
|
| 127 |
p.add_argument("--max-prompt-tokens", type=int, default=2048)
|
| 128 |
p.add_argument("--trackio-project", default="invoiceguard-round2")
|
| 129 |
-
p.add_argument("--run-name", default="
|
| 130 |
p.add_argument("--skip-upload", action="store_true",
|
| 131 |
help="Reuse the existing code repo (no re-upload).")
|
| 132 |
args = p.parse_args()
|
|
|
|
| 16 |
--hf-username <your-username> \
|
| 17 |
--flavor a10g-large \
|
| 18 |
--timeout 4h \
|
| 19 |
+
--base-model Qwen/Qwen3-4B-Instruct-2507 \
|
| 20 |
--num-iterations 3 --group-size 4
|
| 21 |
|
| 22 |
Requires:
|
|
|
|
| 114 |
p = argparse.ArgumentParser()
|
| 115 |
p.add_argument("--hf-username", required=True)
|
| 116 |
p.add_argument("--code-repo-name", default="invoiceguard-code")
|
| 117 |
+
p.add_argument("--hub-model-id", default="invoiceguard-qwen3-4b-grpo")
|
| 118 |
+
p.add_argument("--base-model", default="Qwen/Qwen3-4B-Instruct-2507")
|
| 119 |
p.add_argument("--flavor", default="a10g-large")
|
| 120 |
p.add_argument("--timeout", default="4h")
|
| 121 |
p.add_argument("--num-iterations", type=int, default=3)
|
|
|
|
| 126 |
p.add_argument("--max-new-tokens", type=int, default=96)
|
| 127 |
p.add_argument("--max-prompt-tokens", type=int, default=2048)
|
| 128 |
p.add_argument("--trackio-project", default="invoiceguard-round2")
|
| 129 |
+
p.add_argument("--run-name", default="qwen3-4b-grpo")
|
| 130 |
p.add_argument("--skip-upload", action="store_true",
|
| 131 |
help="Reuse the existing code repo (no re-upload).")
|
| 132 |
args = p.parse_args()
|
training/train_grpo.py
CHANGED
|
@@ -143,11 +143,11 @@ from training.rollout import Trajectory, TrajectoryStep, rollout_episode # type
|
|
| 143 |
|
| 144 |
@dataclass
|
| 145 |
class TrainConfig:
|
| 146 |
-
base_model: str = os.environ.get("BASE_MODEL", "Qwen/
|
| 147 |
hub_username: Optional[str] = os.environ.get("HF_USERNAME")
|
| 148 |
-
hub_model_id: str = os.environ.get("HUB_MODEL_ID", "invoiceguard-
|
| 149 |
trackio_project: str = os.environ.get("TRACKIO_PROJECT", "invoiceguard-round2")
|
| 150 |
-
trackio_run_name: str = os.environ.get("TRACKIO_RUN_NAME", "
|
| 151 |
artifact_dir: str = os.environ.get("ARTIFACT_DIR", "/tmp/invoiceguard-training-artifacts")
|
| 152 |
|
| 153 |
seed: int = 42
|
|
|
|
| 143 |
|
| 144 |
@dataclass
|
| 145 |
class TrainConfig:
|
| 146 |
+
base_model: str = os.environ.get("BASE_MODEL", "Qwen/Qwen3-4B-Instruct-2507")
|
| 147 |
hub_username: Optional[str] = os.environ.get("HF_USERNAME")
|
| 148 |
+
hub_model_id: str = os.environ.get("HUB_MODEL_ID", "invoiceguard-qwen3-4b-grpo")
|
| 149 |
trackio_project: str = os.environ.get("TRACKIO_PROJECT", "invoiceguard-round2")
|
| 150 |
+
trackio_run_name: str = os.environ.get("TRACKIO_RUN_NAME", "qwen3-4b-grpo")
|
| 151 |
artifact_dir: str = os.environ.get("ARTIFACT_DIR", "/tmp/invoiceguard-training-artifacts")
|
| 152 |
|
| 153 |
seed: int = 42
|