Commit ·
2a5ee2b
1
Parent(s): cc71c5f
feat(evaluation): add the ViT-GPT2 Phase 3 greedy run
Browse filesnlpconnect/vit-gpt2-image-captioning at the pinned revision dc68f91,
greedy (num_beams 1, no sampling, max_new_tokens 40, repetition penalty
1.0, no prompt, float32), scored on the committed 500-image slice under
EVAL_METHODOLOGY.md section 8. Written by scripts/compare_models.py.
- results/phase3-vit-gpt2-greedy/comparison_meta.json +30 -0
- results/phase3-vit-gpt2-greedy/diagnostics.jsonl +0 -0
- results/phase3-vit-gpt2-greedy/metrics.json +11 -0
- results/phase3-vit-gpt2-greedy/predictions.jsonl +0 -0
- results/phase3-vit-gpt2-greedy/report.md +20 -0
- results/phase3-vit-gpt2-greedy/run_meta.json +12 -0
results/phase3-vit-gpt2-greedy/comparison_meta.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"protocol": "docs/EVAL_METHODOLOGY.md § 8",
|
| 3 |
+
"model_id": "vit-gpt2",
|
| 4 |
+
"backend": "huggingface",
|
| 5 |
+
"captioner": "HFCaptioner",
|
| 6 |
+
"hub_repo": "nlpconnect/vit-gpt2-image-captioning",
|
| 7 |
+
"revision": "dc68f91c06a1ba6f15268e5b9c13ae7a7c514084",
|
| 8 |
+
"decode_settings": {
|
| 9 |
+
"num_beams": 1,
|
| 10 |
+
"do_sample": false,
|
| 11 |
+
"max_new_tokens": 40,
|
| 12 |
+
"repetition_penalty": 1.0,
|
| 13 |
+
"no_repeat_ngram_size": 0,
|
| 14 |
+
"precision": "float32"
|
| 15 |
+
},
|
| 16 |
+
"normalisation": "preprocess_caption -> strip_sentinels",
|
| 17 |
+
"slice": {
|
| 18 |
+
"source": "results/stabilized-greedy/predictions.jsonl",
|
| 19 |
+
"fingerprint_sha256": "6b5628bfa410ed233ef9603beed3c05c9e63acebc8bfa31d7e63e634f9e25116",
|
| 20 |
+
"images": 500,
|
| 21 |
+
"references": 732
|
| 22 |
+
},
|
| 23 |
+
"batch_size": 1,
|
| 24 |
+
"device": "cpu",
|
| 25 |
+
"seed": 42,
|
| 26 |
+
"metrics": {
|
| 27 |
+
"meteor": true,
|
| 28 |
+
"cider": true
|
| 29 |
+
}
|
| 30 |
+
}
|
results/phase3-vit-gpt2-greedy/diagnostics.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/phase3-vit-gpt2-greedy/metrics.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_examples": 500,
|
| 3 |
+
"bleu1": 49.12276232739276,
|
| 4 |
+
"bleu2": 33.41039138807018,
|
| 5 |
+
"bleu3": 22.91337848185665,
|
| 6 |
+
"bleu4": 15.84152396302579,
|
| 7 |
+
"rouge_l": 44.509803731649235,
|
| 8 |
+
"meteor": 19.782713178028878,
|
| 9 |
+
"cider": 1.2629916931297422,
|
| 10 |
+
"errors": {}
|
| 11 |
+
}
|
results/phase3-vit-gpt2-greedy/predictions.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/phase3-vit-gpt2-greedy/report.md
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Evaluation run — vit-gpt2
|
| 2 |
+
|
| 3 |
+
- Decode strategy: `greedy`
|
| 4 |
+
- Weights: `nlpconnect/vit-gpt2-image-captioning@dc68f91c06a1ba6f15268e5b9c13ae7a7c514084`
|
| 5 |
+
- Tokenizer dir: `nlpconnect/vit-gpt2-image-captioning@dc68f91c06a1ba6f15268e5b9c13ae7a7c514084`
|
| 6 |
+
- Samples: **500**
|
| 7 |
+
- Timestamp (UTC): 2026-10-06T14:46:39.934422+00:00
|
| 8 |
+
- Repetition penalty: 1.0
|
| 9 |
+
|
| 10 |
+
## Metrics
|
| 11 |
+
|
| 12 |
+
| Metric | Value |
|
| 13 |
+
|---|---|
|
| 14 |
+
| BLEU-1 | 49.12 |
|
| 15 |
+
| BLEU-2 | 33.41 |
|
| 16 |
+
| BLEU-3 | 22.91 |
|
| 17 |
+
| BLEU-4 | 15.84 |
|
| 18 |
+
| ROUGE-L | 44.51 |
|
| 19 |
+
| METEOR | 19.78 |
|
| 20 |
+
| CIDEr | 1.26 |
|
results/phase3-vit-gpt2-greedy/run_meta.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_id": "vit-gpt2",
|
| 3 |
+
"decode_strategy": "greedy",
|
| 4 |
+
"weights_path": "nlpconnect/vit-gpt2-image-captioning@dc68f91c06a1ba6f15268e5b9c13ae7a7c514084",
|
| 5 |
+
"tokenizer_dir": "nlpconnect/vit-gpt2-image-captioning@dc68f91c06a1ba6f15268e5b9c13ae7a7c514084",
|
| 6 |
+
"n_samples": 500,
|
| 7 |
+
"max_length": 40,
|
| 8 |
+
"beam_width": null,
|
| 9 |
+
"length_penalty": null,
|
| 10 |
+
"repetition_penalty": 1.0,
|
| 11 |
+
"timestamp_utc": "2026-10-06T14:46:39.934422+00:00"
|
| 12 |
+
}
|