File size: 1,047 Bytes
d13981b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 | import json
from data_utils.chart.data_collector import prepare_chart_rl_data, prepare_chart_sft_data
def test_chart_reference_answer_stays_short_when_sft_target_contains_hint(tmp_path):
dataset_path = tmp_path / "chart.json"
dataset_path.write_text(
json.dumps(
[
{
"question": "What is the minimum?",
"answer": "70",
"image": "missing.png",
"human_or_machine": 0,
"hint": "Goal: inspect the chart.\nConclusion: the minimum is 70.",
}
]
),
encoding="utf-8",
)
rl_row = prepare_chart_rl_data(str(dataset_path))[0]
sft_row = prepare_chart_sft_data(str(dataset_path))[0]
assert rl_row["reference_answer"] == "Answer: 70"
assert rl_row["answer"] == "Answer: 70"
assert sft_row["reference_answer"] == "Answer: 70"
assert sft_row["answer"].startswith("Goal: inspect the chart.")
assert sft_row["answer"].endswith("Answer: 70")
|