from __future__ import annotations import json import os import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from reward_utils.chart_cot_verifier import ( numbers_equivalent, parse_chart_cot, parse_deplot_table, parse_number, verify_chart_cot_trajectory, verify_conclusion_answer_consistency, verify_grounded_claims, verify_reasoning, ) def test_parse_complete_chart_cot_with_escaped_newlines_and_final_answer() -> None: parsed = parse_chart_cot( "Goal: Find the minimum.\\n" "Observation: 2019: 70 and 2020: 77.\\n" "Reasoning: 70 is smaller than 77.\\n" "Conclusion: The minimum is 70.\\n" "Answer: draft\\nAnswer: 70" ) assert parsed.goal == "Find the minimum." assert parsed.observation == "2019: 70 and 2020: 77." assert parsed.reasoning == "70 is smaller than 77." assert parsed.conclusion == "The minimum is 70." assert parsed.answer == "70" assert parsed.structure_valid is False assert parsed.duplicate_sections == ("answer",) def test_parse_chart_cot_reports_missing_and_empty_sections() -> None: parsed = parse_chart_cot( "Goal: Find the minimum.\n" "Observation:\n" "Conclusion: The minimum is 70.\n" "Answer: 70" ) assert parsed.missing_sections == ("reasoning",) assert parsed.empty_sections == ("observation",) assert parsed.structure_valid is False def test_parse_json_encoded_deplot_and_normalize_row_width() -> None: raw = json.dumps( { "source": "google/deplot", "parsed_table": "Year | Revenue | Profit\n2019 | 1,200 | 39%\n2020 | 900", } ) table = parse_deplot_table(raw) assert table is not None assert table.columns == ("Year", "Revenue", "Profit") assert table.rows == (("2019", "1,200", "39%"), ("2020", "900", "")) def test_parse_deplot_rejects_placeholder() -> None: raw = json.dumps( { "source": "deplot_placeholder", "parsed_table": "Year | Value\n2020 | 70", } ) assert parse_deplot_table(raw) is None def test_numeric_normalization_is_percent_aware() -> None: assert parse_number("1,200").value == 1200 assert parse_number("-3.5%").is_percent is True assert numbers_equivalent(parse_number("0.39"), parse_number("39%")) is True assert numbers_equivalent(parse_number("0.39"), parse_number("39")) is False assert numbers_equivalent(parse_number("70"), parse_number("70.0")) is True def test_grounding_supports_and_contradicts_explicit_label_value_claims() -> None: table = parse_deplot_table("Year | Value\n2019 | 70\n2020 | 77") assert table is not None supported = verify_grounded_claims("Observation: 2019: 70.", table) contradicted = verify_grounded_claims("Observation: In 2019, the value is 71.", table) assert [(claim.label, claim.value, claim.status) for claim in supported] == [ ("2019", "70", "supported") ] assert any(claim.value == "71" and claim.status == "contradicted" for claim in contradicted) def test_grounding_supports_value_before_unambiguous_row_label() -> None: table = parse_deplot_table("Year | Value\n2019 | 70\n2020 | 77") assert table is not None claims = verify_grounded_claims("Observation: The lowest value is 70 in 2019.", table) assert [(claim.label, claim.value, claim.status) for claim in claims] == [ ("2019", "70", "supported") ] def test_grounding_contradicts_wrong_value_before_unambiguous_row_label() -> None: table = parse_deplot_table("Year | Value\n2019 | 70\n2020 | 77") assert table is not None claims = verify_grounded_claims("Observation: The lowest value is 71 in 2019.", table) assert [(claim.label, claim.value, claim.status) for claim in claims] == [ ("2019", "71", "contradicted") ] def test_grounding_keeps_bare_derived_number_unknown() -> None: table = parse_deplot_table("Year | Value\n2019 | 70\n2020 | 77") assert table is not None claims = verify_grounded_claims("Reasoning: The difference is 7.", table) assert len(claims) == 1 assert claims[0].value == "7" assert claims[0].status == "unknown" def test_grounding_does_not_bind_derived_result_to_nearby_year_or_series() -> None: table = parse_deplot_table( "Year | Keep same | Cut back\n2012 | 53 | 30\n" "Category | Will be eliminated | Here to stay\nValue | 43 | 49" ) assert table is not None year_sum = verify_grounded_claims( "Conclusion: The sum of Cut back and Keep same in 2012 is 83.", table ) difference = verify_grounded_claims( "Conclusion: The difference between Will be eliminated and Here to stay is 6.", table ) assert not any(claim.status == "contradicted" for claim in year_sum) assert not any(claim.status == "contradicted" for claim in difference) def test_grounding_aggregates_repeated_row_labels_across_series() -> None: table = parse_deplot_table( "Year | Entity | Value\n2004 | Armenia | 0.6831\n2004 | Singapore | 0.5147" ) assert table is not None claims = verify_grounded_claims( "Observation: Armenia has 2004: 0.6831 and Singapore has 2004: 0.5147.", table ) assert not any(claim.status == "contradicted" for claim in claims) assert sum(claim.status == "supported" for claim in claims) >= 2 def test_grounding_keeps_unmatched_multi_series_column_claim_unknown() -> None: table = parse_deplot_table( "Entity | 2004\nArmenia | 0.6831\nSingapore |" ) assert table is not None claims = verify_grounded_claims( "Observation: Singapore has 2004: 0.5147.", table ) assert not any(claim.status == "contradicted" for claim in claims) assert any(claim.value == "0.5147" and claim.status == "unknown" for claim in claims) def test_grounding_tolerates_small_deplot_rounding_difference() -> None: table = parse_deplot_table("Year | Value\n2017 | 22.29") assert table is not None claims = verify_grounded_claims("Observation: 2017: 22.26.", table) assert any(claim.value == "22.26" and claim.status == "supported" for claim in claims) def test_conclusion_answer_consistency_handles_text_and_numeric_values() -> None: text = verify_conclusion_answer_consistency("The peak occurs in 2019.", "2019") percent = verify_conclusion_answer_consistency("The result is 39%.", "0.39") mismatch = verify_conclusion_answer_consistency("The minimum is 70.", "72") assert text.status == "consistent" assert percent.status == "consistent" assert mismatch.status == "inconsistent" def test_consistency_normalizes_percent_punctuation_and_keeps_implicit_yes_no_unknown() -> None: percent = verify_conclusion_answer_consistency("The difference is 17.36%.", "17.36") decimal_percent = verify_conclusion_answer_consistency( "The percentage is 0.37.", "37%" ) label = verify_conclusion_answer_consistency( 'The blue graph represents "NET Excellent/good".', "NET Excellent/ good" ) yes_no = verify_conclusion_answer_consistency( "The sum is greater than 100.", "Yes" ) explicit_no = verify_conclusion_answer_consistency( "The sum is not more than XMRig.", "No" ) incorrect_no = verify_conclusion_answer_consistency( "The statement is incorrect.", "No" ) assert percent.status == "consistent" assert decimal_percent.status == "consistent" assert label.status == "consistent" assert yes_no.status == "unknown" assert explicit_no.status == "consistent" assert incorrect_no.status == "consistent" def test_consistency_recognizes_high_confidence_yes_no_conclusion_polarity() -> None: same_yes = verify_conclusion_answer_consistency( "The value of Directly Operated Store is the same as Wholesale.", "Yes" ) indeed_yes = verify_conclusion_answer_consistency( "The value of the Don't know segment is indeed 7%.", "Yes" ) differs_no = verify_conclusion_answer_consistency( "The value for Germany differs from the value for France.", "No" ) unequal_yes = verify_conclusion_answer_consistency( "The two values are unequal.", "Yes" ) assert same_yes.status == "consistent" assert indeed_yes.status == "consistent" assert differs_no.status == "consistent" assert unequal_yes.status == "inconsistent" def test_consistency_finds_answer_before_trailing_dates_and_accepts_rounded_result_and_lists() -> None: dated = verify_conclusion_answer_consistency( "The price moved by 1.93 points from Mar 21 to May 21.", "1.93" ) rounded = verify_conclusion_answer_consistency( "The ratio is approximately 11.83.", "11.83333333" ) labels = verify_conclusion_answer_consistency( "The two countries are Turkey and MEDIAN.", "[Turkey, MEDIAN]" ) assert dated.status == "consistent" assert rounded.status == "consistent" assert labels.status == "consistent" def test_reasoning_checks_extrema_count_and_unknown_conservatively() -> None: table = parse_deplot_table("Year | Value\n2019 | 70\n2020 | 77\n2021 | 40") assert table is not None maximum = verify_reasoning("Comparing the values, the maximum is 77.", table) bad_minimum = verify_reasoning("The minimum value is 70.", table) count = verify_reasoning("Two values are above 50, so the count is 2.", table) unknown = verify_reasoning("The chart clearly supports the conclusion.", table) assert maximum[0].status == "valid" assert bad_minimum[0].status == "invalid" assert count[0].status == "valid" assert unknown[0].status == "unknown" def test_reasoning_does_not_treat_argmax_label_or_difference_as_extrema_result() -> None: table = parse_deplot_table("Year | Value\n2015 | 80\n2017 | 93") assert table is not None argmax = verify_reasoning("The highest value occurs in 2017.", table) difference = verify_reasoning( "Subtract the smallest share 20.8 from the largest share 99.8 to get 79.", table ) assert argmax[0].status == "unknown" assert difference[0].status == "unknown" def test_reasoning_does_not_classify_sum_above_threshold_as_count() -> None: table = parse_deplot_table("Country | Value\nJapan | 24\nEstonia | 42.8") assert table is not None result = verify_reasoning( "Summing the values above 20 gives 24 + 42.8 = 66.8.", table ) assert result[0].status == "unknown" def test_reasoning_extrema_is_unknown_for_multi_series_table() -> None: table = parse_deplot_table( "Year | Blue | Orange\n2019 | 13.16 | 0.65\n2020 | 20.0 | 5.0" ) assert table is not None result = verify_reasoning("The smallest percentage value is 13.16%.", table) assert result[0].status == "unknown" def test_quality_classifier_assigns_q3_q2_q1_and_q0() -> None: table = "Year | Value\n2019 | 70\n2020 | 77" q3 = verify_chart_cot_trajectory( "Goal: Find the maximum.\n" "Observation: 2019: 70 and 2020: 77.\n" "Reasoning: The maximum is 77.\n" "Conclusion: The maximum is 77.\n" "Answer: 77", table, answer_correct=True, ) q2 = verify_chart_cot_trajectory( "Goal: Find the answer.\n" "Observation: Inspect the chart.\n" "Reasoning: Use the visual evidence.\n" "Conclusion: Therefore this is the result.\n" "Answer: 77", table, answer_correct=True, ) q1 = verify_chart_cot_trajectory( "Reasoning: The maximum is 77.\nAnswer: 77", table, answer_correct=True, ) q0 = verify_chart_cot_trajectory( "Goal: Find the maximum.\n" "Observation: 2019: 71.\n" "Reasoning: The maximum is 77.\n" "Conclusion: The maximum is 70.\n" "Answer: 77", table, answer_correct=True, ) assert q3.quality == "Q3" assert q2.quality == "Q2" assert q1.quality == "Q1" assert q0.quality == "Q0" assert "grounding_contradiction" in q0.reason_codes assert "conclusion_answer_inconsistent" in q0.reason_codes def test_quality_grounding_uses_observation_not_derived_conclusion() -> None: result = verify_chart_cot_trajectory( "Goal: Find the sum.\n" "Observation: 2012: 53 and another value is 30.\n" "Reasoning: The sum of 53 and 30 is 83.\n" "Conclusion: The sum in 2012 is 83.\n" "Answer: 83", "Year | First | Second\n2012 | 53 | 30", answer_correct=True, ) assert result.quality == "Q3" assert "grounding_contradiction" not in result.reason_codes def test_quality_q3_requires_an_explicit_supported_observation_claim() -> None: result = verify_chart_cot_trajectory( "Goal: Find the minimum.\n" "Observation: The lowest value is 70.\n" "Reasoning: Comparing the values, the minimum is 70.\n" "Conclusion: The minimum is 70.\n" "Answer: 70", "Year | Value\n2019 | 70\n2020 | 77", answer_correct=True, ) assert result.quality == "Q2" assert "grounding_missing_supported_claim" in result.reason_codes def test_quality_q3_requires_two_concrete_bindings_for_multi_row_comparison() -> None: result = verify_chart_cot_trajectory( "Goal: Find the minimum.\n" "Observation: 2019: 70.\n" "Reasoning: Comparing the values, the minimum is 70.\n" "Conclusion: The minimum is 70.\n" "Answer: 70", "Year | Value\n2019 | 70\n2020 | 77", answer_correct=True, ) assert result.quality == "Q2" assert "grounding_missing_supported_claim" in result.reason_codes def test_quality_accepts_one_concrete_binding_when_recipe_disables_strict_multirow_mode() -> None: result = verify_chart_cot_trajectory( "Goal: Find the minimum.\n" "Observation: The lowest value is 70 in 2019.\n" "Reasoning: Comparing the values, the minimum is 70.\n" "Conclusion: The minimum is 70.\n" "Answer: 70", "Year | Value\n2019 | 70\n2020 | 77", answer_correct=True, require_two_bindings_for_multirow=False, ) assert result.quality == "Q3"