Hugging Face's logo

Spaces:
DineshAI
/
3WPDFjZ1UT
Running

App Files Community
3WPDFjZ1UT
File size: 5,193 Bytes
cac559b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13a5fd7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cac559b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
"""Frozen paper source of truth for arXiv 2603.00039 (CARE).

Every number here was transcribed from the ar5iv HTML render whose SHA-256 is
recorded in SOURCE. Nothing downstream may edit these values; verifiers compare
against them.
"""

from __future__ import annotations

SOURCE = {
    "arxiv_id": "2603.00039",
    "openreview": "3WPDFjZ1UT",
    "url": "https://ar5iv.labs.arxiv.org/html/2603.00039",
    "retrieved_utc": "2026-08-01T04:20:00Z",
    "sha256": "2e733a1609e1dd907dd839b9eed8d9fb7d88549f45d40fafbca7af94ee5e77ea",
    "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/120 Safari/537.36",
    "official_code": "https://github.com/SprocketLab/CARE",
    "official_code_sha": "72f5b29a822d9934d31777c10a5c38369884c9dc",
}

# Table 1: MAE (lower is better) across continuous-scoring datasets.
TABLE1_DATASETS = ["ASSET", "FeedbackQA", "Review-5K", "Summarize", "UltraFeedback", "Yelp"]
TABLE1_MAE = {
    "MV":       [31.153, 0.822, 2.608, 1.417, 0.851, 0.923],
    "AVG":      [33.663, 0.830, 2.274, 1.394, 0.686, 1.037],
    "WS":       [29.073, 0.793, 2.593, 1.364, 0.829, 0.977],
    "UWS":      [33.928, 0.875, 2.602, 1.362, 0.680, 0.987],
    "CARE-SVD": [27.629, 0.730, 1.957, 1.325, 0.623, 0.694],
}
TABLE1_STD = {
    "MV":       [0.000, 0.000, 0.000, 0.000, 0.000, 0.000],
    "AVG":      [0.000, 0.000, 0.000, 0.000, 0.000, 0.000],
    "WS":       [0.436, 0.009, 0.052, 0.007, 0.009, 0.008],
    "UWS":      [0.000, 0.000, 0.000, 0.000, 0.000, 0.000],
    "CARE-SVD": [0.156, 0.002, 0.018, 0.004, 0.006, 0.004],
}

# Table 7 (Appendix E.8): the same six scoring datasets, same MAE metric, under the
# ablation over which recovered latent factor is treated as the quality direction. The
# appendix states "We use the same scoring-task setup as in Table 1", and the "1st
# Factor" row IS the CARE-SVD default heuristic ("the default heuristic (choosing the
# first factor)"). So TABLE7_MAE["1st Factor"] and TABLE1_MAE["CARE-SVD"] are two
# published reports of one quantity, and can be compared without any data.
# "--" marks a dataset with no higher-order factor above the 1e-8 eigenvalue floor.
TABLE7_FACTORS = ["1st Factor", "2nd Factor", "3rd Factor", "4th Factor", "5th Factor"]
TABLE7_MAE = {
    "1st Factor": [27.148, 0.753, 1.950, 1.325, 0.622, 0.694],
    "2nd Factor": [31.757, None, 2.222, None, 0.781, 1.067],
    "3rd Factor": [32.399, None, 3.115, None, 0.939, 0.955],
    "4th Factor": [30.529, None, 2.667, None, 0.996, 1.125],
    "5th Factor": [31.807, None, 3.369, None, 1.040, 1.099],
}
TABLE7_STD = {
    "1st Factor": [0.133, 0.003, 0.006, 0.003, 0.006, 0.005],
    "2nd Factor": [0.100, None, 0.007, None, 0.045, 0.062],
    "3rd Factor": [0.159, None, 0.006, None, 0.010, 0.023],
    "4th Factor": [0.144, None, 0.004, None, 0.005, 0.012],
    "5th Factor": [0.170, None, 0.004, None, 0.062, 0.003],
}

# Table 2: Accuracy (higher is better) across classification / preference datasets.
TABLE2_DATASETS = [
    "Chatbot-Arena", "CivilComments", "PKU-BETTER", "PKU-SAFER", "SHP", "Summarize",
]
TABLE2_ACC = {
    "MV":           [0.517, 0.691, 0.701, 0.698, 0.626, 0.600],
    "AVG":          [0.551, 0.690, 0.726, 0.717, 0.634, 0.683],
    "WS":           [0.543, 0.739, 0.575, 0.570, 0.619, 0.705],
    "UWS":          [0.507, 0.713, 0.703, 0.701, 0.629, 0.713],
    "Dawid-Skene":  [0.546, 0.735, 0.551, 0.548, 0.612, 0.705],
    "GLAD":         [0.510, 0.695, 0.697, 0.671, 0.644, 0.718],
    "MACE":         [0.550, 0.732, 0.734, 0.735, 0.580, 0.706],
    "CARE-SVD":     [0.580, 0.778, 0.691, 0.690, 0.543, 0.695],
    "CARE-Tensor":  [0.564, 0.749, 0.779, 0.731, 0.695, 0.814],
}
TABLE2_BASELINES = ["MV", "AVG", "WS", "UWS", "Dawid-Skene", "GLAD", "MACE"]
TABLE2_CARE = ["CARE-SVD", "CARE-Tensor"]
# Bold (best) cell per column as typeset in the paper.
TABLE2_BOLD = {
    "Chatbot-Arena": "CARE-SVD",
    "CivilComments": "CARE-SVD",
    "PKU-BETTER": "CARE-Tensor",
    "PKU-SAFER": "MACE",
    "SHP": "CARE-Tensor",
    "Summarize": "CARE-Tensor",
}

# Prose quantifiers under test (Section 5.1).
PROSE = {
    "ultrafeedback_mv_reduction_pct": 26.8,
    "avg_relative_improvement_over_AVG_pct": 17.37,
    "avg_relative_improvement_over_MV_pct": 12.75,
    "table2_best_of_n_datasets": (5, 6),
    "table2_care_tensor_leads_on": ["PKU-BETTER", "SHP", "Summarize"],
    "summarize_relative_improvement_pct": 13.4,
    # The claim string circulated with this reproduction quotes
    # "0.814 +- 0.001 vs 0.705 +- 0.000" for the Summarize comparison. 0.705 is the
    # WS / Dawid-Skene entry, not the strongest baseline on that column; the paper's
    # own words are "over the strongest baseline", which is GLAD at 0.718. Both
    # readings are decided explicitly by the verifier.
    "summarize_claimstring_pair": (0.814, 0.705),
}

# Datasets for which the authors released judge-score matrices in the official repo.
# Everything else in Tables 1-2 requires re-running 11-20 LLM judges, which the
# paper itself (Appendix E.2) reports doing on an NVIDIA A100.
RELEASED_JUDGE_OUTPUTS = {
    "table1": ["ASSET"],
    "table2": ["CivilComments", "PKU-BETTER"],
}