| from __future__ import annotations |
|
|
| import hashlib |
| import json |
|
|
| from ..reward_models import ( |
| ChecklistItem, |
| PresentBenchEvalSpec, |
| TaskSpec, |
| to_serializable, |
| ) |
|
|
| SPEC_VERSION = "1.0" |
|
|
| DEFAULT_PRESENTBENCH_SCORING_CONFIG = { |
| "dimension_weights": { |
| "fundamentals": 0.20, |
| "visual_layout": 0.20, |
| "completeness": 0.20, |
| "correctness": 0.20, |
| "fidelity": 0.20, |
| }, |
| "aesthetic_weight": 0.15, |
| "hard_caps": { |
| "blank_title_only_ratio_threshold": 0.4, |
| }, |
| "soft_penalties": { |
| "slide_count_violation": 0.02, |
| "overlap": 0.01, |
| "tiny_text": 0.01, |
| "redundancy": 0.03, |
| "wrong_slot_behavior": 0.02, |
| "staleness": 0.30, |
| }, |
| } |
|
|
|
|
| def generate_checklist(task_spec: TaskSpec) -> list[ChecklistItem]: |
| checklist: list[ChecklistItem] = [ |
| ChecklistItem( |
| item_id="fundamentals_slide_count", |
| dimension="fundamentals", |
| prompt_text="Is the slide count within the requested range?", |
| item_kind="slide_count_range", |
| ), |
| ChecklistItem( |
| item_id="fundamentals_theme", |
| dimension="fundamentals", |
| prompt_text="Does the deck maintain a central theme aligned with the prompt?", |
| item_kind="theme_alignment", |
| relevant_sections=task_spec.required_sections, |
| ), |
| ChecklistItem( |
| item_id="fundamentals_audience", |
| dimension="fundamentals", |
| prompt_text="Is the deck appropriate for the intended audience and tone?", |
| item_kind="audience_tone", |
| ), |
| ChecklistItem( |
| item_id="visual_readability", |
| dimension="visual_layout", |
| prompt_text="Is the text readable without tiny fonts?", |
| item_kind="readable_text", |
| ), |
| ChecklistItem( |
| item_id="visual_overlap", |
| dimension="visual_layout", |
| prompt_text="Are there no major overlaps or clipping risks?", |
| item_kind="no_major_overlap", |
| ), |
| ChecklistItem( |
| item_id="visual_consistency", |
| dimension="visual_layout", |
| prompt_text="Is the visual design reasonably consistent across slides?", |
| item_kind="design_consistency", |
| ), |
| ] |
|
|
| for index, section in enumerate(task_spec.required_sections, start=1): |
| checklist.append( |
| ChecklistItem( |
| item_id=f"completeness_section_{index:02d}", |
| dimension="completeness", |
| prompt_text=f"Does the deck include a clear '{section}' section?", |
| item_kind="required_section", |
| relevant_sections=[section], |
| ) |
| ) |
|
|
| for index, point in enumerate(task_spec.required_points, start=1): |
| source_refs = [ |
| fact["ref"] |
| for fact in task_spec.metadata.get("source_facts", []) |
| if point in fact["text"] |
| ][:2] |
| checklist.append( |
| ChecklistItem( |
| item_id=f"completeness_point_{index:02d}", |
| dimension="completeness", |
| prompt_text=f"Does the deck cover this required point: {point}?", |
| item_kind="required_point", |
| source_refs=source_refs, |
| ) |
| ) |
| checklist.append( |
| ChecklistItem( |
| item_id=f"correctness_point_{index:02d}", |
| dimension="correctness", |
| prompt_text=f"Is this required point stated correctly: {point}?", |
| item_kind="correct_required_point", |
| source_refs=source_refs, |
| ) |
| ) |
|
|
| if task_spec.required_slides: |
| for slide in task_spec.required_slides: |
| checklist.append( |
| ChecklistItem( |
| item_id=f"fidelity_slide_{slide.slide_index:02d}", |
| dimension="fidelity", |
| prompt_text=f"Is all content on Slide {slide.slide_index} supported by the source pack?", |
| item_kind="slide_fidelity", |
| required_slide_scope=[slide.slide_index], |
| relevant_sections=[slide.slide_role], |
| ) |
| ) |
| else: |
| checklist.append( |
| ChecklistItem( |
| item_id="fidelity_deck", |
| dimension="fidelity", |
| prompt_text="Is all deck content supported by the source pack?", |
| item_kind="deck_fidelity", |
| ) |
| ) |
|
|
| return checklist |
|
|
|
|
| def generate_slide_checklists(task_spec: TaskSpec) -> dict[int, list[ChecklistItem]]: |
| slide_checklists: dict[int, list[ChecklistItem]] = {} |
| for slide in task_spec.required_slides or []: |
| items = [ |
| ChecklistItem( |
| item_id=f"slide_{slide.slide_index:02d}_prompt_alignment", |
| dimension="prompt_alignment", |
| prompt_text=f"Does Slide {slide.slide_index} match the intended role '{slide.slide_role}'?", |
| item_kind="slide_role_match", |
| required_slide_scope=[slide.slide_index], |
| relevant_sections=[slide.slide_role], |
| ), |
| ChecklistItem( |
| item_id=f"slide_{slide.slide_index:02d}_title_alignment", |
| dimension="prompt_alignment", |
| prompt_text=f"Is Slide {slide.slide_index} title aligned with '{slide.title_hint or slide.instructions}'?", |
| item_kind="slide_title_alignment", |
| required_slide_scope=[slide.slide_index], |
| relevant_sections=[slide.slide_role], |
| ), |
| ChecklistItem( |
| item_id=f"slide_{slide.slide_index:02d}_fidelity", |
| dimension="local_fidelity", |
| prompt_text=f"Is all content on Slide {slide.slide_index} supported by the source pack?", |
| item_kind="slide_fidelity", |
| required_slide_scope=[slide.slide_index], |
| relevant_sections=[slide.slide_role], |
| ), |
| ChecklistItem( |
| item_id=f"slide_{slide.slide_index:02d}_usability", |
| dimension="local_usability", |
| prompt_text=f"Is Slide {slide.slide_index} readable and free of major clutter?", |
| item_kind="slide_readability", |
| required_slide_scope=[slide.slide_index], |
| relevant_sections=[slide.slide_role], |
| ), |
| ] |
| for point_index, point in enumerate(slide.required_points, start=1): |
| items.append( |
| ChecklistItem( |
| item_id=f"slide_{slide.slide_index:02d}_point_{point_index:02d}", |
| dimension="local_completeness", |
| prompt_text=f"Does Slide {slide.slide_index} cover this point: {point}?", |
| item_kind="slide_required_point", |
| required_slide_scope=[slide.slide_index], |
| relevant_sections=[slide.slide_role], |
| ) |
| ) |
| for exact_index, exact_value in enumerate(slide.required_exact_values, start=1): |
| items.append( |
| ChecklistItem( |
| item_id=f"slide_{slide.slide_index:02d}_exact_{exact_index:02d}", |
| dimension="local_correctness", |
| prompt_text=f"Does Slide {slide.slide_index} include the exact value '{exact_value}' correctly?", |
| item_kind="slide_exact_value", |
| required_slide_scope=[slide.slide_index], |
| relevant_sections=[slide.slide_role], |
| ) |
| ) |
| if slide.required_shape_kinds: |
| items.append( |
| ChecklistItem( |
| item_id=f"slide_{slide.slide_index:02d}_visual_kind", |
| dimension="local_completeness", |
| prompt_text=f"Does Slide {slide.slide_index} include the expected supported visual forms?", |
| item_kind="slide_required_visual", |
| required_slide_scope=[slide.slide_index], |
| relevant_sections=[slide.slide_role], |
| ) |
| ) |
| slide_checklists[slide.slide_index] = items |
| return slide_checklists |
|
|
|
|
| def build_presentbench_eval_spec( |
| task_spec: TaskSpec, *, mode: str = "eval" |
| ) -> PresentBenchEvalSpec: |
| checklist = generate_checklist(task_spec) |
| slide_checklists = generate_slide_checklists(task_spec) |
| payload = { |
| "task_spec": to_serializable(task_spec), |
| "checklist": to_serializable(checklist), |
| "slide_checklists": to_serializable(slide_checklists), |
| "scoring_config": DEFAULT_PRESENTBENCH_SCORING_CONFIG, |
| "spec_version": SPEC_VERSION, |
| } |
| spec_hash = hashlib.sha256( |
| json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") |
| ).hexdigest() |
| return PresentBenchEvalSpec( |
| task_spec=task_spec, |
| checklist=checklist, |
| slide_checklists=slide_checklists, |
| scoring_config={**DEFAULT_PRESENTBENCH_SCORING_CONFIG, "mode": mode}, |
| spec_version=SPEC_VERSION, |
| spec_hash=spec_hash, |
| ) |
|
|
|
|
| __all__ = [ |
| "DEFAULT_PRESENTBENCH_SCORING_CONFIG", |
| "SPEC_VERSION", |
| "build_presentbench_eval_spec", |
| "generate_checklist", |
| "generate_slide_checklists", |
| ] |
|
|
