from __future__ import annotations from typing import Any from .metrics import ( compute_aesthetics_scores, compute_presentation_diagnostics, compute_slide_staleness_penalty, mean_scores_by_dimension, redundancy_score, score_checklist_item, score_generic_slide_checklist_items, score_slide_checklist_item, ) from ..reward_metrics import ( clamp_reward, compute_overlap_ratio, is_blank_or_title_only, ) from ..reward_models import ( IntermediateSlideRewardResult, PresentBenchEvalSpec, PresentBenchScoreResult, ExtractedPresentation, ExtractedSlide, TaskSpec, ) def score_presentbench( task_spec: TaskSpec, presentation_extraction: ExtractedPresentation, eval_spec: PresentBenchEvalSpec, *, aesthetics_service: Any | None = None, ) -> PresentBenchScoreResult: diagnostics = compute_presentation_diagnostics(presentation_extraction, task_spec) checklist_results = [ score_checklist_item(item, presentation_extraction, task_spec, diagnostics) for item in eval_spec.checklist ] dimension_scores = mean_scores_by_dimension(checklist_results) aesthetics_scores = ( aesthetics_service.score_presentation(presentation_extraction) if aesthetics_service and hasattr(aesthetics_service, "score_presentation") else compute_aesthetics_scores(presentation_extraction) ) s_fund = dimension_scores.get("fundamentals", 0.0) s_visual = dimension_scores.get("visual_layout", 0.0) s_complete = dimension_scores.get("completeness", 0.0) s_correct = dimension_scores.get("correctness", 0.0) s_fidelity = dimension_scores.get("fidelity", 0.0) dimension_weights = eval_spec.scoring_config["dimension_weights"] r_pb = ( dimension_weights["fundamentals"] * s_fund + dimension_weights["visual_layout"] * s_visual + dimension_weights["completeness"] * s_complete + dimension_weights["correctness"] * s_correct + dimension_weights["fidelity"] * s_fidelity ) c_fidelity_critical = ( 0.5 if any( r["dimension"] == "fidelity" and r["verdict"] == "no" for r in checklist_results ) else 1.0 ) blankness_threshold = eval_spec.scoring_config["hard_caps"][ "blank_title_only_ratio_threshold" ] c_blankness = ( 0.6 if diagnostics["blank_title_only_ratio"] > blankness_threshold else 1.0 ) c_hard = min(c_fidelity_critical, c_blankness) penalty_config = eval_spec.scoring_config["soft_penalties"] soft_penalties = { "slide_count_violation": penalty_config["slide_count_violation"] * diagnostics["slide_count_violation"], "overlap": penalty_config["overlap"] * diagnostics["max_overlap_ratio"], "tiny_text": penalty_config["tiny_text"] * ( 1.0 if diagnostics["min_font_size_pt"] is not None and diagnostics["min_font_size_pt"] < 10 else 0.0 ), "staleness": penalty_config["staleness"] * diagnostics["mean_staleness_penalty"], } p_soft = sum(soft_penalties.values()) aesthetic_weight = eval_spec.scoring_config.get("aesthetic_weight", 0.15) reward_total = clamp_reward( c_hard * ( (1.0 - aesthetic_weight) * r_pb + aesthetic_weight * aesthetics_scores["aesthetic"] ) - p_soft ) return PresentBenchScoreResult( reward_total=reward_total, reward_breakdown={ "R_pb": reward_total, "S_fundamentals": s_fund, "S_visual_layout": s_visual, "S_completeness": s_complete, "S_correctness": s_correct, "S_fidelity": s_fidelity, "S_aesthetic": aesthetics_scores["aesthetic"], "S_harmony": aesthetics_scores["harmony"], "S_engagement": aesthetics_scores["engagement"], "S_usability": aesthetics_scores["usability"], "S_rhythm": aesthetics_scores["rhythm"], }, hard_caps={ "C_fidelity_critical": c_fidelity_critical, "C_blankness": c_blankness, "C_hard": c_hard, }, soft_penalties=soft_penalties, checklist_results=checklist_results, aesthetics_results=aesthetics_scores, metadata={ "task_id": eval_spec.task_spec.task_id, "item_count": len(eval_spec.checklist), "slide_count": presentation_extraction.slide_count, "spec_hash": eval_spec.spec_hash, }, ) def score_presentbench_slide( task_spec: TaskSpec, eval_spec: PresentBenchEvalSpec, slide_index: int, slide_extraction: ExtractedSlide, *, previous_slide_extractions: list[ExtractedSlide] | None = None, use_mllm: bool = False, mode: str = "eval", ) -> IntermediateSlideRewardResult: target_slides = { slide.slide_index: slide for slide in task_spec.required_slides or [] } target_slide = target_slides.get(slide_index) target_mode = "required_slide" if target_slide is not None else "fallback_generic" if target_slide is None: checklist_results = score_generic_slide_checklist_items( slide_extraction, task_spec ) else: slide_checklist = eval_spec.slide_checklists.get(slide_index, []) checklist_results = [ score_slide_checklist_item( item, slide_extraction, target_slide.slide_role, target_slide.title_hint, target_slide.required_shape_kinds, task_spec, ) for item in slide_checklist ] dimension_scores = mean_scores_by_dimension(checklist_results) s_prompt_alignment = dimension_scores.get("prompt_alignment", 0.0) s_local_completeness = dimension_scores.get("local_completeness", 0.0) s_local_correctness = dimension_scores.get("local_correctness", 0.0) s_local_fidelity = dimension_scores.get("local_fidelity", 0.0) s_local_usability = dimension_scores.get("local_usability", 0.0) c_slide_fidelity_critical = ( 0.5 if ( target_slide is not None and s_local_fidelity < 1.0 and target_slide.required_exact_values ) else 1.0 ) c_slide_blankness = ( 0.4 if is_blank_or_title_only(slide_extraction) and ( (target_slide is not None and target_slide.required_points) or target_mode == "fallback_generic" ) else 1.0 ) c_slide_hard = min(c_slide_fidelity_critical, c_slide_blankness) penalty_config = eval_spec.scoring_config["soft_penalties"] overlap = compute_overlap_ratio(slide_extraction) min_font = slide_extraction.text_metrics.get("min_font_size_pt") redundancy = redundancy_score(slide_extraction, previous_slide_extractions) staleness = compute_slide_staleness_penalty( slide_extraction, role=target_slide.slide_role if target_slide is not None else None, ) wrong_slot_behavior = ( 0.0 if target_mode == "fallback_generic" or s_prompt_alignment >= 1.0 else 1.0 ) soft_penalties = { "redundancy": penalty_config["redundancy"] * redundancy, "wrong_slot_behavior": penalty_config["wrong_slot_behavior"] * wrong_slot_behavior, "tiny_text": penalty_config["tiny_text"] * (1.0 if min_font is not None and min_font < 10 else 0.0), "overlap": penalty_config["overlap"] * overlap, "staleness": penalty_config["staleness"] * float(staleness["penalty"]), } p_slide_soft = sum(soft_penalties.values()) reward_total = clamp_reward( c_slide_hard * ( 0.35 * s_prompt_alignment + 0.20 * s_local_completeness + 0.15 * s_local_correctness + 0.20 * s_local_fidelity + 0.10 * s_local_usability ) - p_slide_soft ) return IntermediateSlideRewardResult( slide_index=slide_index, reward_total=reward_total, reward_breakdown={ "R_slide": reward_total, "S_prompt_alignment": s_prompt_alignment, "S_local_completeness": s_local_completeness, "S_local_correctness": s_local_correctness, "S_local_fidelity": s_local_fidelity, "S_local_usability": s_local_usability, }, hard_caps={ "C_slide_fidelity_critical": c_slide_fidelity_critical, "C_slide_blankness": c_slide_blankness, "C_slide_hard": c_slide_hard, }, soft_penalties=soft_penalties, checklist_results=checklist_results, metadata={ "slide_index": slide_index, "slide_id": slide_extraction.slide_id, "target_mode": target_mode, "target_slide_role": target_slide.slide_role if target_slide else None, "target_title_hint": target_slide.title_hint if target_slide else None, "required_points": target_slide.required_points if target_slide else [], "judge_call_count": 0, "used_previous_slide_context": previous_slide_extractions is not None, "staleness": staleness, "spec_hash": eval_spec.spec_hash, "mode": mode, }, ) __all__ = ["score_presentbench", "score_presentbench_slide"]