contiai / apps /api /tests /test_appendix_dry.py
aliabozahra315's picture
Claude Opus 5
Take a 100-page book from $7.74 to about a tenth of it
b688fe8
Raw History Blame Contribute Delete
15.7 kB
"""The chapter appendix, run end to end with no model behind it.
Seven agents in one crew: an analyst who reads the chapter and decides what
each specialist writes about, then six specialists who each produce one
section. The failure this guards against is the one that has cost most here -
a section the renderer cannot find renders nothing at all, and the run reports
success with a document missing a third of its content.
So the run is driven through the engine, and the finished Word document is
checked for the text each of the six sections contributed.
"""
from __future__ import annotations
import base64
import io
import uuid
from typing import Any
import pytest
import pytest_asyncio
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from app.db.models import (
Artifact,
Base,
Project,
Run,
RunStatus,
StepRun,
StepStatus,
User,
UserRole,
UserStatus,
)
from app.pipelines.engine import execute_run
from app.pipelines.registry import load_all
CHAPTER_TITLE = "القيادة التشاركية في البيئة المدرسية"
CHAPTER_TEXT = (
"القيادة التشاركية ليست تنازلًا عن الصلاحية، بل توزيع للمسؤولية. "
"يبدأ القائد بتحديد القرارات التي تحتمل المشاركة وتلك التي لا تحتملها. "
) * 12
def answers() -> dict[str, Any]:
"""One minimally valid answer per task, keyed by its output schema."""
return {
"ChapterAnalysis": {
"main_themes": ["توزيع المسؤولية", "حدود المشاركة"],
"key_concepts": ["القيادة التشاركية", "نطاق القرار"],
"tools_opportunities": ["مصفوفة القرارات", "خريطة الأدوار"],
"reflection_angles": ["أين أشارك؟", "متى أحتفظ بالقرار؟"],
"scenario_angles": ["اجتماع قسم", "أزمة جدول"],
"case_study_angle": "قائد يوزّع قرارًا لا يحتمل التوزيع",
"checklist_angle": "ممارسات المشاركة اليومية",
"indicators_angles": ["مؤشرات تنظيمية", "مؤشرات إنسانية"],
},
"ToolsSection": {"tools": [{
"name": "مصفوفة نطاق القرار",
"purpose": "الفصل بين ما يُشارَك فيه وما لا يُشارَك.",
"steps": ["راجع قرارات الأسبوع", "حدد أثر كل قرار", "وزّع ما يحتمل"],
"example": "قائد مدرسة أعاد توزيع جدول الإشراف بعد جلسة قسم.",
"when_to_use": ["قبل بداية الفصل", "عند تغيير الجدول"],
"reflection_question": "أي قرار احتفظت به دون سبب؟",
"common_mistakes": ["توزيع قرار لا يحتمل التأخير"],
"success_indicators": ["انخفاض القرارات المرتجعة"],
}]},
"ReflectionCardsSection": {"cards": [{
"title": "أين أشارك فعلًا؟",
"questions": ["ما آخر قرار شاركت فيه فريقك؟",
"ما الذي منعك من المشاركة؟",
"ماذا كسبت حين شاركت؟"],
"closing_message": "المشاركة قرار، لا مزاج.",
}]},
"ScenariosSection": {"scenarios": [{
"title": "اجتماع القسم",
"description": "طرح القائد جدولًا جاهزًا فصمت الفريق.",
"discussion_questions": ["لماذا صمت الفريق؟", "ما البديل؟"],
}]},
"CaseStudySection": {"case_studies": [{
"title": "قرار وُزّع في غير موضعه",
"description": "قائد وزّع قرار الميزانية على لجنة بلا صلاحية.",
"leader_tasks": ["شخّص سبب التعطّل", "حدد نطاق القرار",
"اقترح إجراءً وقِس أثره"],
}]},
"ChecklistSection": {
"title": "هل توزّع المسؤولية فعلًا؟",
"items": ["أراجع قراراتي أسبوعيًا", "أحدد نطاق كل قرار",
"أشرك الفريق مبكرًا", "أعلن حدود المشاركة",
"أتابع أثر ما وزّعته", "أراجع ما احتفظت به"],
"interpretation": [
{"range_label": "5-6 (نعم)", "meaning": "ممارسة راسخة."},
{"range_label": "3-4 (نعم)", "meaning": "ممارسة متذبذبة."},
{"range_label": "0-2 (نعم)", "meaning": "المشاركة شكلية."},
],
},
"IndicatorsSection": {
"title": "مؤشرات توزيع المسؤولية",
"categories": [
{"category_name": "مؤشرات تنظيمية",
"indicators": ["عدد القرارات الموزَّعة", "زمن إغلاق القرار"]},
{"category_name": "مؤشرات إنسانية",
"indicators": ["مشاركة الفريق في الاجتماع"]},
],
"interpretation": [
{"range_label": "تحسّن متزامن", "meaning": "التوزيع أثمر."},
{"range_label": "تحسّن أحادي", "meaning": "التوزيع شكلي."},
],
"when_to_use": "كل فصل دراسي.",
},
}
class StubOutput:
"""What CrewAI leaves on a task: `.output.pydantic`."""
def __init__(self, model, payload):
self.pydantic = model(**payload)
@pytest_asyncio.fixture
async def engine():
eng = create_async_engine("sqlite+aiosqlite:///:memory:", future=True)
async with eng.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
yield eng
await eng.dispose()
@pytest_asyncio.fixture
async def db(engine) -> AsyncSession:
factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
async with factory() as session:
yield session
@pytest_asyncio.fixture
async def appendix_run(db, engine, monkeypatch) -> Run:
load_all()
user = User(id=uuid.uuid4(), email="t@example.com", role=UserRole.user,
status=UserStatus.active, allowed_task_types=["appendix"])
project = Project(owner_id=user.id, kind="appendix", title="ملحقات")
db.add_all([user, project])
await db.commit()
run = Run(
project_id=project.id, pipeline_key="appendix", created_by=user.id,
status=RunStatus.queued,
params={"variant": "from_text", "brief": {
"chapter_title": CHAPTER_TITLE,
"chapter_text": CHAPTER_TEXT,
}},
)
db.add(run)
await db.commit()
factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
monkeypatch.setattr("app.db.session.SessionLocal", factory)
# Answer every task from its own output schema, and leave the answer where
# the runner looks for it: `task.output.pydantic`.
from crewai import Crew
replies = answers()
async def fake_kickoff(self, inputs=None):
for task in self.tasks:
model = getattr(task, "output_pydantic", None)
if model is not None:
task.output = StubOutput(model, replies[model.__name__])
return None
monkeypatch.setattr(Crew, "kickoff_async", fake_kickoff)
return run
# ---------------------------------------------------------------------------
async def test_the_run_finishes_and_produces_a_word_document(db, appendix_run):
status = await execute_run(db, appendix_run.id)
rows = {
r.step_key: r for r in
(await db.scalars(select(StepRun).where(StepRun.run_id == appendix_run.id))).all()
}
failed = [f"{k}: {(r.error or '')[:150]}"
for k, r in rows.items() if r.status is StepStatus.failed]
assert not failed, "steps failed: " + " | ".join(failed)
assert status is RunStatus.succeeded
artifact = await db.scalar(
select(Artifact).where(Artifact.run_id == appendix_run.id)
)
assert artifact is not None, "the run finished without producing a file"
assert artifact.fmt == "docx"
assert base64.b64decode(artifact.content)[:2] == b"PK", "not a Word document"
async def test_all_six_sections_reach_the_document(db, appendix_run):
"""A section the renderer cannot find renders nothing and raises nothing.
Each is checked by its own text."""
from docx import Document
await execute_run(db, appendix_run.id)
artifact = await db.scalar(
select(Artifact).where(Artifact.run_id == appendix_run.id)
)
document = Document(io.BytesIO(base64.b64decode(artifact.content)))
text = "\n".join(p.text for p in document.paragraphs)
for table in document.tables:
for row in table.rows:
text += "\n" + " ".join(c.text for c in row.cells)
for section, probe in [
("الأدوات التطبيقية", "مصفوفة نطاق القرار"),
("بطاقات التأمل", "أين أشارك فعلًا؟"),
("السيناريوهات", "اجتماع القسم"),
("دراسة الحالة", "قرار وُزّع في غير موضعه"),
("قائمة التحقق", "هل توزّع المسؤولية فعلًا؟"),
("المؤشرات", "مؤشرات توزيع المسؤولية"),
]:
assert probe in text, f"«{section}» لم يصل المستند"
assert CHAPTER_TITLE in text, "the chapter title never reached the document"
async def test_the_analysis_is_kept_not_discarded(db, appendix_run):
"""It is the only record of what the distributor told the six to write
about. The notebook writes it to a second file "للمراجعة فقط"."""
await execute_run(db, appendix_run.id)
row = await db.scalar(
select(StepRun).where(StepRun.run_id == appendix_run.id,
StepRun.step_key == "appendix.generate")
)
analysis = (row.output or {}).get("analysis") or {}
assert analysis.get("main_themes"), "the distribution was thrown away"
assert analysis.get("case_study_angle")
async def test_a_chapter_too_short_is_refused_before_anything_runs(db, appendix_run):
"""The appendix is built from the chapter text alone. Two sentences give
the analyst nothing to distribute, and six specialists would invent."""
from pydantic import ValidationError
from app.pipelines.appendix.steps import ChapterIn
with pytest.raises(ValidationError):
ChapterIn(chapter_title="T", chapter_text="قصير جدًا")
def test_the_chapter_field_publishes_its_word_budget():
"""The form counts words and warns past the budget, and it learns the
budget from the schema alone - there is no per-task code in the renderer.
So the hint has to survive in the published schema or the warning silently
stops appearing with nothing failing."""
from app.pipelines.appendix.steps import CHAPTER_MAX_WORDS, ChapterIn
field = ChapterIn.model_json_schema()["properties"]["chapter_text"]
assert field["maxWords"] == CHAPTER_MAX_WORDS
# Advisory, not a limit: the character ceiling has to sit well above the
# word budget, otherwise the browser truncates the paste before the author
# ever sees the warning and the advice becomes a silent edit.
assert field["maxLength"] > CHAPTER_MAX_WORDS * 8
def test_the_card_offers_the_format_the_export_writes():
from app.api.v1.task_types import DEFAULTS
assert DEFAULTS["appendix"]["outputs"] == ["DOCX"]
def test_every_renderer_reads_a_key_its_schema_produces():
"""Six sections, six renderers. A renamed key is invisible until a run
finishes and the section is silently absent."""
import inspect
import re
import app.schemas.appendix as schemas
from app.services.appendix import renderer
for fn, model in [
("render_tools_section", "ToolsSection"),
("render_reflection_cards_section", "ReflectionCardsSection"),
("render_scenarios_section", "ScenariosSection"),
("render_case_study_section", "CaseStudySection"),
("render_checklist_section", "ChecklistSection"),
("render_indicators_section", "IndicatorsSection"),
]:
source = inspect.getsource(getattr(renderer, fn))
# The top-level key each renderer pulls off the section it is given.
top = set(re.findall(r"\w+_data\.get\(\"(\w+)\"", source))
produced = set(getattr(schemas, model).model_fields)
unknown = top - produced
assert not unknown, f"{fn} reads {sorted(unknown)}, not in {model}"
async def test_the_upload_route_reaches_the_same_generator(db, engine, monkeypatch):
"""Two ways in, one shape after them. The pasted route and the uploaded one
both end as a string handed to the same crew - if they diverged, the second
would be a feature nobody had run."""
from crewai import Crew
load_all()
user = User(id=uuid.uuid4(), email="u@example.com", role=UserRole.user,
status=UserStatus.active, allowed_task_types=["appendix"])
project = Project(owner_id=user.id, kind="appendix", title="ملحقات")
db.add_all([user, project])
await db.commit()
run = Run(
project_id=project.id, pipeline_key="appendix", created_by=user.id,
status=RunStatus.queued,
params={"variant": "from_file", "brief": {
"upload_ids": ["11111111-1111-1111-1111-111111111111"],
"chapter_title": CHAPTER_TITLE,
}},
)
db.add(run)
await db.commit()
factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
monkeypatch.setattr("app.db.session.SessionLocal", factory)
# The file reader is exercised on its own elsewhere; here it stands in for
# the upload so the rest of the route is what gets tested.
async def fake_read_source(*, upload_ids, pasted_text):
assert upload_ids, "the upload ids never reached the reader"
return CHAPTER_TEXT
monkeypatch.setattr("app.services.blueprint.text.read_source", fake_read_source)
replies = answers()
async def fake_kickoff(self, inputs=None):
for task in self.tasks:
model = getattr(task, "output_pydantic", None)
if model is not None:
task.output = StubOutput(model, replies[model.__name__])
return None
monkeypatch.setattr(Crew, "kickoff_async", fake_kickoff)
status = await execute_run(db, run.id)
assert status is RunStatus.succeeded
artifact = await db.scalar(select(Artifact).where(Artifact.run_id == run.id))
assert artifact is not None, "the upload route produced no file"
assert base64.b64decode(artifact.content)[:2] == b"PK"
assert CHAPTER_TITLE in (artifact.title or "")
def test_both_routes_read_the_chapter_through_one_reader():
"""Pasted text and an uploaded file converge before anything branches on
them, so nothing downstream knows or cares how the chapter arrived."""
import inspect
from app.pipelines.appendix.steps import GenerateStep
source = inspect.getsource(GenerateStep.run)
assert source.count("read_source(") == 1, "the two routes read separately"
assert "upload_ids" in source and "chapter_text" in source