| from crewai import Agent, Task, Crew, Process, LLM |
| import os |
| from modules import llm_oss |
| from crewai_tools import SerperDevTool |
| from schemas import SearchEngineOutput |
| from tools import URLValidatorTool |
|
|
| |
| search_tool = SerperDevTool() |
| url_validate_tool = URLValidatorTool(name="url_validator") |
|
|
| |
| search_engine_agent = Agent( |
| role="Search and Validation agent", |
| goal="\n".join( |
| [ |
| "Search for diverse, high-quality, and reliable information related to {topic} using the suggested search queries.", |
| "Prioritize **scholarly and educational resources**: academic journals, books, research articles, and trusted institutional websites and our trusted sites we know: {TRUSTED_SITES}.", |
| "Include Arabic sources when contextually valuable, but prioritize English academic sources for broader coverage.", |
| "Ensure all results are concise, relevant, and aligned with the outline headings to directly support course design and educational content.", |
| "For each query, return at most {no_links} results with structured metadata (title, url, content summary, score, search_query).", |
| "Validate URLs before retrieving them and if it is not valid dont retieve", |
| ] |
| ), |
| backstory=( |
| "This agent acts like a scientific researcher and educational content curator." |
| "It executes the generated queries and gathers high-value, trustworthy information." |
| "The agent is trained to filter out irrelevant, commercial, or low-quality content, ensuring that " |
| "only academic and pedagogically useful resources are kept." |
| "for building interactive e-learning content are retrieved." |
| "It balances global knowledge with local cultural enrichment by including both English and Arabic sources." |
| "Uses Serper search and validates each found URL." |
| ), |
| verbose=True, |
| llm=llm_oss, |
| tools=[search_tool, url_validate_tool], |
| allow_delegation=False, |
| ) |
|
|
| |
| search_engine_task = Task( |
| name="Academic Web Search", |
| description="\n".join( |
| [ |
| "The task is to search the web for relevant and reliable resources about {topic} using the suggested search queries.", |
| "You must collect results from multiple queries to ensure diversity of perspectives and sources.", |
| "Here are the queries you can use to search:", |
| "{queries}", |
| "use unit title: {unit_title} and subtopic title: {subtopic_title} to complete the output and search correctly for each query.", |
| "For each query, retrieve at least 1 link and a maximum of {no_links} links only.", |
| "Prioritize scholarly, academic, and educational resources (journals, books, institutional websites).", |
| "Filter out irrelevant, suspicious, or low-quality results (e.g., personal blogs, spam, commercial ads).", |
| "Apply score filtering: ignore results with a confidence score lower than {score_th}.", |
| "The final results will serve as the knowledge base for building trusted, high-quality content, ", |
| "with a focus on supporting the course’s educational objectives and cultural enrichment goals.", |
| "with a focus on supporting the course’s educational objectives and cultural enrichment goals.", |
| "Get sites that newer than 2021.", |
| "Validate URLs before retrieving them and if it is not valid dont retieve", |
|
|
| ] |
| ), |
| expected_output=( |
| "Return ONLY a valid UTF-8 safe Python dictionary (no markdown, no code blocks). " |
| "It must be directly parsable with ast.literal_eval. Use this format:\n\n" |
| "{\n" |
| ' "unit_title": "...",\n' |
| ' "subtopic_title": "...",\n' |
| ' "query": "...",\n' |
| ' "results": [\n' |
| ' {"url": "...", "title": "...", "content": "...", "score": 0.91}\n' |
| " ]\n" |
| "}" |
| ), |
| output_json=SearchEngineOutput, |
| agent=search_engine_agent, |
| output_file="search_results.json", |
| ) |
|
|
