TrainingCourseGen / pipelines /generate_module_content.py
aimanzoqi's picture
saving module content and creating endpoint for module creation
4983616
Raw History Blame Contribute Delete
4.65 kB
from haystack import Pipeline
from haystack.components.builders import PromptBuilder
from haystack.components.generators import HuggingFaceAPIGenerator
from haystack.utils import Secret
from pathlib import Path
from haystack.core.errors import PipelineDrawingError
# Template for generating detailed module content
content_template = """
You are an instructional designer. Create a detailed, educational Markdown module titled "{{name}}" that takes {{duration}} to read.
- Use **200 WPM** for general topics or **150 WPM** for technical ones.
- Target word count: {{word_count}} ±5%.
- Output valid **Markdown only** using the structure below.
- Do **not** include code fences, extra commentary, or repeat "### End".
### Structure:
# {{name}}
**Estimated Reading Time: {{duration}}**
## Introduction
- Brief intro to the topic.
## Key Concepts
{{formatted_subtopics}}
## Explanation
- Detailed explanation using paragraphs, bullets, and examples.
## Examples
- Include at least one relevant example.
## Summary
- Key takeaways.
### End
"""
# Create pipeline components
builder = PromptBuilder(template=content_template)
generator = HuggingFaceAPIGenerator(
api_type="serverless_inference_api",
api_params={"model": "meta-llama/Llama-3.2-3B-Instruct"},
token=Secret.from_env_var("HF_API_TOKEN"),
generation_kwargs={"max_new_tokens": 5000, "stop": ["<stop>"]},
)
pipeline = Pipeline()
pipeline.add_component("prompt_builder", builder)
pipeline.add_component("llm", generator)
pipeline.connect("prompt_builder", "llm")
# Save the pipeline as a YAML file and attempt to draw a diagram
with open("./pipelines/generate_module_content.yml", "w") as file:
pipeline.dump(file)
try:
pipeline.draw(path=Path("generate_module_content_pipeline.png"))
except PipelineDrawingError:
print("⚠️ Mermaid diagram could not be generated (likely offline). Skipping pipeline drawing.")
except Exception as e:
print(f"⚠️ Unexpected error while drawing pipeline: {e}")
# Ensure output directory exists
Path("./modules").mkdir(parents=True, exist_ok=True)
# Detect technical modules
def is_technical_topic(module_name, subtopics):
keywords = ["API", "database", "code", "technical", "machine learning", "AI", "infrastructure", "engineering", "system", "security", "algorithm"]
combined_text = f"{module_name} {' '.join(subtopics)}".lower()
return any(keyword in combined_text for keyword in keywords)
# Generate content for all modules in the outline
def generate_content_for_outline(outline):
content_results = {}
for module in outline["modules"]:
is_technical = is_technical_topic(module['name'], module['subtopics'])
reading_speed = 150 if is_technical else 200
minutes = int(module['duration'].split()[0])
word_count = int(minutes * reading_speed)
formatted_subtopics = "\n".join([f"- {sub}" for sub in module["subtopics"]])
print(f"Generating '{module['name']}' at {reading_speed} WPM → {word_count} words")
try:
result = pipeline.run({
"prompt_builder": {
"name": module['name'],
"duration": module['duration'],
"word_count": word_count,
"formatted_subtopics": formatted_subtopics,
}
})
markdown_output = result["llm"]["replies"][0]
content_results[module["name"]] = markdown_output
# Save to .md file
filename = module["name"].replace(" ", "_").lower()
with open(f"./modules/{filename}.md", "w", encoding="utf-8") as f:
f.write(markdown_output)
except Exception as e:
print(f"Error generating content for module '{module['name']}': {e}")
content_results[module["name"]] = {"error": str(e)}
return content_results
# Example outline to test the script
updated_outline = {
"title": "Customer Service Essentials",
"modules": [
{
"name": "Introduction to Customer Service",
"duration": "10 minutes",
"subtopics": [
"Definition of customer service",
"Importance of customer service",
],
},
{
"name": "Handling Customer Complaints",
"duration": "20 minutes",
"subtopics": [
"Types of customer complaints",
"How to handle customer complaints",
],
},
],
}
# Generate content for the updated outline
module_content = generate_content_for_outline(updated_outline)
print(module_content)