Abdul-Haseeb commited on
Commit
c4912ad
·
verified ·
1 Parent(s): e95696d

Upload pipelines.py

Browse files
Files changed (1) hide show
  1. pipelines.py +78 -0
pipelines.py ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import requests
2
+ import json
3
+ import streamlit as st
4
+ from haystack import Pipeline
5
+ from haystack.components.converters import HTMLToDocument
6
+ from haystack.components.fetchers import LinkContentFetcher
7
+ from haystack.components.builders import PromptBuilder
8
+ from haystack.components.generators import HuggingFaceAPIGenerator
9
+ from haystack.components.builders import DynamicChatPromptBuilder
10
+ from haystack.utils import Secret
11
+ from dotenv import load_dotenv
12
+ import os
13
+
14
+ load_dotenv() # Load environment variables from a .env file
15
+
16
+ HUGGINGFACE_API_KEY = os.getenv('HUGGINGFACE_API_KEY')
17
+
18
+ # Define your quiz generation template
19
+ quiz_generation_template = """Given the following text, create only 4 multiple-choice or true-false questions in JSON format randomly.
20
+ The options should be unambiguous.
21
+ For multiple-choice questions, each option should begin with a letter followed by a period and a space (e.g., "a. option").
22
+ For true-false questions, there should be only two options that is ('true','false').
23
+ The question should also briefly mention the general topic of the text so that it can be understood in isolation.
24
+ Include challenging questions that require reasoning.
25
+ Respond with JSON only, no markdown or descriptions.
26
+ Example JSON format you should absolutely follow:
27
+ {
28
+ "questions": [
29
+ {
30
+ "question": "text of the question",
31
+ "options": ["a. 1st option", "b. 2nd option", "c. 3rd option", "d. 4th option"],
32
+ "right_option": "c"
33
+ }
34
+ ]
35
+ }
36
+ IMPORTANT: Do not write anything else and stop generating after once!!!
37
+ text:
38
+ {% for doc in documents %}{{ doc.content|truncate(3800) }}{% endfor %}
39
+ """
40
+
41
+ def generate_quiz_pipeline():
42
+ prompt_builder = DynamicChatPromptBuilder()
43
+ api_type = 'serverless_inference_api'
44
+ quiz_generation_pipeline = Pipeline()
45
+ quiz_generation_pipeline.add_component("link_content_fetcher", LinkContentFetcher())
46
+ quiz_generation_pipeline.add_component("html_converter", HTMLToDocument())
47
+ quiz_generation_pipeline.add_component("prompt_builder", PromptBuilder(template=quiz_generation_template))
48
+ quiz_generation_pipeline.add_component(
49
+ "generator",
50
+ HuggingFaceAPIGenerator(api_type=api_type,
51
+ api_params={'model': 'meta-llama/Meta-Llama-3-8B-Instruct'},
52
+ token=Secret.from_token(HUGGINGFACE_API_KEY),
53
+ )
54
+ )
55
+ quiz_generation_pipeline.connect("link_content_fetcher", "html_converter")
56
+ quiz_generation_pipeline.connect("html_converter", "prompt_builder")
57
+ quiz_generation_pipeline.connect("prompt_builder", "generator")
58
+
59
+ return quiz_generation_pipeline
60
+
61
+ def generate_quiz(url):
62
+ pipeline = generate_quiz_pipeline()
63
+ try:
64
+ results = pipeline.run({"link_content_fetcher": {"urls": [url]}})
65
+ if "generator" in results and "replies" in results["generator"]:
66
+ raw_reply = results["generator"]["replies"][0]
67
+ # Extract the JSON part of the reply
68
+ json_start = raw_reply.find("{")
69
+ json_end = raw_reply.rfind("}") + 1
70
+ if json_start == -1 or json_end == -1:
71
+ raise ValueError("JSON not found in the reply")
72
+ json_reply = raw_reply[json_start:json_end]
73
+ quiz_data = json.loads(json_reply) # Safely parse the JSON string
74
+ return quiz_data
75
+ else:
76
+ raise ValueError("Unexpected response structure")
77
+ except Exception as e:
78
+ raise ValueError(f"Error generating quiz: {e}")