test

contents_extractor_v2.py

@@ -44,8 +44,8 @@ class ContentsExtractor:
                 
                 Instructions:
                 1. Identify the "Contents" section, which lists all topics, subtopics, and their corresponding pages.
-                2. Extract only the **highest-level, subject-related topics** (ignore organizational or administrative sections).
-                3. If a topic has subtopics, include the full range of pages from the first to the last subtopic.
+                2. Extract only the **highest-level, subject-related subtopics** (ignore organizational or administrative sections).
+                3. For subtopics, include the full range of pages from the first to the last subtopic.
                 4. Return the output in the following JSON format:
                 
                     {{
@@ -53,58 +53,87 @@ class ContentsExtractor:
                     }}
                 
                 Important Notes:
-                - Ignore non-subject-related sections (e.g., "Introduction", "Exam Guidelines", "Appendices").
-                - If a topic has subtopics, **only extract the main topic**, ensuring the page range covers all subtopics.
-                - The extracted topics should represent major academic areas, not organizational or structural elements.
-                - Only extract main topics without sub-topic numeration. Any topic with additional numbering (e.g., '3.1 Some Topic')
-                  should be ignored, as it is a sub-topic rather than a primary subject-related topic.
-                - Make sure that all of the pages for a topic are included, end page should be the start page of the topic
+                - Ignore non-subject-related sections (e.g., "Introduction", "Exam Guidelines", "Appendices", "Assessment, Qualification at a glance").
+                - The extracted subtopics should represent major academic areas, not organizational or structural elements.
+                - Make sure that all of the pages for a subtopic are included, end page should be the start page of the topic
                   that comes next after the extracted one in contents section.
                 
                 Examples:
                 1. Given this table of contents:
-                    
-                    1. Introduction - 1  
-                    2. Exam Rules - 4  
-                    3. Subject content - 8  
-                       3.1 Algebra - 12  
-                       3.2 Geometry - 16  
-                       3.3 Probability - 20  
-                    4. The topics of subject of physics - 25  
-                       4.1 Mechanics - 30  
-                       4.2 Thermodynamics - 35  
-                    5. Appendices - 40  
+
+                    1 Introduction – 2
+                        Why choose Edexcel A Level Mathematics? - 2
+                        Supporting you in planning and implementing this qualification - 3
+                        Qualification at a glance - 5
+                    2 Subject content and assessment information – 7
+                        Paper 1 and Paper 2: Pure Mathematics - 11
+                        Paper 3: Statistics and Mechanics - 30
+                        Assessment Objectives - 40
+                    3 Administration and general information – 42
+                        Entries - 42
+                        Access arrangements, reasonable adjustments, special consideration and malpractice - 42
+                        Student recruitment and progression - 45
+                    Appendix 1: Formulae – 49
+                    Appendix 2: Notation – 53
+                    Appendix 3: Use of calculators – 59
+                    Appendix 4: Assessment Objectives – 60
+                    Appendix 5: The context for the development of this qualification – 62
+                    Appendix 6: Transferable skills – 64
+                    Appendix 7: Level 3 Extended Project qualification – 65
+                    Appendix 8: Codes – 67
                 
                    The correct output should be:
                 
                     {{
-                        "3. Subject content": [8, 25],
-                        "4. The topics of subject of physics": [25, 40]
+                        "Paper 1 and Paper 2: Pure Mathematics": [11, 29],
+                        "Paper 3: Statistics and Mechanics": [30, 42]
                     }}
                 
                 2. Given this table of contents:
-                
-                    1. Welcome Note - 1  
-                    2. Exam Overview - 3  
-                    3. Biology - 5  
-                       3.1 Cell Biology - 7  
-                       3.2 Genetics - 12  
-                       3.3 Ecology - 18  
-                    4. Chemistry - 22  
-                       4.1 Organic Chemistry - 25  
-                       4.2 Inorganic Chemistry - 30  
-                       4.3 Physical Chemistry - 35  
-                    5. References - 43
+
+                    Qualification at a glance – 1
+                        Assessment Objectives and weightings - 4
+                    Knowledge, skills and understanding – 5
+                        Theme 1: Introduction to markets and market failure - 5
+                        Theme 2: The UK economy – performance and policies - 11
+                        Theme 3: Business behaviour and the labour market - 21
+                        Theme 4: A global perspective - 29
+                    Assessment – 39
+                        Assessment summary - 39
+                        Assessment objectives - 41
+                        Assessment overview - 42
+                        Breakdown of assessment objectives - 42
+                            Synoptic assessment - 43
+                            Discount code and performance tables - 43
+                            Access arrangements, reasonable adjustments and special consideration - 44
+                            Malpractice - 45
+                            Equality Act 2010 and Pearson equality policy - 45
+                            Synoptic assessment - 46
+                            Awarding and reporting - 47
+                    Other information – 49
+                        Student recruitment -49
+                        Prior learning and other requirements -49
+                        Progression - 49
+                    Appendix 1: Transferable skills – 53
+                    Appendix 2: Level 3 Extended Project qualification – 55
+                    Appendix 3: Quantitative skills – 59
+                    Appendix 4: Codes – 61
+                    Appendix 5: Index – 63
                 
                    The correct output should be:
                 
                     {{
                         "Biology": [5, 22],
                         "Chemistry": [22, 43]
+
+                        "Theme 1: Introduction to markets and market failure": [5, 10]
+                        "Theme 2: The UK economy – performance and policies": - [11, 20]
+                        "Theme 3: Business behaviour and the labour market": [21, 28]
+                        "Theme 4: A global perspective": [29, 38]
                     }}
                     
                     Now, extract topics from this text: {content}
                 """],
             config=types.GenerateContentConfig(temperature=0.)
         )
-        return response.text.strip().replace("```json", "").replace("```", "")
+        return response.text.strip().replace("```json", "").replace("```", "")

topic_extraction.py

@@ -15,20 +15,13 @@ logging.basicConfig(
     format="%(asctime)s [%(levelname)s] %(name)s - %(message)s",
     handlers=[
         logging.StreamHandler(),
-        logging.FileHandler('selective_pdf_extractor.log')
+        # logging.FileHandler('selective_pdf_extractor.log')
     ]
 )
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
 class SelectivePDFProcessor:
-    """
-    Uses Gemini to identify:
-      - The '2 Subject content...' section from the Contents
-      - Each subtopic name + page range
-    Then extracts only those subtopic pages, processes them with LocalPDFProcessor,
-    and removes all text except the final table rows from the output markdown.
-    """
 
     def __init__(self, output_folder: str, api_key: str):
         self.output_folder = output_folder
@@ -100,7 +93,6 @@ class SelectivePDFProcessor:
             raw_json = raw_json.replace("```json", "").replace("```", "").strip()
             data = json.loads(raw_json)
 
-            # data might look like:
             # {
             #   "1 Introduction": [1, 5],
             #   "2 Subject content and assessment information": {