Upload test.txt

test.txt

@@ -0,0 +1,119 @@
+import json
+import os
+from typing import List, Dict
+
+def merge_paragraph_chunks(input_json_path: str, output_dir: str, merge_count: int = 5):
+    """
+    Merges consecutive paragraph chunks of the same page from the input JSON file.
+
+    Args:
+        input_json_path (str): Path to the input JSON file.
+        output_dir (str): Directory where output JSON will be saved.
+        merge_count (int): Number of paragraphs to merge together.
+    """
+
+    # Read input JSON
+    with open(input_json_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+
+    merged_result = []
+    temp_group: List[Dict] = []
+    current_page = None
+
+    def flush_group(group: List[Dict]):
+        """Helper to merge a group of paragraphs and return merged items."""
+        merged = []
+        for i in range(0, len(group), merge_count):
+            group_slice = group[i:i + merge_count]
+            merged_content = " ".join(item["content"].strip() for item in group_slice)
+            merged.append({
+                "type": "paragraph",
+                "content": merged_content.strip(),
+                "page": group_slice[0]["page"]
+            })
+        return merged
+
+    for item in data:
+        if item.get("type") == "paragraph":
+            page = item.get("page")
+            if current_page is None:
+                current_page = page
+
+            if page == current_page:
+                temp_group.append(item)
+            else:
+                # flush group from previous page
+                merged_result.extend(flush_group(temp_group))
+                temp_group = [item]
+                current_page = page
+        else:
+            # flush any existing group before adding a non-paragraph item
+            if temp_group:
+                merged_result.extend(flush_group(temp_group))
+                temp_group = []
+                current_page = None
+            merged_result.append(item)
+
+    # Final flush
+    if temp_group:
+        merged_result.extend(flush_group(temp_group))
+
+    # Save to output directory
+    os.makedirs(output_dir, exist_ok=True)
+    output_path = os.path.join(output_dir, 'merged_chunks.json')
+    with open(output_path, 'w', encoding='utf-8') as f:
+        json.dump(merged_result, f, indent=4)
+
+    print(f"✅ Merged chunks saved to: {output_path}")
+
+------------------------------------
+
+import json
+import os
+from typing import List, Dict
+from collections import defaultdict
+
+def merge_one_chunk_per_page(input_json_path: str, output_dir: str):
+    """
+    Merge all paragraph chunks on the same page into one chunk per page.
+
+    Args:
+        input_json_path (str): Path to input JSON file.
+        output_dir (str): Directory to save the merged output JSON.
+    """
+
+    # Load input JSON
+    with open(input_json_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+
+    # Dictionary to collect paragraphs by page
+    page_paragraphs = defaultdict(list)
+    other_elements = []
+
+    for item in data:
+        if item.get("type") == "paragraph":
+            page = item.get("page")
+            page_paragraphs[page].append(item["content"].strip())
+        else:
+            other_elements.append(item)
+
+    # Create one merged paragraph per page
+    merged_paragraphs = []
+    for page, paragraphs in sorted(page_paragraphs.items(), key=lambda x: int(x[0])):
+        merged_paragraphs.append({
+            "type": "paragraph",
+            "content": " ".join(paragraphs),
+            "page": page
+        })
+
+    # Combine with other elements (optional sorting logic can be added if needed)
+    final_output: List[Dict] = merged_paragraphs + other_elements
+
+    # Write to output
+    os.makedirs(output_dir, exist_ok=True)
+    output_path = os.path.join(output_dir, "merged_chunks.json")
+    with open(output_path, 'w', encoding='utf-8') as f:
+        json.dump(final_output, f, indent=4)
+
+    print(f"✅ One-chunk-per-page JSON saved at: {output_path}")
+