# scripts/extract_from_pdf.py
import pdfplumber
import os

def extract_text_from_pdf(pdf_path, output_path):
    # Open the file for writing in append mode to ensure each page's content is saved immediately
    os.makedirs(os.path.dirname(output_path), exist_ok=True)
    with open(output_path, 'w', encoding='utf-8') as f:
        with pdfplumber.open(pdf_path) as pdf:
            for page_num, page in enumerate(pdf.pages, start=1):
                # Define bounding boxes for each column
                right_column = (page.width * 0.7, 0, page.width, page.height)

                # Extract text from the right column
                right_text = page.within_bbox(right_column).extract_text(x_tolerance=3, y_tolerance=3, layout=True)
                
                # Debug output to check extracted text
                print(f"\n--- Page {page_num} ---")
                print("Right column (proverbs):")
                print(right_text if right_text else "No text found in right column")

                # Write each line from the right column directly to the file
                if right_text:
                    right_lines = right_text.split('\n')
                    for line in right_lines:
                        # Check and print each line before writing to verify it has content
                        line = line.strip()
                        if line:
                            print(f"Writing line to file: {line}")
                            f.write(line + '\n')

    print(f"Data successfully saved to {output_path}")

if __name__ == "__main__":
    # Path to your PDF file
    pdf_path = r"C:\Users\moham\iraqi_arabic_llm_project\iraqi_proverbs.pdf"
    
    # Output path for the extracted proverbs
    output_path = "../data/raw_data/iraqi_data.txt"
    
    extract_text_from_pdf(pdf_path, output_path)