Spaces:

IOE-IFAD
/

test

Paused

test / _backend /reset_vdb_minutes_data.py

Automated build & deploy

e454383 verified over 1 year ago

3.41 kB

import os
from dotenv import load_dotenv
from utils.document_management import load_document, generate_filename_from_details
from utils.vector_db import VectorDB
from langchain.schema import Document
from uuid import uuid4
import re


# Load environment variables
load_dotenv()

# Initialize services
# Load all the transcriptions from the "transcription_data" directory
working_dir = os.path.join(os.getcwd(), "_backend" ,"data", "meeting_minutes_data")
if os.getenv("ENVIRONMENT") == "production":
    working_dir = os.path.join(os.getcwd(), "data","meeting_minutes_data")
vector_db_name = "meeting_minutes"
vector_db_embedding_length = 1536

# First, collect all documents
all_documents = []
all_files = []

# Walk through the directory to find all *_transcribed.srt files
for root, dirs, files in os.walk(working_dir):
    for file in files:
        if file.endswith(".md"):
            file_path = os.path.join(root, file)
            # clipping everything before honduras_data 
            relative_path = os.path.relpath(file_path, working_dir)
            all_files.append(relative_path)

minute_start_pattern = re.compile(
    r"^\|\s*---\s*\|\s*---\s*\|\s*$"  # Matches | --- | --- |
    r"\n"
    r"^\|[^\|\n]*\w[^\|\n]*?\|\s*\|\s*$"  # Matches a generic content line (e.g., date/name)
    r"\n"
    r"^\|\s*\*\*Interviewee Name:\*\*",  # Matches the line with | **Interviewee Name:**
    re.MULTILINE
)

for file_path in all_files:
    path = os.path.join(working_dir, file_path)
    
    content = load_document(path, raw_text=True)

    if content is None:
        print(f"Skipping file {file_path} as it could not be loaded.")
        continue
    
    # Matching the content with the regex pattern
    matches = list(minute_start_pattern.finditer(content))
    header_block = ""
    start_index = matches[0].start() if matches else 0
    end_index = matches[-1].end() if matches else len(content)

    header_block = content[start_index:end_index].strip()
    body = content[end_index:].strip()
    body = body.rstrip("|  |  |")

    filename, info = generate_filename_from_details(header_block)

    docuemnt = Document(
        page_content=body,
        metadata={
            "file_name": filename,
            "interview_date": info[0],
            "interviewee_name_role": info[1],
            "id": str(uuid4()),
            "url": f"https://ifad.sharepoint.com/:t:/r/sites/IFAD-CLEIFAD11andIFAD12/Shared%20Documents/General/07.%20HQ%20Interviews/Minutes/{filename}?csf=1&web=1",
            "content": body,
        }
    )

    all_documents.append(docuemnt)
            
# Now create the vector database with all documents at once (this will delete and recreate)
if all_documents:
    if os.path.exists(os.path.join(working_dir, "chroma_db")):
        import shutil
        # Remove the existing vector database directory
        shutil.rmtree(os.path.join(working_dir, "chroma_db"))
        print("Removed existing vector database directory.")

    vector_db = VectorDB(
        collection_name=vector_db_name,
        embedding_length=vector_db_embedding_length,
        open_ai_api_key=os.getenv("OPENAI_API_KEY"),
        working_dir=working_dir,
        documents=all_documents
    )
    print(f"Created new vector database with {len(all_documents)} documents")
else:
    print("No documents found to process")

print("Vector database reset and data upload complete with ChromaDB.")

	import os
	from dotenv import load_dotenv
	from utils.document_management import load_document, generate_filename_from_details
	from utils.vector_db import VectorDB
	from langchain.schema import Document
	from uuid import uuid4
	import re


	# Load environment variables
	load_dotenv()

	# Initialize services
	# Load all the transcriptions from the "transcription_data" directory
	working_dir = os.path.join(os.getcwd(), "_backend" ,"data", "meeting_minutes_data")
	if os.getenv("ENVIRONMENT") == "production":
	working_dir = os.path.join(os.getcwd(), "data","meeting_minutes_data")
	vector_db_name = "meeting_minutes"
	vector_db_embedding_length = 1536

	# First, collect all documents
	all_documents = []
	all_files = []

	# Walk through the directory to find all *_transcribed.srt files
	for root, dirs, files in os.walk(working_dir):
	for file in files:
	if file.endswith(".md"):
	file_path = os.path.join(root, file)
	# clipping everything before honduras_data
	relative_path = os.path.relpath(file_path, working_dir)
	all_files.append(relative_path)

	minute_start_pattern = re.compile(
	r"^\\|\s---\s\\|\s---\s\\|\s*$" # Matches \| --- \| --- \|
	r"\n"
	r"^\\|[^\\|\n]\w[^\\|\n]?\\|\s\\|\s$" # Matches a generic content line (e.g., date/name)
	r"\n"
	r"^\\|\s\\Interviewee Name:\\", # Matches the line with \| Interviewee Name:*
	re.MULTILINE
	)

	for file_path in all_files:
	path = os.path.join(working_dir, file_path)

	content = load_document(path, raw_text=True)

	if content is None:
	print(f"Skipping file {file_path} as it could not be loaded.")
	continue

	# Matching the content with the regex pattern
	matches = list(minute_start_pattern.finditer(content))
	header_block = ""
	start_index = matches[0].start() if matches else 0
	end_index = matches[-1].end() if matches else len(content)

	header_block = content[start_index:end_index].strip()
	body = content[end_index:].strip()
	body = body.rstrip("\| \| \|")

	filename, info = generate_filename_from_details(header_block)

	docuemnt = Document(
	page_content=body,
	metadata={
	"file_name": filename,
	"interview_date": info[0],
	"interviewee_name_role": info[1],
	"id": str(uuid4()),
	"url": f"https://ifad.sharepoint.com/:t:/r/sites/IFAD-CLEIFAD11andIFAD12/Shared%20Documents/General/07.%20HQ%20Interviews/Minutes/{filename}?csf=1&web=1",
	"content": body,
	}
	)

	all_documents.append(docuemnt)

	# Now create the vector database with all documents at once (this will delete and recreate)
	if all_documents:
	if os.path.exists(os.path.join(working_dir, "chroma_db")):
	import shutil
	# Remove the existing vector database directory
	shutil.rmtree(os.path.join(working_dir, "chroma_db"))
	print("Removed existing vector database directory.")

	vector_db = VectorDB(
	collection_name=vector_db_name,
	embedding_length=vector_db_embedding_length,
	open_ai_api_key=os.getenv("OPENAI_API_KEY"),
	working_dir=working_dir,
	documents=all_documents
	)
	print(f"Created new vector database with {len(all_documents)} documents")
	else:
	print("No documents found to process")

	print("Vector database reset and data upload complete with ChromaDB.")