File size: 2,686 Bytes
bd6bce1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | import pandas as pd
import os
def create_intent_dataset_from_folder(folder_path, output_csv_filename):
"""
Reads all .txt files from a specified folder, uses their filenames as
intent labels, and compiles the contents into a single CSV file.
"""
all_data = []
print(f"Starting to process files from folder: '{folder_path}'...")
if not os.path.isdir(folder_path):
print(f"Error: Folder '{folder_path}' not found. Please make sure the folder exists.")
return
# Loop through each file in the specified directory
for filename in os.listdir(folder_path):
# Process only text files
if filename.endswith(".txt"):
file_path = os.path.join(folder_path, filename)
try:
# Extract the intent from the filename
# os.path.splitext splits the filename and extension, we take the first part
intent_label = os.path.splitext(filename)[0]
print(f"Processing file: '{filename}' -> Intent: '{intent_label}'")
# Read each line from the file
with open(file_path, 'r', encoding='utf-8') as f:
utterances = f.readlines()
# Process each line (utterance) in the file
for utterance in utterances:
# Clean up the line by removing leading/trailing whitespace
cleaned_utterance = utterance.strip()
# Ensure the line is not empty before adding it
if cleaned_utterance:
# Append the data as a dictionary to list
all_data.append({
'utterance': cleaned_utterance,
'intent': intent_label
})
except Exception as e:
print(f"An error occurred while processing '{filename}': {e}")
# Convert the list of dictionaries to a pandas DataFrame
if not all_data:
print("No data was processed. The output file will not be created.")
return
df = pd.DataFrame(all_data)
# Save the DataFrame to a CSV file
try:
df.to_csv(output_csv_filename, index=False)
print(f"\nSuccessfully created dataset with {len(df)} records.")
print(f"Data saved to '{output_csv_filename}'")
except Exception as e:
print(f"An error occurred while saving the CSV file: {e}")
if __name__ == "__main__":
intents_folder = "Intents"
output_filename = 'real_estate_intent_dataset.csv'
create_intent_dataset_from_folder(intents_folder, output_filename)
|