Data/validate_list.py

import os

def check_and_remove_missing_files(input_file, base_dir=None):
    # Normalize base_dir (e.g., "../") if provided
    if base_dir is not None:
        base_dir = os.path.normpath(base_dir)

    # Read the text file containing the list of files to check
    with open(input_file, 'r', encoding='utf-8') as f:
        all_lines = f.readlines()

    files_to_check = []
    for line in all_lines:
        # Split the line into path and the rest of the text (after the | character)
        parts = line.strip().split('|', 1)
        if parts and parts[0]:
            file_path = parts[0].strip()
            files_to_check.append(file_path)

    # Check for missing files (using base_dir if provided)
    missing_files = []
    for file_path in files_to_check:
        resolved_path = (
            os.path.join(base_dir, file_path) if base_dir is not None else file_path
        )
        if not os.path.exists(resolved_path):
            missing_files.append(file_path)

    # Print missing files
    if missing_files:
        print("Missing files (resolved paths):")
        for file_path in missing_files:
            resolved_path = (
                os.path.join(base_dir, file_path) if base_dir is not None else file_path
            )
            print(resolved_path)
        print(f"\nNumber of missing files: {len(missing_files)}")

        # Remove lines with missing files (compare using original paths)
        new_lines = []
        missing_set = set(missing_files)
        for line in all_lines:
            parts = line.strip().split('|', 1)
            keep = True
            if parts and parts[0]:
                file_path = parts[0].strip()
                if file_path in missing_set:
                    keep = False
            if keep:
                new_lines.append(line)

        # Write the updated list back to the file
        with open(input_file, 'w', encoding='utf-8') as f:
            f.writelines(new_lines)

        print(f"\nRemoved {len(missing_files)} entries from {input_file}")
    else:
        print("All files exist.")

# Example usage:
# Set base_dir to "../" or any other directory you want to prepend
input_file = "train_3_filtered_dur_fixed_copy.txt"  # Change to your actual filename
check_and_remove_missing_files(input_file, base_dir="../")