Upload full repo excluding dump_40, dump_100, precomputed_tokens, precomputed_data

fb0011a verified 8 months ago

3.21 kB

#!/usr/bin/env python3
import glob
import os

# ---------------- CONFIG ----------------
INPUT_GLOB = "*.txt"       # which files to check (e.g. "*.txt", "data_*.txt")
EXPECT_WAV_EXT = True      # set False if path doesn't have to end with .wav
# ----------------------------------------


def check_line_format(line: str, line_num: int, filename: str):
    """
    Check if a line matches: path|phonemes|speaker_id|language_id|rawtext
    Returns None if OK, or an error message string if NOT OK.
    """
    stripped = line.rstrip("\n")

    # Ignore empty lines
    if not stripped.strip():
        return None

    # Ignore comment lines (optional)
    if stripped.lstrip().startswith("#"):
        return None

    parts = stripped.split("|")

    if len(parts) < 5:
        return f"{filename}:{line_num}: expected >=5 fields, found {len(parts)} → {stripped!r}"

    path = parts[0].strip()
    phonemes = parts[1].strip()
    speaker_id_str = parts[2].strip()
    lang_id_str = parts[3].strip()
    rawtext = "|".join(parts[4:]).strip()  # in case rawtext itself had pipes in future

    if not path:
        return f"{filename}:{line_num}: empty path field"

    if EXPECT_WAV_EXT and not path.lower().endswith(".wav"):
        return f"{filename}:{line_num}: path does not end with .wav → {path!r}"

    # speaker_id must be int
    try:
        int(speaker_id_str)
    except ValueError:
        return f"{filename}:{line_num}: speaker_id not an int → {speaker_id_str!r}"

    # language_id must be int
    try:
        int(lang_id_str)
    except ValueError:
        return f"{filename}:{line_num}: language_id not an int → {lang_id_str!r}"

    if not rawtext:
        return f"{filename}:{line_num}: empty rawtext field"

    # phonemes we just require to exist (can be empty if you allow that)
    # if you *require* non-empty phonemes, uncomment:
    # if not phonemes:
    #     return f"{filename}:{line_num}: empty phonemes field"

    return None  # all good


def main():
    files = sorted(glob.glob(INPUT_GLOB))
    if not files:
        print("⚠️  No files matched pattern:", INPUT_GLOB)
        return

    print(f"📂 Checking {len(files)} file(s): {files}")

    total_errors = 0
    for fname in files:
        errors_for_file = []
        with open(fname, "r", encoding="utf-8") as f:
            for i, line in enumerate(f, start=1):
                err = check_line_format(line, i, fname)
                if err:
                    errors_for_file.append(err)

        if errors_for_file:
            total_errors += len(errors_for_file)
            print(f"\n❌ {fname}: {len(errors_for_file)} line(s) with format issues")
            # Show first few errors to avoid spam
            for msg in errors_for_file[:10]:
                print("   ", msg)
            if len(errors_for_file) > 10:
                print(f"   ... and {len(errors_for_file) - 10} more")
        else:
            print(f"✅ {fname}: all lines OK")

    if total_errors == 0:
        print("\n🎉 All checked files are in correct format.")
    else:
        print(f"\n⚠️  Done. Total problematic lines across all files: {total_errors}")


if __name__ == "__main__":
    main()

	#!/usr/bin/env python3
	import glob
	import os

	# ---------------- CONFIG ----------------
	INPUT_GLOB = ".txt" # which files to check (e.g. ".txt", "data_*.txt")
	EXPECT_WAV_EXT = True # set False if path doesn't have to end with .wav
	# ----------------------------------------


	def check_line_format(line: str, line_num: int, filename: str):
	"""
	Check if a line matches: path\|phonemes\|speaker_id\|language_id\|rawtext
	Returns None if OK, or an error message string if NOT OK.
	"""
	stripped = line.rstrip("\n")

	# Ignore empty lines
	if not stripped.strip():
	return None

	# Ignore comment lines (optional)
	if stripped.lstrip().startswith("#"):
	return None

	parts = stripped.split("\|")

	if len(parts) < 5:
	return f"{filename}:{line_num}: expected >=5 fields, found {len(parts)} → {stripped!r}"

	path = parts[0].strip()
	phonemes = parts[1].strip()
	speaker_id_str = parts[2].strip()
	lang_id_str = parts[3].strip()
	rawtext = "\|".join(parts[4:]).strip() # in case rawtext itself had pipes in future

	if not path:
	return f"{filename}:{line_num}: empty path field"

	if EXPECT_WAV_EXT and not path.lower().endswith(".wav"):
	return f"{filename}:{line_num}: path does not end with .wav → {path!r}"

	# speaker_id must be int
	try:
	int(speaker_id_str)
	except ValueError:
	return f"{filename}:{line_num}: speaker_id not an int → {speaker_id_str!r}"

	# language_id must be int
	try:
	int(lang_id_str)
	except ValueError:
	return f"{filename}:{line_num}: language_id not an int → {lang_id_str!r}"

	if not rawtext:
	return f"{filename}:{line_num}: empty rawtext field"

	# phonemes we just require to exist (can be empty if you allow that)
	# if you require non-empty phonemes, uncomment:
	# if not phonemes:
	# return f"{filename}:{line_num}: empty phonemes field"

	return None # all good


	def main():
	files = sorted(glob.glob(INPUT_GLOB))
	if not files:
	print("⚠️ No files matched pattern:", INPUT_GLOB)
	return

	print(f"📂 Checking {len(files)} file(s): {files}")

	total_errors = 0
	for fname in files:
	errors_for_file = []
	with open(fname, "r", encoding="utf-8") as f:
	for i, line in enumerate(f, start=1):
	err = check_line_format(line, i, fname)
	if err:
	errors_for_file.append(err)

	if errors_for_file:
	total_errors += len(errors_for_file)
	print(f"\n❌ {fname}: {len(errors_for_file)} line(s) with format issues")
	# Show first few errors to avoid spam
	for msg in errors_for_file[:10]:
	print(" ", msg)
	if len(errors_for_file) > 10:
	print(f" ... and {len(errors_for_file) - 10} more")
	else:
	print(f"✅ {fname}: all lines OK")

	if total_errors == 0:
	print("\n🎉 All checked files are in correct format.")
	else:
	print(f"\n⚠️ Done. Total problematic lines across all files: {total_errors}")


	if __name__ == "__main__":
	main()