Upload full repo excluding dump_40, dump_100, precomputed_tokens, precomputed_data

fb0011a verified 8 months ago

4.17 kB

#!/usr/bin/env python3
import os

INPUT_FILE = "syspin.txt"
OUTPUT_FILE = "syspin_fixed.txt"
DEFAULT_LANG_ID = 2           # Hindi language id
PATH_PREFIX = "syspin/"       # what a “real” record line starts with


def join_logical_lines(path_prefix=PATH_PREFIX):
    """
    Join wrapped lines so each logical record becomes a single string.
    Any line starting with `path_prefix` (e.g. 'syspin/') starts a new record.
    Any following non-empty line that does NOT start with that prefix is
    treated as continuation of the previous record's raw text.
    """
    logical = []
    current = None

    with open(INPUT_FILE, "r", encoding="utf-8") as f:
        for raw in f:
            stripped = raw.rstrip("\n")
            if not stripped.strip():
                continue  # skip empty lines

            if stripped.startswith(path_prefix):
                # Start of a new record
                if current is not None:
                    logical.append(current)
                current = stripped
            else:
                # Continuation of previous record's text
                if current is None:
                    # stray line, ignore or keep separately if you want
                    continue
                current += " " + stripped.strip()

        if current is not None:
            logical.append(current)

    return logical


def fix_record(line: str) -> str:
    """
    Parse a single logical record and ensure it has:
        path|phonemes|speaker|lang|rawtext...
    Handles:
    - old 4-field format: path|phonemes|speaker|rawtext...
    - new 5-field format: path|phonemes|speaker|lang|rawtext...
    Rawtext can contain '|' and spaces.
    """
    line = line.strip()
    if not line:
        return line

    # 1) Split off path
    try:
        path, rest = line.split("|", 1)
    except ValueError:
        # No '|' at all, just return as-is
        return line

    path = path.strip()

    # 2) Split off phonemes
    try:
        phonemes, rest2 = rest.split("|", 1)
    except ValueError:
        return line  # malformed, leave as-is

    # 3) Try to parse speaker, lang, rawtext with controlled splits
    # rest2 is "speaker|lang|rawtext..." or "speaker|rawtext..."
    parts = rest2.split("|", 2)

    # Case A: we have at least speaker, lang, rawtext
    if len(parts) == 3:
        speaker_str = parts[0].strip()
        lang_or_raw = parts[1].strip()
        raw_rest = parts[2]  # may contain more '|'

        # speaker must be int
        try:
            int(speaker_str)
        except ValueError:
            return line  # not our expected format

        # If lang_or_raw is int, this is already 5-field format
        try:
            int(lang_or_raw)
            lang_str = lang_or_raw
            rawtext = raw_rest
            return f"{path}|{phonemes}|{speaker_str}|{lang_str}|{rawtext}"
        except ValueError:
            # This is actually 4-field, but rawtext itself had '|'
            # Original shape: speaker|raw_part1|raw_rest
            # So we reassemble rawtext:
            rawtext = lang_or_raw + "|" + raw_rest
            lang_str = str(DEFAULT_LANG_ID)
            return f"{path}|{phonemes}|{speaker_str}|{lang_str}|{rawtext}"

    # Case B: we only have "speaker|rawtext..."
    elif len(parts) == 2:
        speaker_str = parts[0].strip()
        rawtext = parts[1]

        try:
            int(speaker_str)
        except ValueError:
            return line

        lang_str = str(DEFAULT_LANG_ID)
        return f"{path}|{phonemes}|{speaker_str}|{lang_str}|{rawtext}"

    # Anything else: leave unchanged
    return line


def main():
    print(f"🔍 Joining logical lines from {INPUT_FILE}...")
    logical_lines = join_logical_lines()

    print(f"📦 Found {len(logical_lines)} logical records. Fixing format...")
    fixed = [fix_record(l) for l in logical_lines]

    with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
        for l in fixed:
            f.write(l + "\n")

    print(f"✅ Wrote fixed file to {OUTPUT_FILE}")
    print("👉 Now run your check script on syspin_fixed.txt instead of syspin.txt")


if __name__ == "__main__":
    main()

	#!/usr/bin/env python3
	import os

	INPUT_FILE = "syspin.txt"
	OUTPUT_FILE = "syspin_fixed.txt"
	DEFAULT_LANG_ID = 2 # Hindi language id
	PATH_PREFIX = "syspin/" # what a “real” record line starts with


	def join_logical_lines(path_prefix=PATH_PREFIX):
	"""
	Join wrapped lines so each logical record becomes a single string.

	Any line starting with `path_prefix` (e.g. 'syspin/') starts a new record.
	Any following non-empty line that does NOT start with that prefix is
	treated as continuation of the previous record's raw text.
	"""
	logical = []
	current = None

	with open(INPUT_FILE, "r", encoding="utf-8") as f:
	for raw in f:
	stripped = raw.rstrip("\n")
	if not stripped.strip():
	continue # skip empty lines

	if stripped.startswith(path_prefix):
	# Start of a new record
	if current is not None:
	logical.append(current)
	current = stripped
	else:
	# Continuation of previous record's text
	if current is None:
	# stray line, ignore or keep separately if you want
	continue
	current += " " + stripped.strip()

	if current is not None:
	logical.append(current)

	return logical


	def fix_record(line: str) -> str:
	"""
	Parse a single logical record and ensure it has:
	path\|phonemes\|speaker\|lang\|rawtext...

	Handles:
	- old 4-field format: path\|phonemes\|speaker\|rawtext...
	- new 5-field format: path\|phonemes\|speaker\|lang\|rawtext...

	Rawtext can contain '\|' and spaces.
	"""
	line = line.strip()
	if not line:
	return line

	# 1) Split off path
	try:
	path, rest = line.split("\|", 1)
	except ValueError:
	# No '\|' at all, just return as-is
	return line

	path = path.strip()

	# 2) Split off phonemes
	try:
	phonemes, rest2 = rest.split("\|", 1)
	except ValueError:
	return line # malformed, leave as-is

	# 3) Try to parse speaker, lang, rawtext with controlled splits
	# rest2 is "speaker\|lang\|rawtext..." or "speaker\|rawtext..."
	parts = rest2.split("\|", 2)

	# Case A: we have at least speaker, lang, rawtext
	if len(parts) == 3:
	speaker_str = parts[0].strip()
	lang_or_raw = parts[1].strip()
	raw_rest = parts[2] # may contain more '\|'

	# speaker must be int
	try:
	int(speaker_str)
	except ValueError:
	return line # not our expected format

	# If lang_or_raw is int, this is already 5-field format
	try:
	int(lang_or_raw)
	lang_str = lang_or_raw
	rawtext = raw_rest
	return f"{path}\|{phonemes}\|{speaker_str}\|{lang_str}\|{rawtext}"
	except ValueError:
	# This is actually 4-field, but rawtext itself had '\|'
	# Original shape: speaker\|raw_part1\|raw_rest
	# So we reassemble rawtext:
	rawtext = lang_or_raw + "\|" + raw_rest
	lang_str = str(DEFAULT_LANG_ID)
	return f"{path}\|{phonemes}\|{speaker_str}\|{lang_str}\|{rawtext}"

	# Case B: we only have "speaker\|rawtext..."
	elif len(parts) == 2:
	speaker_str = parts[0].strip()
	rawtext = parts[1]

	try:
	int(speaker_str)
	except ValueError:
	return line

	lang_str = str(DEFAULT_LANG_ID)
	return f"{path}\|{phonemes}\|{speaker_str}\|{lang_str}\|{rawtext}"

	# Anything else: leave unchanged
	return line


	def main():
	print(f"🔍 Joining logical lines from {INPUT_FILE}...")
	logical_lines = join_logical_lines()

	print(f"📦 Found {len(logical_lines)} logical records. Fixing format...")
	fixed = [fix_record(l) for l in logical_lines]

	with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
	for l in fixed:
	f.write(l + "\n")

	print(f"✅ Wrote fixed file to {OUTPUT_FILE}")
	print("👉 Now run your check script on syspin_fixed.txt instead of syspin.txt")


	if __name__ == "__main__":
	main()