code/src/preprocessing/build_mzl_abz_map.py · savastakan/hitit-cuneiform-ocr at main

hitit-cuneiform-ocr / code /src /preprocessing /build_mzl_abz_map.py

Initial upload: code + 5 record checkpoints + fuse

f211247 verified 6 months ago

3.9 kB

#!/usr/bin/env python3
"""Build MZL ↔ ABZ mapping from Oracc OSL (osl.asl).
Input: /tmp/osl/00lib/osl.asl (Oracc Global Sign List, CC0)
Output: JSON with:
  - mzl_to_abz: MZL_num → ABZ_num
  - mzl_to_sign: MZL_num → canonical sign name
  - abz_to_sign: ABZ_num → canonical sign name
  - sign_to_mzl: sign → MZL_num
  - sign_to_abz: sign → ABZ_num
"""
import json, re, argparse
from pathlib import Path


def parse_asl(path):
    """Parse @sign ... @end sign blocks from osl.asl."""
    blocks = []
    cur = None
    with open(path) as f:
        for line in f:
            line = line.rstrip()
            if line.startswith('@sign '):
                name = line[6:].strip()
                cur = {'sign': name, 'lists': {}, 'uname': None, 'ucun': None}
            elif line.startswith('@sign- '):
                cur = None  # skip deprecated signs
            elif cur is not None and line.startswith('@list\t'):
                token = line[6:].strip()
                # token format: "MZL001" or "ABZ123" or "ABZ123v" etc
                m = re.match(r'^([A-Z]+)(\S+)$', token)
                if m:
                    lst, num = m.group(1), m.group(2)
                    # Prefer first occurrence per list
                    if lst not in cur['lists']:
                        cur['lists'][lst] = num
            elif cur is not None and line.startswith('@uname\t'):
                cur['uname'] = line[7:].strip()
            elif cur is not None and line.startswith('@ucun\t'):
                cur['ucun'] = line[6:].strip()
            elif line.startswith('@end sign'):
                if cur and cur['sign']:
                    blocks.append(cur)
                cur = None
    return blocks


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--osl', default='/tmp/osl/00lib/osl.asl')
    ap.add_argument('--output', required=True)
    args = ap.parse_args()

    blocks = parse_asl(args.osl)
    print(f"Parsed {len(blocks)} signs")

    mzl_to_abz = {}
    mzl_to_sign = {}
    abz_to_sign = {}
    sign_to_mzl = {}
    sign_to_abz = {}

    for b in blocks:
        sign = b['sign']
        mzl = b['lists'].get('MZL')
        # In Oracc OSL the Borger list appears as ABZL (Assyrisch-Babylonische
        # Zeichenliste); our codebase historically calls it "ABZ", so treat
        # ABZL as the ABZ source.
        abz = b['lists'].get('ABZL') or b['lists'].get('ABZ')
        if mzl:
            # MZL numbers may have leading zeros and optional 'a','b','v' suffixes
            # strip leading zeros of pure-numeric prefix
            mzl_clean = re.sub(r'^0+', '', mzl) or '0'
            mzl_to_sign[mzl] = sign
            mzl_to_sign[mzl_clean] = sign  # also with stripped
            sign_to_mzl[sign] = mzl
            if abz:
                abz_clean = re.sub(r'^0+', '', abz) or '0'
                mzl_to_abz[mzl] = abz
                mzl_to_abz[mzl_clean] = abz
        if abz:
            abz_to_sign[abz] = sign
            abz_to_sign[re.sub(r'^0+', '', abz) or '0'] = sign
            sign_to_abz[sign] = abz

    print(f"MZL entries: {len(sign_to_mzl)}")
    print(f"ABZ entries: {len(sign_to_abz)}")
    print(f"MZL→ABZ mappings: {len(mzl_to_abz)}")

    # Report a few examples
    for mzl in ['MZL001', 'MZL839', 'MZL252', 'MZL090']:
        if mzl in mzl_to_sign:
            print(f"  {mzl} = {mzl_to_sign[mzl]} (ABZ: {mzl_to_abz.get(mzl, '?')})")

    out = {
        'mzl_to_abz': mzl_to_abz,
        'mzl_to_sign': mzl_to_sign,
        'abz_to_sign': abz_to_sign,
        'sign_to_mzl': sign_to_mzl,
        'sign_to_abz': sign_to_abz,
        'n_signs_total': len(blocks),
    }
    Path(args.output).parent.mkdir(parents=True, exist_ok=True)
    with open(args.output, 'w') as f:
        json.dump(out, f, indent=1, ensure_ascii=False)
    print(f"Saved → {args.output}")

if __name__ == '__main__':
    main()

	#!/usr/bin/env python3
	"""Build MZL ↔ ABZ mapping from Oracc OSL (osl.asl).

	Input: /tmp/osl/00lib/osl.asl (Oracc Global Sign List, CC0)
	Output: JSON with:
	- mzl_to_abz: MZL_num → ABZ_num
	- mzl_to_sign: MZL_num → canonical sign name
	- abz_to_sign: ABZ_num → canonical sign name
	- sign_to_mzl: sign → MZL_num
	- sign_to_abz: sign → ABZ_num
	"""
	import json, re, argparse
	from pathlib import Path


	def parse_asl(path):
	"""Parse @sign ... @end sign blocks from osl.asl."""
	blocks = []
	cur = None
	with open(path) as f:
	for line in f:
	line = line.rstrip()
	if line.startswith('@sign '):
	name = line[6:].strip()
	cur = {'sign': name, 'lists': {}, 'uname': None, 'ucun': None}
	elif line.startswith('@sign- '):
	cur = None # skip deprecated signs
	elif cur is not None and line.startswith('@list\t'):
	token = line[6:].strip()
	# token format: "MZL001" or "ABZ123" or "ABZ123v" etc
	m = re.match(r'^([A-Z]+)(\S+)$', token)
	if m:
	lst, num = m.group(1), m.group(2)
	# Prefer first occurrence per list
	if lst not in cur['lists']:
	cur['lists'][lst] = num
	elif cur is not None and line.startswith('@uname\t'):
	cur['uname'] = line[7:].strip()
	elif cur is not None and line.startswith('@ucun\t'):
	cur['ucun'] = line[6:].strip()
	elif line.startswith('@end sign'):
	if cur and cur['sign']:
	blocks.append(cur)
	cur = None
	return blocks


	def main():
	ap = argparse.ArgumentParser()
	ap.add_argument('--osl', default='/tmp/osl/00lib/osl.asl')
	ap.add_argument('--output', required=True)
	args = ap.parse_args()

	blocks = parse_asl(args.osl)
	print(f"Parsed {len(blocks)} signs")

	mzl_to_abz = {}
	mzl_to_sign = {}
	abz_to_sign = {}
	sign_to_mzl = {}
	sign_to_abz = {}

	for b in blocks:
	sign = b['sign']
	mzl = b['lists'].get('MZL')
	# In Oracc OSL the Borger list appears as ABZL (Assyrisch-Babylonische
	# Zeichenliste); our codebase historically calls it "ABZ", so treat
	# ABZL as the ABZ source.
	abz = b['lists'].get('ABZL') or b['lists'].get('ABZ')
	if mzl:
	# MZL numbers may have leading zeros and optional 'a','b','v' suffixes
	# strip leading zeros of pure-numeric prefix
	mzl_clean = re.sub(r'^0+', '', mzl) or '0'
	mzl_to_sign[mzl] = sign
	mzl_to_sign[mzl_clean] = sign # also with stripped
	sign_to_mzl[sign] = mzl
	if abz:
	abz_clean = re.sub(r'^0+', '', abz) or '0'
	mzl_to_abz[mzl] = abz
	mzl_to_abz[mzl_clean] = abz
	if abz:
	abz_to_sign[abz] = sign
	abz_to_sign[re.sub(r'^0+', '', abz) or '0'] = sign
	sign_to_abz[sign] = abz

	print(f"MZL entries: {len(sign_to_mzl)}")
	print(f"ABZ entries: {len(sign_to_abz)}")
	print(f"MZL→ABZ mappings: {len(mzl_to_abz)}")

	# Report a few examples
	for mzl in ['MZL001', 'MZL839', 'MZL252', 'MZL090']:
	if mzl in mzl_to_sign:
	print(f" {mzl} = {mzl_to_sign[mzl]} (ABZ: {mzl_to_abz.get(mzl, '?')})")

	out = {
	'mzl_to_abz': mzl_to_abz,
	'mzl_to_sign': mzl_to_sign,
	'abz_to_sign': abz_to_sign,
	'sign_to_mzl': sign_to_mzl,
	'sign_to_abz': sign_to_abz,
	'n_signs_total': len(blocks),
	}
	Path(args.output).parent.mkdir(parents=True, exist_ok=True)
	with open(args.output, 'w') as f:
	json.dump(out, f, indent=1, ensure_ascii=False)
	print(f"Saved → {args.output}")

	if __name__ == '__main__':
	main()