File size: 1,917 Bytes
1e1bc1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | """Create a derived char vocab with additional path characters."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Append missing characters to an AniFileBERT char vocab JSON."
)
parser.add_argument("--input", required=True, help="Base vocab.char.json path")
parser.add_argument("--output", required=True, help="Derived vocab output path")
parser.add_argument(
"--chars",
default="/\\",
help="Characters to ensure in the vocab. Default adds slash and backslash.",
)
return parser.parse_args()
def main() -> None:
args = parse_args()
input_path = Path(args.input)
output_path = Path(args.output)
vocab = json.loads(input_path.read_text(encoding="utf-8"))
if not isinstance(vocab, dict):
raise TypeError(f"Expected object vocab JSON: {input_path}")
next_id = max(int(value) for value in vocab.values()) + 1
added: list[tuple[str, int]] = []
for char in args.chars:
if char not in vocab:
vocab[char] = next_id
added.append((char, next_id))
next_id += 1
ordered = dict(sorted(vocab.items(), key=lambda item: int(item[1])))
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(
json.dumps(ordered, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
print(
json.dumps(
{
"input": str(input_path),
"output": str(output_path),
"base_size": len(vocab) - len(added),
"output_size": len(vocab),
"added": [{"char": char, "id": idx} for char, idx in added],
},
ensure_ascii=False,
indent=2,
)
)
if __name__ == "__main__":
main()
|