File size: 1,425 Bytes
c83c617 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | """FP32 inference and candidate scoring using the included VimeML V4 source."""
import argparse
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--prompt", default="今日は雨が降っているので、")
parser.add_argument("--candidate", action="append", help="Candidate string; repeat to rerank.")
parser.add_argument("--max-new-tokens", type=int, default=8)
parser.add_argument("--threads", type=int, default=4)
args = parser.parse_args()
if args.threads < 1 or args.max_new_tokens < 1:
parser.error("threads and max-new-tokens must be positive.")
sys.path.insert(0, str(ROOT / "source/src"))
import torch
from vimeml.deployment.v4_coreml import BundleLMV4
from vimeml.training.evaluate_ime import score_candidates
torch.set_num_threads(args.threads)
lm = BundleLMV4(ROOT / "inference")
if args.candidate:
result = score_candidates(lm, args.prompt, args.candidate)
result["ranked_candidates"] = sorted(
result["candidates"], key=lambda item: item["log_probability_sum"], reverse=True
)
else:
result = lm.generate(args.prompt, max_new_tokens=args.max_new_tokens, temperature=0)
print(json.dumps(result, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
|