"""Export the same shared runtime used by both installed tokenizers.""" from __future__ import annotations import json from pathlib import Path import shutil def save_runtime(directory: str | Path) -> list[str]: """Bundle portable code and AutoProcessor metadata beside fitted assets.""" root = Path(directory) source = Path(__file__).absolute().parent destination = root / 'runtime' if destination.resolve() != source: if destination.exists(): shutil.rmtree(destination) shutil.copytree(source, destination, ignore=shutil.ignore_patterns('__pycache__', '*.pyc', '*.so', 'native')) entry = root / 'processing_actioncodec2.py' entry.write_text((source / 'catalog.py').read_text(encoding='utf-8'), encoding='utf-8') for name in ('config.json', 'processor_config.json'): path = root / name metadata = json.loads(path.read_text()) metadata['auto_map'] = {'AutoProcessor': 'processing_actioncodec2.ActionCodec2'} path.write_text(json.dumps(metadata, indent=2) + '\n') kernels = root / 'actioncodec2_kernels' if kernels.exists(): shutil.rmtree(kernels) native_source = source / 'backends/actioncodec2/native' if not native_source.is_dir(): native_source = source.parent / 'actioncodec2_kernels' shutil.copytree(native_source, kernels, ignore=shutil.ignore_patterns('__pycache__', '*.so', '*.o')) (kernels / '__init__.py').write_text('') (root / 'setup.py').write_text('''import sys from setuptools import setup from pybind11.setup_helpers import Pybind11Extension, build_ext from pathlib import Path native = 'actioncodec2_kernels' openmp = ['-fopenmp'] if sys.platform.startswith('linux') else [] setup(name='actioncodec2-kernels', version='0.1.0', packages=['actioncodec2_kernels'], ext_modules=[Pybind11Extension('actioncodec2_kernels._native', sources=sorted(str(p) for p in Path(native).glob('*.cpp')), include_dirs=[native], cxx_std=17, extra_compile_args=['-O3', '-fvisibility=hidden', '-ffp-contract=off', *openmp], extra_link_args=openmp)], cmdclass={'build_ext': build_ext}) ''') (root / 'pyproject.toml').write_text('[build-system]\nrequires = ["setuptools>=77", "wheel", "pybind11>=2.12"]\nbuild-backend = "setuptools.build_meta"\n') license_path = next((p / 'LICENSE' for p in source.parents if (p / 'LICENSE').is_file()), None) if license_path is not None and license_path.resolve() != (root / 'LICENSE').resolve(): shutil.copyfile(license_path, root / 'LICENSE') return [str(entry), *map(str, destination.rglob('*.py'))]