File size: 2,660 Bytes
1bfc3f0 ce32ca3 1bfc3f0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | """Export the same shared runtime used by both installed tokenizers."""
from __future__ import annotations
import json
from pathlib import Path
import shutil
def save_runtime(directory: str | Path) -> list[str]:
"""Bundle portable code and AutoProcessor metadata beside fitted assets."""
root = Path(directory)
source = Path(__file__).absolute().parent
destination = root / 'runtime'
if destination.resolve() != source:
if destination.exists():
shutil.rmtree(destination)
shutil.copytree(source, destination, ignore=shutil.ignore_patterns('__pycache__', '*.pyc', '*.so', 'native'))
entry = root / 'processing_actioncodec2.py'
entry.write_text((source / 'catalog.py').read_text(encoding='utf-8'), encoding='utf-8')
for name in ('config.json', 'processor_config.json'):
path = root / name
metadata = json.loads(path.read_text())
metadata['auto_map'] = {'AutoProcessor': 'processing_actioncodec2.ActionCodec2'}
path.write_text(json.dumps(metadata, indent=2) + '\n')
kernels = root / 'actioncodec2_kernels'
if kernels.exists():
shutil.rmtree(kernels)
native_source = source / 'backends/actioncodec2/native'
if not native_source.is_dir():
native_source = source.parent / 'actioncodec2_kernels'
shutil.copytree(native_source, kernels,
ignore=shutil.ignore_patterns('__pycache__', '*.so', '*.o'))
(kernels / '__init__.py').write_text('')
(root / 'setup.py').write_text('''import sys
from setuptools import setup
from pybind11.setup_helpers import Pybind11Extension, build_ext
from pathlib import Path
native = 'actioncodec2_kernels'
openmp = ['-fopenmp'] if sys.platform.startswith('linux') else []
setup(name='actioncodec2-kernels', version='0.1.0', packages=['actioncodec2_kernels'],
ext_modules=[Pybind11Extension('actioncodec2_kernels._native',
sources=sorted(str(p) for p in Path(native).glob('*.cpp')),
include_dirs=[native], cxx_std=17,
extra_compile_args=['-O3', '-fvisibility=hidden', '-ffp-contract=off', *openmp],
extra_link_args=openmp)], cmdclass={'build_ext': build_ext})
''')
(root / 'pyproject.toml').write_text('[build-system]\nrequires = ["setuptools>=77", "wheel", "pybind11>=2.12"]\nbuild-backend = "setuptools.build_meta"\n')
license_path = next((p / 'LICENSE' for p in source.parents if (p / 'LICENSE').is_file()), None)
if license_path is not None and license_path.resolve() != (root / 'LICENSE').resolve():
shutil.copyfile(license_path, root / 'LICENSE')
return [str(entry), *map(str, destination.rglob('*.py'))]
|