Buckets:

download
raw
3.2 kB
import itertools
import sys
import time
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
from typing import ClassVar, List
from filelock import FileLock
from grpc_tools import protoc
from nos.constants import NOS_CACHE_DIR, NOS_PATH
from nos.logging import logger
PROTO_PATHS = [NOS_PATH / "proto"]
logger.disable(__name__)
@dataclass
class DynamicProtobufCompiler:
_instance = None
"""Singleton instance."""
cache_dir: ClassVar[str] = NOS_CACHE_DIR / "protobuf"
"""Cache directory for compiled protobuf modules."""
def __init__(self) -> None:
self.cache_dir.mkdir(parents=True, exist_ok=True)
# Add cache dir to sys.path
sys.path.append(str(self.cache_dir))
# Compile all proto files from all paths
logger.debug(f"Compiling protos [dirs={PROTO_PATHS}]")
for path in itertools.chain.from_iterable(Path(path).glob("*.proto") for path in PROTO_PATHS):
logger.debug(f"Compiling ... [filename={path}]")
self.compile(str(path))
logger.debug(f"Compiled modules [modules={self.list_modules()}]")
@classmethod
def get(cls: "DynamicProtobufCompiler") -> "DynamicProtobufCompiler":
"""Get DynamicProtobufCompiler."""
if cls._instance is None:
cls._instance = cls()
return cls._instance
def compile(self, proto_filename: str):
"""Compile the proto file to generate the Python modules"""
logger.debug(f"Compiling proto [proto={proto_filename}]")
cmd = [
"",
"-I" + str(Path(protoc.__file__).parent / "_proto/"),
f"--python_out={self.cache_dir}",
f"--grpc_python_out={self.cache_dir}",
f"--proto_path={Path(proto_filename).parent}",
f"{Path(proto_filename).name}",
]
logger.debug(f"Compiling proto [cmd=protoc {' '.join(cmd)}]")
st = time.time()
protoc.main(cmd)
logger.debug(f"Compilation done [elapsed={(time.time() - st)*1e3:.1f}ms]")
def list_modules(self) -> List[str]:
"""Return a list of compiled modules."""
return [Path(path).stem for path in self.cache_dir.glob("*_pb2*.py")]
def import_module(self, module_name: str):
"""Import the specified module and return the imported module object."""
import importlib.util
# Load the module
st = time.time()
module_path = f"{Path(self.cache_dir) / module_name}.py"
logger.debug(f"Loading module [module={module_path}]")
spec = importlib.util.spec_from_file_location(module_name, module_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
logger.debug(f"Loaded module [module={module_path}, elapsed={(time.time() - st)*1e3:.1f} ms]")
return module
@lru_cache(maxsize=None)
def import_module(module_name: str):
"""Import the specified module and return the imported module object."""
with FileLock(str(Path(str(DynamicProtobufCompiler.cache_dir)) / "protoc.lock")):
compiler = DynamicProtobufCompiler.get()
return compiler.import_module(module_name)

Xet Storage Details

Size:
3.2 kB
·
Xet hash:
0c40e8eab2541c9ca678cf2aa9265f895f44be427d7a034bf045f35b2e966e93

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.