import os from typing import Any from sentence_transformers import SentenceTransformer import pytube from whisper_jax import FlaxWhisperPipline import jax.numpy as jnp import mimetypes class EndpointHandler(): def __init__(self, path=""): self.model = SentenceTransformer('sentence-transformers/paraphrase-multilingual-mpnet-base-v2') self.pipeline = FlaxWhisperPipline("openai/whisper-tiny", dtype=jnp.float16, batch_size=16) def __call__(self, data: Any) -> Any: """ Args: data (:obj:): includes the input data and the parameters for the inference. Return: A :obj:`list`:. The list contains the embeddings of the inference inputs """ if data.get("task") == "transcribe": yt_id = data.get("inputs") if yt_id is None: return {"error": "inputs is required, should contain youtube id or youtube video url"} yt_url = 'https://youtu.be/' + yt_id filename = yt_id + ".m4a" # first try to load captions from YT yt = pytube.YouTube(yt_url) captions = yt.captions if captions: for c in captions: caption = yt.captions.get_by_language_code(c.code) return {"transcript": caption.generate_srt_captions()} if not os.path.isfile(filename): try: stream = yt.streams.filter(only_audio=True)[0] extension = mimetypes.guess_extension(stream.mime_type) if extension: filename = yt_id + extension stream.download(filename=filename) except KeyError: return {"error":"An error occurred while loading the YouTube video. Please try again."} try: if os.path.isfile(filename): text = self.pipeline(filename, task="transcribe", return_timestamps=True) else: text = "file not found" except Exception as e: return {"error": "An error occurred while transcribing the audio. Please try again."+ str(e)} finally: if os.path.isfile(filename): os.remove(filename) return { "transcript": text} else: inputs = data.get("inputs") embeddings = self.model.encode(inputs) return {"embeddings": embeddings.tolist()}