Spaces:
Runtime error
Runtime error
File size: 2,006 Bytes
b711dfb e5afc07 b711dfb 79e3cda b711dfb 64e7804 b711dfb 79e3cda b711dfb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | from PIL import Image
from transformers import AutoProcessor, AutoModelForCausalLM, AutoTokenizer
from parler_tts import ParlerTTSForConditionalGeneration
import torch
import soundfile as sf
import subprocess
subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
def generate_image_caption(image_path):
model_name = "microsoft/Florence-2-large"
prompt = "<DETAILED_CAPTION>"
model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
image = Image.open(image_path)
inputs = processor(text=prompt, images=image, return_tensors="pt")
generated_ids = model.generate(
input_ids=inputs["input_ids"],
pixel_values=inputs["pixel_values"],
max_new_tokens=1024,
num_beams=3,
do_sample=False
)
generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
caption = processor.post_process_generation(generated_text, task="<DETAILED_CAPTION>", image_size=(image.width, image.height))
return caption
def convert_text_to_speech(text):
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model_name = "parler-tts/parler_tts_mini_v0.1"
model = ParlerTTSForConditionalGeneration.from_pretrained(model_name, trust_remote_code=True).to(device)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
description = "A female speaker with a slightly low-pitched voice delivers her words quite expressively, in a very confined sounding environment with clear audio quality. She speaks very fast."
input_ids = tokenizer(description, return_tensors="pt").input_ids.to(device)
prompt_input_ids = tokenizer(text, return_tensors="pt").input_ids.to(device)
generation = model.generate(input_ids=input_ids, prompt_input_ids=prompt_input_ids)
audio_arr = generation.cpu().numpy().squeeze()
return audio_arr
|