Workpam commited on
Commit
2da178a
·
verified ·
1 Parent(s): 32ba8bc

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +28 -82
app.py CHANGED
@@ -1,94 +1,40 @@
1
  import gradio as gr
 
 
2
  from PIL import Image
3
  import requests
4
- import base64
5
- import io
6
- import os
7
-
8
- import torch
9
- from transformers import CLIPProcessor, CLIPModel
10
- from openai import OpenAI
11
-
12
- # Load OpenAI client
13
- client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
14
-
15
- # Load CLIP model from Hugging Face
16
- clip_model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
17
- clip_processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
18
-
19
-
20
- def extract_text_with_ocr_space(image):
21
- buffered = io.BytesIO()
22
- image.save(buffered, format="JPEG")
23
- img_str = base64.b64encode(buffered.getvalue()).decode()
24
-
25
- # Use your OCR.Space API key from Hugging Face Secrets
26
- api_key = os.getenv("K86540556188957")
27
 
28
- response = requests.post(
29
- "https://api.ocr.space/parse/image",
30
- data={
31
- "base64Image": f"data:image/jpeg;base64,{img_str}",
32
- "language": "eng",
33
- "isOverlayRequired": False,
34
- "apikey": api_key
35
- }
36
- )
37
- result = response.json()
38
- return result['ParsedResults'][0]['ParsedText'] if "ParsedResults" in result else ""
39
 
 
 
 
40
 
 
41
  def generate_alt_text(image):
42
- # Step 1: OCR
43
- ocr_text = extract_text_with_ocr_space(image)
44
 
45
- # Step 2: CLIP visual descriptions
46
- candidate_descriptions = [
47
- "a manga cover with a male character",
48
- "a man wearing a summer floral shirt",
49
- "a character with a disinterested expression",
50
- "a man holding a beach float and a fan",
51
- "an anime illustration with summer theme",
52
- "a comic book style drawing",
53
- "a beach or summer activity scene"
54
- ]
55
 
56
- inputs = clip_processor(text=candidate_descriptions, images=image, return_tensors="pt", padding=True)
57
- outputs = clip_model(**inputs)
58
- probs = outputs.logits_per_image.softmax(dim=1)
59
- sorted_descriptions = sorted(zip(candidate_descriptions, probs[0].tolist()), key=lambda x: x[1], reverse=True)
60
 
61
- # Step 3: Prompt construction
62
- top_descriptions = "\n".join([f"- {desc}" for desc, _ in sorted_descriptions[:5]])
63
- prompt = f"""
64
- Generate a detailed and professional alt text for a manga or anime image using the following extracted text and visual description.
65
-
66
- OCR Text:
67
- {ocr_text}
68
-
69
- Visual Description:
70
- {top_descriptions}
71
-
72
- Alt Text:
73
- """
74
-
75
- # Step 4: GPT generation
76
- response = client.chat.completions.create(
77
- model="gpt-3.5-turbo", # change from gpt-4
78
- messages=[
79
- {"role": "system", "content": "You are a helpful assistant."},
80
- {"role": "user", "content": prompt}
81
- ]
82
- )
83
-
84
- return response.choices[0].message.content
85
-
86
-
87
- # Gradio app interface
88
- gr.Interface(
89
  fn=generate_alt_text,
90
- inputs=gr.Image(type="pil"),
91
  outputs=gr.Textbox(label="Generated Alt Text"),
92
- title="Anime/Manga Image Alt Text Generator",
93
- description="Upload a manga or anime-style image to generate a descriptive alt text using OCR, CLIP, and GPT"
94
- ).launch()
 
 
 
 
1
  import gradio as gr
2
+ from transformers import VisionEncoderDecoderModel, ViTImageProcessor, AutoTokenizer
3
+ import torch
4
  from PIL import Image
5
  import requests
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
 
7
+ # Load model & processor
8
+ model_name = "nlpconnect/vit-gpt2-image-captioning"
9
+ model = VisionEncoderDecoderModel.from_pretrained(model_name)
10
+ feature_extractor = ViTImageProcessor.from_pretrained(model_name)
11
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
 
 
 
 
 
 
12
 
13
+ # Use GPU if available
14
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
15
+ model.to(device)
16
 
17
+ # Caption generation function
18
  def generate_alt_text(image):
19
+ if image is None:
20
+ return "No image uploaded"
21
 
22
+ image = image.convert("RGB")
23
+ pixel_values = feature_extractor(images=image, return_tensors="pt").pixel_values
24
+ pixel_values = pixel_values.to(device)
 
 
 
 
 
 
 
25
 
26
+ output_ids = model.generate(pixel_values, max_length=64, num_beams=4)
27
+ caption = tokenizer.decode(output_ids[0], skip_special_tokens=True)
28
+ return caption.strip().capitalize()
 
29
 
30
+ # Gradio UI
31
+ iface = gr.Interface(
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  fn=generate_alt_text,
33
+ inputs=gr.Image(type="pil", label="Upload an Image"),
34
  outputs=gr.Textbox(label="Generated Alt Text"),
35
+ title="Image Alt Text Generator",
36
+ description="Upload an image and get a detailed, AI-generated alt text using Hugging Face models"
37
+ )
38
+
39
+ # Launch app
40
+ iface.launch()