Spaces:
Running
Running
Download generate.py from davidyang5/VisualAnnotation: direct link, hf CLI and curl.
- Browser
- Download file 10.6 kB
-
https://huggingface.co/spaces/davidyang5/VisualAnnotation/resolve/main/generate.py
- Command line
-
hf download hf://spaces/davidyang5/VisualAnnotation/generate.py
-
curl -L -o generate.py https://huggingface.co/spaces/davidyang5/VisualAnnotation/resolve/main/generate.py
10.6 kB
| import argparse | |
| import torch | |
| import matplotlib | |
| matplotlib.use('Agg') | |
| import matplotlib.pyplot as plt | |
| from PIL import Image | |
| import torchvision.transforms as transforms | |
| import random | |
| import numpy as np | |
| from torchvision.datasets import CocoCaptions | |
| import matplotlib.gridspec as gridspec | |
| import os | |
| from tqdm import tqdm | |
| from model import EncoderCNN, CaptionTransformer, ImageCaptioningModel | |
| from data_utils import Vocabulary, get_transforms | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Generate Image Captions") | |
| parser.add_argument("--model_path", type=str, default="caption_model.pt", | |
| help="Path to trained model (default: caption_model.pt)") | |
| parser.add_argument("--vocab_path", type=str, default="vocab.pkl", | |
| help="Path to vocabulary file (default: vocab.pkl)") | |
| parser.add_argument("--sample_img_dir", type=str, required=True, | |
| help="Path to sample images directory") | |
| parser.add_argument("--sample_ann_file", type=str, default="coco_sample/captions_val_sample.json", | |
| help="Path to sample annotations JSON file (only used if --sample_img_dir is coco_sample/images)") | |
| parser.add_argument("--num_samples", type=int, default=6, | |
| help="Maximum number of samples to display") | |
| parser.add_argument("--beam_size", type=int, default=5, | |
| help="Beam size for beam search") | |
| parser.add_argument("--temperature", type=float, default=1.0, | |
| help="Sampling temperature") | |
| parser.add_argument("--output", type=str, | |
| help="Path to save output image. If not provided, will use predictions.png") | |
| args = parser.parse_args() | |
| # Set default output filename | |
| if args.output is None: | |
| args.output = "predictions.png" | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| print(f"Using device: {device}") | |
| try: | |
| vocab = Vocabulary.load(args.vocab_path) | |
| print(f"Loaded vocabulary with {len(vocab)} tokens from {args.vocab_path}") | |
| except Exception as e: | |
| print(f"Error loading vocabulary: {e}") | |
| return | |
| encoder = EncoderCNN(embed_size=512) | |
| decoder = CaptionTransformer(vocab_size=len(vocab)) | |
| model = ImageCaptioningModel(encoder, decoder) | |
| try: | |
| model.load_state_dict(torch.load(args.model_path, map_location=device)) | |
| model = model.to(device) | |
| model.eval() | |
| print(f"Model loaded from {args.model_path}") | |
| except Exception as e: | |
| print(f"Error loading model: {e}") | |
| return | |
| model_transform = get_transforms(split="val") | |
| image_paths = [] | |
| gt_captions_map = {} | |
| use_coco_dataset = args.sample_img_dir.startswith("coco_sample") | |
| if use_coco_dataset: | |
| image_root = args.sample_img_dir | |
| if not image_root.endswith('images'): | |
| image_root = os.path.join(image_root, 'images') | |
| annotation_file = args.sample_ann_file | |
| print(f"Attempting to load COCO dataset: images from '{image_root}', annotations from '{annotation_file}'") | |
| try: | |
| if not os.path.isdir(image_root): | |
| print(f"Error: Image directory not found: {image_root}") | |
| return | |
| sample_dataset = CocoCaptions( | |
| root=image_root, | |
| annFile=annotation_file, | |
| transform=None | |
| ) | |
| print(f"Loaded COCO sample dataset with {len(sample_dataset)} images initially from annotation file.") | |
| available_img_ids = [] | |
| if not sample_dataset.ids: | |
| print("No image IDs found in the annotation file.") | |
| return | |
| print("Checking for available image files on disk...") | |
| for img_id in sample_dataset.ids: | |
| try: | |
| img_info = sample_dataset.coco.loadImgs(img_id)[0] | |
| img_file_path = os.path.join(image_root, img_info['file_name']) | |
| if os.path.exists(img_file_path): | |
| available_img_ids.append(img_id) | |
| except Exception as e: | |
| print(f"Warning: Could not process img_id {img_id} from annotations: {e}") | |
| continue | |
| if not available_img_ids: | |
| print("No valid image files found on disk corresponding to the annotations.") | |
| return | |
| print(f"Found {len(available_img_ids)} available images on disk.") | |
| num_to_sample = min(args.num_samples, 6, len(available_img_ids)) | |
| if num_to_sample == 0: | |
| print("No images available to sample after checking disk.") | |
| return | |
| selected_img_ids = random.sample(available_img_ids, num_to_sample) | |
| for img_id in selected_img_ids: | |
| img_info = sample_dataset.coco.loadImgs(img_id)[0] | |
| img_path = os.path.join(image_root, img_info['file_name']) | |
| image_paths.append(img_path) | |
| ann_ids = sample_dataset.coco.getAnnIds(imgIds=img_id) | |
| anns = sample_dataset.coco.loadAnns(ann_ids) | |
| gt_caps_for_img = [ann['caption'] for ann in anns] | |
| if gt_caps_for_img: | |
| gt_captions_map[img_path] = random.choice(gt_caps_for_img) | |
| else: | |
| gt_captions_map[img_path] = "N/A (no captions found for this image)" | |
| except FileNotFoundError: | |
| print(f"Error: Annotation file not found or critical issue during initial load: {annotation_file}") | |
| return | |
| except Exception as e: | |
| print(f"Error loading COCO sample dataset: {e}") | |
| return | |
| else: | |
| print(f"Loading images directly from custom directory: {args.sample_img_dir}") | |
| try: | |
| if not os.path.isdir(args.sample_img_dir): | |
| print(f"Error: Directory not found: {args.sample_img_dir}") | |
| return | |
| all_files = [os.path.join(args.sample_img_dir, f) for f in os.listdir(args.sample_img_dir)] | |
| image_files = [f for f in all_files if os.path.isfile(f) and f.lower().endswith(('.png', '.jpg', '.jpeg'))] | |
| if not image_files: | |
| print(f"No image files (.png, .jpg, .jpeg) found in {args.sample_img_dir}") | |
| return | |
| print(f"Found {len(image_files)} images.") | |
| num_available = len(image_files) | |
| num_samples = min(args.num_samples, 6, num_available) | |
| image_paths = random.sample(image_files, num_samples) | |
| except Exception as e: | |
| print(f"Error listing or sampling images from {args.sample_img_dir}: {e}") | |
| return | |
| if not image_paths: | |
| print("No images selected for processing.") | |
| return | |
| num_samples = len(image_paths) | |
| if num_samples == 0: | |
| print("No images to display.") | |
| return | |
| elif num_samples <= 3: | |
| nrows, ncols = 1, num_samples | |
| else: | |
| nrows = 2 | |
| ncols = (num_samples + 1) // 2 | |
| fig = plt.figure(figsize=(5*ncols, 5*nrows)) | |
| gs = gridspec.GridSpec(nrows, ncols, figure=fig) | |
| print(f"Generating captions for {num_samples} images...") | |
| # Wrap image_paths with tqdm for a progress bar | |
| for i, img_path in enumerate(tqdm(image_paths, desc="Generating Captions")): | |
| # Calculate grid position | |
| row, col = i // ncols, i % ncols | |
| ax = fig.add_subplot(gs[row, col]) | |
| try: | |
| # Load original image using PIL | |
| original_img = Image.open(img_path).convert("RGB") | |
| except Exception as e: | |
| print(f"Error opening image {img_path}: {e}") | |
| ax.set_title(f"Error loading image: {os.path.basename(img_path)}", fontsize=9) | |
| ax.axis('off') | |
| continue # Skip to next image | |
| # Apply transform only for model input | |
| model_input = model_transform(original_img).unsqueeze(0).to(device) | |
| # Generate caption | |
| with torch.no_grad(): # Ensure no gradients are calculated | |
| # model.generate now returns a list of (tokens_list, score) tuples, sorted by score | |
| beam_results = model.generate( | |
| model_input, | |
| vocab, | |
| max_len=30, | |
| beam_size=args.beam_size, | |
| temperature=args.temperature | |
| ) | |
| title_parts = [] | |
| # Ground Truth | |
| if use_coco_dataset: | |
| gt_caption = gt_captions_map.get(img_path, "./img") | |
| # Clean GT caption tokens | |
| gt_caption_cleaned = ' '.join([t for t in gt_caption.split() if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}]) # Assuming gt_caption is a string | |
| title_parts.append(f"GT: {gt_caption_cleaned}") | |
| if beam_results: | |
| # Best predicted caption | |
| best_pred_tokens, best_pred_score = beam_results[0] | |
| best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}]) | |
| title_parts.append(f"Best Pred (Score: {best_pred_score:.2f}): {best_pred_caption}") | |
| # Other beam search results | |
| if len(beam_results) > 1: | |
| title_parts.append("--- Other Beams ---") | |
| for i, (tokens, score) in enumerate(beam_results[1:]): | |
| if i < args.beam_size -1 : # Show up to beam_size-1 other beams | |
| other_caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}]) | |
| title_parts.append(f"Beam {i+1} (Score: {score:.2f}): {other_caption}") | |
| else: | |
| break | |
| else: | |
| title_parts.append("Pred: No caption generated") | |
| # Display original image and set title | |
| ax.imshow(np.array(original_img)) # Use np.array to ensure correct display | |
| ax.set_title("\n".join(title_parts), fontsize=8) # Adjust fontsize if needed | |
| ax.axis('off') | |
| plt.tight_layout(h_pad=3.0) | |
| # Save to predictions.png | |
| plt.savefig(args.output, bbox_inches='tight', dpi=200) | |
| print(f"Output saved to {args.output}") | |
| if __name__ == "__main__": | |
| main() | |