from pathlib import Path import torch from transformers import CLIPModel, CLIPProcessor import piq import torchvision.io as tio import gradio as gr import pandas as pd PROMPT_PAIRS = { ("openai/clip-vit-large-patch14-336", "none"): { "Head without covering": ("The subject is not wearing any type of headgear, and the hair is visible", "The subject is wearing a hat, cap, bandana, or any other garment that hides the hair"), "Gaze in camera": ("The subject is looking straight at the camera, the gaze is frontal", "The subject is looking away, in a direction different from the camera"), "Eyes open": ("The eyes are open and clearly visible, with the iris in view", "The eyes are partially or fully closed, making them difficult to see clearly"), "No/light makeup": ("The subject is either not wearing makeup or has light, natural makeup that does not alter facial features", "The subject is wearing heavy makeup, with bold, unnatural colors that alter facial features"), "Neutral expression": ("The facial expression is natural, does not reveal any particular emotions, and does not alter the facial structure", "The subject's expression reflects an emotion and alters the geometry of the face, mouth, or eyes"), "No sunglasses": ("The subject is not wearing sun glasses that occlude the eyes making them not visible", "The subject wears sun glasses that hide the eyes"), "Frontal pose": ("The head pose is frontal without rotations in terms of roll, pitch and yaw", "The head is rotated, causing the facial features to appear asymmetric in at least one direction"), "Correct exposure": ("The photograph has proper exposure, neither too light nor too dark", "The exposure of the photograph is incorrect, resulting in an image that is either too light or too dark"), "In focus photo": ("The photograph is in focus, and the fine details of the image are clearly visible", "The photograph is blurred and smooth, no fine details are clearly visible"), "Correct saturation": ("The colors in the image are natural, neither too saturated nor too flat", "The colors in the image are unnatural, either too saturated or too faded"), "Uniform background": ("The background is uniform, with no irregularities, even small ones. It may have a slight color gradient in one direction.", "The background is not uniform and shows irregularities, streaks, or a complex background with various objects present"), "Uniform face lighting": ("The lighting on the face is even, with no shadows, not even small ones, including around the nose area. There are no shadows caused by glasses.", "The lighting is uneven, with shadows or areas of the face that are brighter than others"), "No pixelation": ("The image appears detailed and of good resolution, without visible pixelation or discretization effects", "The image is of low quality and highly quantized, with visible pixelation effects"), "No posterization": ("The image is characterized by a good distribution of colors, with no visible discretization effects", "The image shows a posterization effect, with a noticeable discretization of colors"), }, ("openai/clip-vit-large-patch14-336", "handcrafted"): { "Head without covering": ("Hair fully visible, no headwear", "Head covered with a hat, cap, or bandana"), "Gaze in camera": ("Eyes centered, straight at camera", "Eyes off-centered, looking aside"), "Eyes open": ("Both eyes are open", "One or both eyes closed"), "No/light makeup": ("Bare face without obstructions", "Face with obstructive, vibrant makeup"), "Neutral expression": ("Face shows no emotion, no structure change", "Facial features changed by smile"), "No sunglasses": ("Eyes not hidden", "Sunglasses obscure eyes"), "Frontal pose": ("Face symmetry, no tilt or shift", "Asymmetric from any rotation"), "Correct exposure": ("Photo is well-defined, clear image", "Photo is indistinct, lacking clarity"), "In focus photo": ("Photo is crisp and precise", "Photo is blurry and imprecise"), "Correct saturation": ("Image is unmodified", "Image modified or filtered"), "Uniform background": ("Photo looks original, no digital effects", "Photo has digital effects"), "Uniform face lighting": ("Face not altered by light", "Face lighting digitally altered"), "No pixelation": ("Photo is crisp, no noise", "Photo is fuzzy with noise"), "No posterization": ("Photo pristine, no effects", "Photo altered with digital effects"), }, ("openai/clip-vit-large-patch14-336", "requirement"): { "Head without covering": ("Face visible from ear to ear, no distortion", "Ear to ear face covered by headwear"), "Gaze in camera": ("Eyes locked onto the camera", "Gazing away"), "Eyes open": ("Pupils exposed, naturally open eyes", "Pupils hidden, unnaturally closed eyes"), "No/light makeup": ("Natural face", "Heavy makeup"), "Neutral expression": ("Serious look", "Smiling"), "No sunglasses": ("Glasses with clear lenses", "Glasses with tinted lenses"), "Frontal pose": ("Head aligned, camera", "Head tilted away from camera"), "Correct exposure": ("Clear contrast in facial details", "Blurred contrast in facial details"), "In focus photo": ("Clear 1mm detail visibility", "Blurry 1mm detail visibility"), "Correct saturation": ("Srgb standard capture", "Artificial colour capture"), "Uniform background": ("Plain color, no designs", "Designed background"), "Uniform face lighting": ("Even brightness across facial features", "Extreme brightness variation on face"), "No pixelation": ("Portrait in high-resolution raw", "Portrait in low-resolution pdf"), "No posterization": ("Photo meets intensity contrast needs, ear to ear", "Photo fails intensity contrast needs, ear to ear"), }, ("clip-iqa", "none"): { "Head without covering": ("The subject is not wearing any type of headgear, and the hair is visible", "The subject is wearing a hat, cap, bandana, or any other garment that hides the hair"), "Gaze in camera": ("The subject is looking straight at the camera, the gaze is frontal", "The subject is looking away, in a direction different from the camera"), "Eyes open": ("The eyes are open and clearly visible, with the iris in view", "The eyes are partially or fully closed, making them difficult to see clearly"), "No/light makeup": ("The subject is either not wearing makeup or has light, natural makeup that does not alter facial features", "The subject is wearing heavy makeup, with bold, unnatural colors that alter facial features"), "Neutral expression": ("The facial expression is natural, does not reveal any particular emotions, and does not alter the facial structure", "The subject's expression reflects an emotion and alters the geometry of the face, mouth, or eyes"), "No sunglasses": ("The subject is not wearing sun glasses that occlude the eyes making them not visible", "The subject wears sun glasses that hide the eyes"), "Frontal pose": ("The head pose is frontal without rotations in terms of roll, pitch and yaw", "The head is rotated, causing the facial features to appear asymmetric in at least one direction"), "Correct exposure": ("The photograph has proper exposure, neither too light nor too dark", "The exposure of the photograph is incorrect, resulting in an image that is either too light or too dark"), "In focus photo": ("The photograph is in focus, and the fine details of the image are clearly visible", "The photograph is blurred and smooth, no fine details are clearly visible"), "Correct saturation": ("The colors in the image are natural, neither too saturated nor too flat", "The colors in the image are unnatural, either too saturated or too faded"), "Uniform background": ("The background is uniform, with no irregularities, even small ones. It may have a slight color gradient in one direction.", "The background is not uniform and shows irregularities, streaks, or a complex background with various objects present"), "Uniform face lighting": ("The lighting on the face is even, with no shadows, not even small ones, including around the nose area. There are no shadows caused by glasses.", "The lighting is uneven, with shadows or areas of the face that are brighter than others"), "No pixelation": ("The image appears detailed and of good resolution, without visible pixelation or discretization effects", "The image is of low quality and highly quantized, with visible pixelation effects"), "No posterization": ("The image is characterized by a good distribution of colors, with no visible discretization effects", "The image shows a posterization effect, with a noticeable discretization of colors"), }, ("clip-iqa", "handcrafted"): { "Head without covering": ("Hair visible, no coverings", "Headwear hiding hair"), "Gaze in camera": ("Front view, eyes directly at camera", "Side view, eyes elsewhere"), "Eyes open": ("Eyes clearly visible, open", "Eyes obstructed or closed"), "No/light makeup": ("No visible makeup on eyes", "Visible makeup on eyes"), "Neutral expression": ("Unsmiling, neutral expression", "Smiling or frowning"), "No sunglasses": ("No sunglasses worn", "Sunglasses hide eyes"), "Frontal pose": ("Symmetrical face", "Asymmetric due to pose"), "Correct exposure": ("The photo avoids exposure issues, keeping uniformity", "The photo has exposure issues, disrupting uniformity"), "In focus photo": ("Picture has high clarity", "Picture has low clarity"), "Correct saturation": ("Photo colors untouched, natural", "Photo colors unnatural, edited"), "Uniform background": ("Photo is genuine, not manipulated", "Photo is manipulated or modified"), "Uniform face lighting": ("Face fully visible, unobstructed", "Face partially visible, lens shadows"), "No pixelation": ("Sharp, well-defined image", "Image is fuzzy, undefined"), "No posterization": ("Image is sharp, no pixelation", "Image is pixelated"), }, ("clip-iqa", "requirement"): { "Head without covering": ("Eyes and brows visible", "Eyes and brows obscured by headwear"), "Gaze in camera": ("Direct camera gaze, head upright", "Gaze away, head not straight"), "Eyes open": ("Irises not shaded, eyes open", "Irises shaded, eyes closed"), "No/light makeup": ("Face without visible makeup", "Heavy makeup distorting features"), "Neutral expression": ("Unsmiling, steady look", "Big teeth showing, wide smile"), "No sunglasses": ("No sunglasses during passport photo", "Wear sunglasses during passport photo"), "Frontal pose": ("Front pose, eyes front", "Side pose, glance away"), "Correct exposure": ("Clear distinction between face and background", "Face blends with background"), "In focus photo": ("Fine details sharp", "Fine details blurred"), "Correct saturation": ("Naturally rendered colors", "Altered rendered colors"), "Uniform background": ("Clear background, subject distinct", "Subject blending into background"), "Uniform face lighting": ("Distinct forehead lines", "Forehead lines lost in lighting"), "No pixelation": ("Portrait pixel density is consistent", "Portrait pixel density is inconsistent"), "No posterization": ("Photo captures necessary brightness contrast", "Photo fails to capture necessary brightness contrast"), }, } def load_image(filename: Path) -> torch.Tensor: img = tio.read_image(filename, mode="RGB").to(torch.float32) return img.unsqueeze(0) / 255.0 def get_model_and_processor(model_type: str) -> tuple[CLIPModel, CLIPProcessor]: if model_type == "clip-iqa": model = piq.clip_iqa.clip.load().eval() processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch16") else: model = CLIPModel.from_pretrained(model_type).eval() processor = CLIPProcessor.from_pretrained(model_type) return model, processor def encode_prompts(model: CLIPModel, processor: CLIPProcessor, prompt_pairs: list[tuple[str, str]], model_type: str, device: str | torch.device) -> torch.Tensor: with torch.inference_mode(): prompts = [x for pair in prompt_pairs for x in pair] if model_type == "clip-iqa": text_processed = processor(text=prompts) anchors_text = torch.zeros(len(prompts), processor.tokenizer.model_max_length, dtype=torch.long, device=device) for i, tp in enumerate(text_processed["input_ids"]): anchors_text[i, :len(tp)] = torch.tensor(tp, dtype=torch.long, device=device) anchors = model.encode_text(anchors_text).float() else: text_processed = processor(text=prompts, return_tensors="pt", padding=True) anchors = model.get_text_features(text_processed["input_ids"].to(device), text_processed["attention_mask"].to(device)).float() anchors = anchors / anchors.norm(p=2, dim=-1, keepdim=True) anchors = anchors.view(len(prompt_pairs), 2, -1) return anchors def encode_images(model: CLIPModel, processor: CLIPProcessor, images: torch.Tensor, model_type: str, device: str | torch.device) -> torch.Tensor: with torch.inference_mode(): if model_type == "clip-iqa": default_mean = torch.tensor([0.48145466, 0.4578275, 0.40821073], device=device).view(1, 3, 1, 1) default_std = torch.tensor([0.26862954, 0.26130258, 0.27577711], device=device).view(1, 3, 1, 1) images = (images - default_mean) / default_std img_features = model.encode_image(images.float(), pos_embedding=False).float() else: processed_images = processor(images=[i.cpu() for i in images], return_tensors="pt", padding=True, do_rescale=False) img_features = model.get_image_features(processed_images["pixel_values"].to(device)).float() return img_features / img_features.norm(p=2, dim=-1, keepdim=True) def compute_scores(prompts_features: torch.Tensor, images_features: torch.Tensor) -> torch.Tensor: with torch.inference_mode(): n_images = images_features.shape[0] prompt_pairs = prompts_features.shape[0] logits_per_image = 100 * images_features @ prompts_features.view(prompt_pairs * 2, -1).t() probs = logits_per_image.reshape(n_images, -1, 2).softmax(-1)[:, :, 0] return probs def inference(image: Path, model_type: str, prompt_learning_mode: str) -> pd.DataFrame: pairs = PROMPT_PAIRS[(model_type, prompt_learning_mode)] device = "cuda" if torch.cuda.is_available() else "cpu" model, processor = get_model_and_processor(model_type) model.to(device) image_tensor = load_image(image).to(device) encoded_image = encode_images(model, processor, image_tensor, model_type, device) encoded_prompts = encode_prompts(model, processor, list(pairs.values()), model_type, device) scores = compute_scores(encoded_prompts, encoded_image).squeeze(0) return pd.DataFrame([(check, score.item()) for check, score in zip(pairs.keys(), scores)], columns=["Check", "Score"]) interface = gr.Interface( fn=inference, inputs=[ gr.Image(type="filepath", label="Input Image"), gr.Dropdown( choices=[ ("CLIP ViT-L/14@336", "openai/clip-vit-large-patch14-336"), ("CLIP-IQA", "clip-iqa"), ], value="openai/clip-vit-large-patch14-336", label="Model Type", ), gr.Radio( choices=[ ("None", "none"), ("Prompt Learning from handcrafted prompt pairs", "handcrafted"), ("Prompt Learning from ISO/ICAO requirement", "requirement"), ], value="none", label="Prompt Learning Mode", ), ], outputs=gr.BarPlot(x="Check", y="Score", title="Compliance Scores", x_label_angle=45, y_range=[0, 1], height=600), examples=[ ["assets/bkg.png", "openai/clip-vit-large-patch14-336", "none"], ["assets/cap.png", "openai/clip-vit-large-patch14-336", "handcrafted"], ["assets/ceg.png", "openai/clip-vit-large-patch14-336", "requirement"], ["assets/expos.png", "clip-iqa", "none"], ["assets/icao.png", "clip-iqa", "handcrafted"], ["assets/la.png", "clip-iqa", "requirement"], ["assets/light.png", "openai/clip-vit-large-patch14-336", "none"], ["assets/mkup.png", "openai/clip-vit-large-patch14-336", "handcrafted"], ["assets/oof.png", "openai/clip-vit-large-patch14-336", "requirement"], ["assets/pixel.png", "clip-iqa", "none"], ["assets/poster.png", "clip-iqa", "handcrafted"], ["assets/sat.png", "clip-iqa", "requirement"], ["assets/sm.png", "openai/clip-vit-large-patch14-336", "none"], ["assets/sun.png", "openai/clip-vit-large-patch14-336", "handcrafted"], ["assets/tq.png", "openai/clip-vit-large-patch14-336", "requirement"], ], cache_examples=False, title="Towards Fully-Automated and Zero-Shot ISO/ICAO Compliance Verification via CLIP-IQA and Natural Language Processing", description=""" This demo showcases the method presented in the paper "Towards Fully-Automated and Zero-Shot ISO/ICAO Compliance Verification via CLIP-IQA and Natural Language Processing" (Di Domenico et al., 2025), presented at IJCB 2025. The model evaluates the compliance of facial images with respect to the ISO/ICAO 19794-5 and 39794-5 standards for passport photos using CLIP-based models and antonym prompt pairs. As presented in the paper, we focus on two models: OpenAI's [CLIP ViT-L/14@336](https://proceedings.mlr.press/v139/radford21a) (Radford et al.) and [CLIP-IQA](https://ojs.aaai.org/index.php/AAAI/article/view/25353) (Wang et al.), a CLIP-based model originally used for image quality assessment. We also explore three distinct ways of obtaining the antonym prompt pairs: * No prompt learning, using prompts based on the ISO/ICAO requirements that have been manually crafted by an expert in the field. * Prompt learning from the above-mentioned handcrafted prompt pairs, where the antonym pairs are refined through a prompt learning procedure leveraging an LLM. * Prompt learning from the actual ISO/ICAO requirements, where an LLM produces and learns the antonym pairs directly from the official ISO/ICAO requirements. To see the method in action, upload an image or choose an example below, then select the model and prompt learning mode to get the compliance scores. A score near 1 means compliance, while a score near 0 means non-compliance. """, allow_flagging="never", ) interface.launch()