import ast import io import math import statistics import string import cairosvg import clip import cv2 import kagglehub import matplotlib.pyplot as plt import numpy as np import pandas as pd import torch import torch.nn as nn from more_itertools import chunked from PIL import Image, ImageFilter from transformers import ( AutoProcessor, BitsAndBytesConfig, PaliGemmaForConditionalGeneration, ) svg_constraints = kagglehub.package_import('metric/svg-constraints') class ParticipantVisibleError(Exception): pass def score( solution: pd.DataFrame, submission: pd.DataFrame, row_id_column_name: str, random_seed: int = 0 ) -> float: """Calculates a fidelity score by comparing generated SVG images to target text descriptions. Parameters ---------- solution : pd.DataFrame A DataFrame containing target questions, choices, and answers about an SVG image. submission : pd.DataFrame A DataFrame containing generated SVG strings. Must have a column named 'svg'. row_id_column_name : str The name of the column containing row identifiers. This column is removed before scoring. random_seed : int A seed to set the random state. Returns ------- float The mean fidelity score (a value between 0 and 1) representing the average similarity between the generated SVGs and their descriptions. A higher score indicates better fidelity. Raises ------ ParticipantVisibleError If the 'svg' column in the submission DataFrame is not of string type or if validation of the SVG fails. Examples -------- >>> import pandas as pd >>> solution = pd.DataFrame({ ... 'id': ["abcde"], ... 'question': ['["Is there a red circle?", "What shape is present?"]'], ... 'choices': ['[["yes", "no"], ["square", "circle", "triangle", "hexagon"]]'], ... 'answer': ['["yes", "circle"]'], ... }) >>> submission = pd.DataFrame({ ... 'id': ["abcde"], ... 'svg': [''], ... }) >>> score(solution, submission, 'row_id', random_seed=42) 0... """ # Convert solution fields to list dtypes and expand for colname in ['question', 'choices', 'answer']: solution[colname] = solution[colname].apply(ast.literal_eval) solution = solution.explode(['question', 'choices', 'answer']) # Validate if not pd.api.types.is_string_dtype(submission.loc[:, 'svg']): raise ParticipantVisibleError('svg must be a string.') # Check that SVG code meets defined constraints constraints = svg_constraints.SVGConstraints() try: for svg in submission.loc[:, 'svg']: constraints.validate_svg(svg) except: raise ParticipantVisibleError('SVG code violates constraints.') # Score vqa_evaluator = VQAEvaluator() aesthetic_evaluator = AestheticEvaluator() results = [] rng = np.random.RandomState(random_seed) try: df = solution.merge(submission, on='id') for i, (_, group) in enumerate(df.loc[ :, ['id', 'question', 'choices', 'answer', 'svg'] ].groupby('id')): questions, choices, answers, svg = [ group[col_name].to_list() for col_name in group.drop('id', axis=1).columns ] svg = svg[0] # unpack singleton from list group_seed = rng.randint(0, np.iinfo(np.int32).max) image_processor = ImageProcessor(image=svg_to_png(svg), seed=group_seed).apply() image = image_processor.image.copy() aesthetic_score = aesthetic_evaluator.score(image) vqa_score = vqa_evaluator.score(questions, choices, answers, image) image_processor.reset().apply_random_crop_resize().apply_jpeg_compression(quality=90) ocr_score = vqa_evaluator.ocr(image_processor.image) instance_score = ( harmonic_mean(vqa_score, aesthetic_score, beta=0.5) * ocr_score ) results.append(instance_score) except: raise ParticipantVisibleError('SVG failed to score.') fidelity = statistics.mean(results) return float(fidelity) class VQAEvaluator: """Evaluates images based on their similarity to a given text description using multiple choice questions.""" def __init__(self): self.quantization_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type='nf4', bnb_4bit_use_double_quant=True, bnb_4bit_compute_dtype=torch.float16, ) self.letters = string.ascii_uppercase self.model_path = kagglehub.model_download( 'google/paligemma-2/transformers/paligemma2-10b-mix-448' ) self.processor = AutoProcessor.from_pretrained(self.model_path) self.model = PaliGemmaForConditionalGeneration.from_pretrained( self.model_path, low_cpu_mem_usage=True, quantization_config=self.quantization_config, ).to('cuda') def score(self, questions, choices, answers, image, n=4): scores = [] batches = (chunked(qs, n) for qs in [questions, choices, answers]) for question_batch, choice_batch, answer_batch in zip(*batches, strict=True): scores.extend( self.score_batch( image, question_batch, choice_batch, answer_batch, ) ) return statistics.mean(scores) def score_batch( self, image: Image.Image, questions: list[str], choices_list: list[list[str]], answers: list[str], ) -> list[float]: """Evaluates the image based on multiple choice questions and answers. Parameters ---------- image : PIL.Image.Image The image to evaluate. questions : list[str] List of questions about the image. choices_list : list[list[str]] List of lists of possible answer choices, corresponding to each question. answers : list[str] List of correct answers from the choices, corresponding to each question. Returns ------- list[float] List of scores (values between 0 and 1) representing the probability of the correct answer for each question. """ prompts = [ self.format_prompt(question, choices) for question, choices in zip(questions, choices_list, strict=True) ] batched_choice_probabilities = self.get_choice_probability( image, prompts, choices_list ) scores = [] for i, _ in enumerate(questions): choice_probabilities = batched_choice_probabilities[i] answer = answers[i] answer_probability = 0.0 for choice, prob in choice_probabilities.items(): if choice == answer: answer_probability = prob break scores.append(answer_probability) return scores def format_prompt(self, question: str, choices: list[str]) -> str: prompt = f'answer en Question: {question}\nChoices:\n' for i, choice in enumerate(choices): prompt += f'{self.letters[i]}. {choice}\n' return prompt def mask_choices(self, logits, choices_list): """Masks logits for the first token of each choice letter for each question in the batch.""" batch_size = logits.shape[0] masked_logits = torch.full_like(logits, float('-inf')) for batch_idx in range(batch_size): choices = choices_list[batch_idx] for i in range(len(choices)): letter_token = self.letters[i] first_token = self.processor.tokenizer.encode( letter_token, add_special_tokens=False )[0] first_token_with_space = self.processor.tokenizer.encode( ' ' + letter_token, add_special_tokens=False )[0] if isinstance(first_token, int): masked_logits[batch_idx, first_token] = logits[ batch_idx, first_token ] if isinstance(first_token_with_space, int): masked_logits[batch_idx, first_token_with_space] = logits[ batch_idx, first_token_with_space ] return masked_logits def get_choice_probability(self, image, prompts, choices_list) -> list[dict]: inputs = self.processor( images=[image] * len(prompts), text=prompts, return_tensors='pt', padding='longest', ).to('cuda') with torch.no_grad(): outputs = self.model(**inputs) logits = outputs.logits[:, -1, :] # Logits for the last (predicted) token masked_logits = self.mask_choices(logits, choices_list) probabilities = torch.softmax(masked_logits, dim=-1) batched_choice_probabilities = [] for batch_idx in range(len(prompts)): choice_probabilities = {} choices = choices_list[batch_idx] for i, choice in enumerate(choices): letter_token = self.letters[i] first_token = self.processor.tokenizer.encode( letter_token, add_special_tokens=False )[0] first_token_with_space = self.processor.tokenizer.encode( ' ' + letter_token, add_special_tokens=False )[0] prob = 0.0 if isinstance(first_token, int): prob += probabilities[batch_idx, first_token].item() if isinstance(first_token_with_space, int): prob += probabilities[batch_idx, first_token_with_space].item() choice_probabilities[choice] = prob # Renormalize probabilities for each question total_prob = sum(choice_probabilities.values()) if total_prob > 0: renormalized_probabilities = { choice: prob / total_prob for choice, prob in choice_probabilities.items() } else: renormalized_probabilities = ( choice_probabilities # Avoid division by zero if total_prob is 0 ) batched_choice_probabilities.append(renormalized_probabilities) return batched_choice_probabilities def ocr(self, image, free_chars=4): inputs = ( self.processor( text='ocr\n', images=image, return_tensors='pt', ) .to(torch.float16) .to(self.model.device) ) input_len = inputs['input_ids'].shape[-1] with torch.inference_mode(): outputs = self.model.generate(**inputs, max_new_tokens=32, do_sample=False) outputs = outputs[0][input_len:] decoded = self.processor.decode(outputs, skip_special_tokens=True) num_char = len(decoded) # Exponentially decreasing towards 0.0 if more than free_chars detected return min(1.0, math.exp(-num_char + free_chars)) class AestheticPredictor(nn.Module): def __init__(self, input_size): super().__init__() self.input_size = input_size self.layers = nn.Sequential( nn.Linear(self.input_size, 1024), nn.Dropout(0.2), nn.Linear(1024, 128), nn.Dropout(0.2), nn.Linear(128, 64), nn.Dropout(0.1), nn.Linear(64, 16), nn.Linear(16, 1), ) def forward(self, x): return self.layers(x) class AestheticEvaluator: def __init__(self): self.model_path = 'improved-aesthetic-predictor/sac+logos+ava1-l14-linearMSE.pth' self.clip_model_path = 'ViT-L/14' self.predictor, self.clip_model, self.preprocessor = self.load() def load(self): """Loads the aesthetic predictor model and CLIP model.""" state_dict = torch.load(self.model_path, weights_only=True, map_location='cuda') # CLIP embedding dim is 768 for CLIP ViT L 14 predictor = AestheticPredictor(768) predictor.load_state_dict(state_dict) predictor.to('cuda') predictor.eval() clip_model, preprocessor = clip.load(self.clip_model_path, device='cuda') return predictor, clip_model, preprocessor def score(self, image: Image.Image) -> float: """Predicts the CLIP aesthetic score of an image.""" image = self.preprocessor(image).unsqueeze(0).to('cuda') with torch.no_grad(): image_features = self.clip_model.encode_image(image) # l2 normalize image_features /= image_features.norm(dim=-1, keepdim=True) image_features = image_features.cpu().detach().numpy() score = self.predictor(torch.from_numpy(image_features).to('cuda').float()) return score.item() / 10.0 # scale to [0, 1] def harmonic_mean(a: float, b: float, beta: float = 1.0) -> float: """ Calculate the harmonic mean of two values, weighted using a beta parameter. Args: a: First value (e.g., precision) b: Second value (e.g., recall) beta: Weighting parameter Returns: Weighted harmonic mean """ # Handle zero values to prevent division by zero if a <= 0 or b <= 0: return 0.0 return (1 + beta**2) * (a * b) / (beta**2 * a + b) def svg_to_png(svg_code: str, size: tuple = (384, 384)) -> Image.Image: """ Converts an SVG string to a PNG image using CairoSVG. If the SVG does not define a `viewBox`, it will add one using the provided size. Parameters ---------- svg_code : str The SVG string to convert. size : tuple[int, int], default=(384, 384) The desired size of the output PNG image (width, height). Returns ------- PIL.Image.Image The generated PNG image. """ # Ensure SVG has proper size attributes if 'viewBox' not in svg_code: svg_code = svg_code.replace('