google

/

gemma-3n-E2B-it

like 229

Follow

Google 33.4k

Hi all, I just noticed that gemma-3n gives huge perplexity value even when a very simple case is given.

Reproduce

import torch

from transformers import AutoProcessor, AutoModelForImageTextToText


model = AutoModelForImageTextToText.from_pretrained(
        "models/gemma-3n-E2B-it",
        torch_dtype=torch.bfloat16,
        attn_implementation="flash_attention_2",
        # attn_implementation="eager",
        device_map='cuda',
    ).eval()
processor = AutoProcessor.from_pretrained("models/gemma-3n-E2B-it", )



messages = [
    {
        "role": "system",
        "content": [{"type": "text", "text": "You are a helpful assistant."}]
    },
    {
        "role": "user",
        "content": [
           {"type": "text", "text": "Hi"}
        ]
    },
    {
        "role": "assistant",
        "content": [
           {"type": "text", "text": "How are you?"}
        ]
    }
]




text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
print(text)
encodings = processor(text=text, images=None, videos=None, padding=False, return_tensors="pt")


input_ids = encodings.input_ids.to('cuda')
target_ids = input_ids.clone()
trg_len = -2
target_ids[:, :trg_len] = -100


with torch.no_grad():
    outputs = model(input_ids, labels=target_ids)

    nll = outputs.loss

ppl = torch.exp(nll)
print(ppl)

Ouptput

tensor(43704.4180, device='cuda:0')

packages:

transformers==4.56.2
torch==2.8.0