app.py

import gradio as gr
import numpy as np
import matplotlib.pyplot as plt
from transformers import AutoTokenizer, AutoModel
import torch
from matplotlib.colors import LinearSegmentedColormap
import seaborn as sns
import io
from PIL import Image

class TransformerVisualizer:
    def __init__(self, model_name):
        self.model_name = model_name
        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
        self.model = AutoModel.from_pretrained(model_name)
        
    def tokenize(self, sentence):
        # Get tokens without special tokens
        tokens = self.tokenizer.tokenize(sentence)
        return tokens, f"Original sentence: '{sentence}'\nTokenized: {tokens}"
    
    def add_special_tokens(self, tokens):
        # Add special tokens manually to show the process
        tokens_with_special = [self.tokenizer.cls_token] + tokens + [self.tokenizer.sep_token]
        return tokens_with_special, f"With special tokens: {tokens_with_special}"
    
    def get_token_ids(self, sentence):
        # Get token IDs with special tokens included
        inputs = self.tokenizer(sentence, return_tensors="pt")
        token_ids = inputs["input_ids"][0].tolist()
        tokens = self.tokenizer.convert_ids_to_tokens(token_ids)
        
        result = "Token ID Mapping:\n"
        for token, token_id in zip(tokens, token_ids):
            result += f"Token: '{token}', ID: {token_id}\n"
        
        return token_ids, tokens, result
    
    def get_embeddings(self, sentence):
        # Get embeddings
        inputs = self.tokenizer(sentence, return_tensors="pt")
        with torch.no_grad():
            outputs = self.model(**inputs)
        
        # Get the embeddings from the first layer
        embeddings = outputs.last_hidden_state[0].numpy()
        tokens = self.tokenizer.convert_ids_to_tokens(inputs["input_ids"][0])
        
        result = f"Embedding shape: {embeddings.shape}\n"
        result += f"Each token is represented by a {embeddings.shape[1]}-dimensional vector"
        
        # Create embedding heatmap
        fig = plt.figure(figsize=(12, len(tokens) * 0.5))
        
        # Only show first few dimensions to make it readable
        dims = 10
        embedding_subset = embeddings[:, :dims]
        
        # Create a custom colormap
        cmap = LinearSegmentedColormap.from_list("custom_cmap", ["#2596be", "#ffffff", "#e74c3c"])
        
        # Plot heatmap
        sns.heatmap(embedding_subset, 
                   cmap=cmap, 
                   center=0,
                   xticklabels=[f"Dim {i+1}" for i in range(dims)],
                   yticklabels=tokens,
                   annot=False)
        
        plt.title(f"Word Embeddings (first {dims} dimensions)")
        plt.tight_layout()
        
        # Convert plot to image
        buf = io.BytesIO()
        plt.savefig(buf, format='png')
        plt.close(fig)
        buf.seek(0)
        embedding_img = Image.open(buf)
        
        return embeddings, tokens, result, embedding_img
    
    def get_positional_encoding(self, seq_length, d_model=768):
        # Create positional encodings
        position = np.arange(seq_length)[:, np.newaxis]
        div_term = np.exp(np.arange(0, d_model, 2) * -(np.log(10000.0) / d_model))
        
        pos_encoding = np.zeros((seq_length, d_model))
        pos_encoding[:, 0::2] = np.sin(position * div_term)
        pos_encoding[:, 1::2] = np.cos(position * div_term)
        
        result = f"Positional encoding shape: {pos_encoding.shape}\n"
        result += f"Generated for sequence length: {seq_length}"
        
        # Visualize positional encodings
        fig1 = plt.figure(figsize=(12, 6))
        
        # Only show first 20 dimensions to make it readable
        dims_to_show = min(20, d_model)
        
        # Create a custom colormap
        cmap = LinearSegmentedColormap.from_list("custom_cmap", ["#2596be", "#ffffff", "#e74c3c"])
        
        sns.heatmap(pos_encoding[:, :dims_to_show], 
                   cmap=cmap,
                   center=0,
                   xticklabels=[f"Dim {i+1}" for i in range(dims_to_show)],
                   yticklabels=[f"Pos {i+1}" for i in range(seq_length)])
        
        plt.title(f"Positional Encodings (first {dims_to_show} dimensions)")
        plt.xlabel("Embedding Dimension")
        plt.ylabel("Position in Sequence")
        plt.tight_layout()
        
        # Convert plot to image
        buf1 = io.BytesIO()
        plt.savefig(buf1, format='png')
        plt.close(fig1)
        buf1.seek(0)
        pos_encoding_img = Image.open(buf1)
        
        # Plot sine waves for a few dimensions
        fig2 = plt.figure(figsize=(12, 6))
        
        dims_to_plot = [0, 2, 4, 20, 100]
        for i, dim in enumerate(dims_to_plot):
            if dim < pos_encoding.shape[1]:
                plt.plot(pos_encoding[:, dim], label=f"Dim {dim} (sin)")
        
        plt.title("Positional Encoding Sine Waves")
        plt.xlabel("Position")
        plt.ylabel("Value")
        plt.legend()
        plt.grid(True)
        plt.tight_layout()
        
        # Convert plot to image
        buf2 = io.BytesIO()
        plt.savefig(buf2, format='png')
        plt.close(fig2)
        buf2.seek(0)
        pos_waves_img = Image.open(buf2)
        
        return result, pos_encoding_img, pos_waves_img

def process_text(sentence, model_name):
    visualizer = TransformerVisualizer(model_name)
    
    # 1. Tokenization
    tokens, tokenization_text = visualizer.tokenize(sentence)
    
    # 2. Special Tokens
    tokens_with_special, special_tokens_text = visualizer.add_special_tokens(tokens)
    
    # 3. Token IDs
    token_ids, tokens, token_ids_text = visualizer.get_token_ids(sentence)
    
    # 4. Word Embeddings
    embeddings, tokens, embeddings_text, embedding_img = visualizer.get_embeddings(sentence)
    
    # 5. Positional Encoding
    pos_encoding_text, pos_encoding_img, pos_waves_img = visualizer.get_positional_encoding(len(token_ids))
    
    return (tokenization_text, special_tokens_text, token_ids_text, 
            embeddings_text, embedding_img, pos_encoding_text, 
            pos_encoding_img, pos_waves_img)

# Create Gradio interface
models = [
    "bert-base-uncased", 
    "roberta-base", 
    "distilbert-base-uncased", 
    "gpt2",
    "albert-base-v2",
    "xlm-roberta-base"
]

with gr.Blocks(title="Transformer Process Visualizer") as demo:
    gr.Markdown("# Transformer Process Visualizer")
    gr.Markdown("This app visualizes the key processes in transformer models: tokenization, special tokens, token IDs, word embeddings, and positional encoding.")
    
    with gr.Row():
        with gr.Column():
            input_text = gr.Textbox(
                label="Input Sentence", 
                placeholder="Enter a sentence to visualize transformer processes",
                value="The transformer architecture revolutionized natural language processing."
            )
            model_dropdown = gr.Dropdown(
                label="Select Model", 
                choices=models,
                value="bert-base-uncased"
            )
            submit_btn = gr.Button("Visualize")
    
    with gr.Tabs():
        with gr.TabItem("Tokenization"):
            tokenization_output = gr.Textbox(label="Tokenization")
        
        with gr.TabItem("Special Tokens"):
            special_tokens_output = gr.Textbox(label="Special Tokens")
        
        with gr.TabItem("Token IDs"):
            token_ids_output = gr.Textbox(label="Token IDs")
        
        with gr.TabItem("Word Embeddings"):
            embeddings_output = gr.Textbox(label="Embeddings Info")
            embedding_plot = gr.Image(label="Embedding Visualization")
        
        with gr.TabItem("Positional Encoding"):
            pos_encoding_output = gr.Textbox(label="Positional Encoding Info")
            pos_encoding_plot = gr.Image(label="Positional Encoding Heatmap")
            pos_waves_plot = gr.Image(label="Positional Encoding Waves")
    
    submit_btn.click(
        process_text,
        inputs=[input_text, model_dropdown],
        outputs=[
            tokenization_output,
            special_tokens_output,
            token_ids_output,
            embeddings_output,
            embedding_plot,
            pos_encoding_output,
            pos_encoding_plot,
            pos_waves_plot
        ]
    )

demo.launch()