adding application file

README.md

@@ -8,6 +8,7 @@ sdk_version: 4.17.0
 app_file: app.py
 pinned: false
 license: mit
+python_version: 3.8
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,240 @@
+import json
+from typing import List, Optional
+
+import gradio as gr
+from gradio.components import Markdown
+
+from src.dto.dto import ExplanationGranularity, ExplanationDto
+from src.utils.registry import EXPLAINERS, MODELS, PERTURBERS, COMPARATORS
+from src.utils.segregate import PercentileBasedSegregator
+from src.utils.visualizer import Visualizer
+
+
+class MockExplainerUI:
+    def __init__(
+        self,
+        logo_path: str,
+        css_path: str,
+        visualizer: Visualizer,
+        window_title: str,
+        title: str,
+        examples: Optional[List[str]] = None,
+    ):
+        self.__logo_path = logo_path
+        self.__css_path = css_path
+        self.__examples = examples
+        self.__window_title = window_title
+        self.__title = title
+        self.__visualizer = visualizer
+
+        self.app: gr.Blocks = self.build_app()
+
+    def build_app(self):
+        with gr.Blocks(
+            theme=gr.themes.Monochrome().set(
+                button_primary_background_fill="#009374",
+                button_primary_background_fill_hover="#009374C4",
+                checkbox_label_background_fill_selected="#028A6EFF",
+            ),
+            css=self.__css_path,
+            title=self.__window_title,
+        ) as demo:
+            self.__build_app_title()
+            (
+                user_input,
+                system_response,
+                granularity,
+                upper_percentile,
+                middle_percentile,
+                lower_percentile,
+                explainer_name,
+                model_name,
+                perturber_name,
+                comparator_name,
+                generator_vis,
+                submit_btn,
+            ) = self.__build_chat_and_explain()
+
+            submit_btn.click(
+                fn=self.run,
+                inputs=[
+                    user_input,
+                    granularity,
+                    upper_percentile,
+                    middle_percentile,
+                    lower_percentile,
+                    explainer_name,
+                    model_name,
+                    perturber_name,
+                    comparator_name,
+                ],
+                outputs=[system_response, generator_vis],
+            )
+
+        return demo
+
+    def run(
+        self,
+        user_input: str,
+        granularity: ExplanationGranularity,
+        upper_percentile: str,
+        middle_percentile: str,
+        lower_percentile: str,
+        explainer_name: str,
+        model_name: str,
+        perturber_name: str,
+        comparator_name: str,
+    ):
+        print(user_input)
+        with open(
+            "data/en_q_001.json",
+            "r",
+        ) as f:
+            data = json.load(f)
+        data = data[0]
+        explanation_dto = ExplanationDto.parse_obj(data)
+        user_input = explanation_dto.input_text
+
+        system_response = explanation_dto.output_text
+        generator_vis = self.__visualize_explanations(
+            user_input=user_input,
+            system_response=system_response,
+            generator_explanations=explanation_dto,
+            upper_percentile=int(upper_percentile),
+            middle_percentile=int(middle_percentile),
+            lower_percentile=int(lower_percentile),
+        )
+        return system_response, generator_vis
+
+    def __build_app_title(self):
+        with gr.Row():
+            with gr.Column(min_width=50, scale=1):
+                gr.Image(
+                    value=self.__logo_path,
+                    width=50,
+                    height=50,
+                    show_download_button=False,
+                    container=False,
+                )
+            with gr.Column(scale=2):
+                Markdown(
+                    f'<p style="text-align: left; font-size:200%; font-weight: bold"'
+                    f">{self.__title}"
+                    f"</p>"
+                )
+
+    def __build_chat_and_explain(self):
+        with gr.Row():
+            with gr.Column(scale=2):
+                gr.Textbox(
+                    label="Attention!",
+                    value="This is a demo version of the tool! For running the full version, please follow the instructions in ...",
+                    container=False,
+                    interactive=False,
+                )
+
+        with gr.Row():
+            with gr.Column(scale=2):
+                user_input = gr.Radio(
+                    # placeholder="Type your question here and press Enter.",
+                    label="Question",
+                    container=True,
+                    choices=["Question 1 EN", "Question 1 DE"],
+                )
+            with gr.Column(scale=1):
+                granularity = gr.Radio(
+                    choices=[e for e in ExplanationGranularity],
+                    value=ExplanationGranularity.SENTENCE_LEVEL,
+                    label="Explanation Granularity",
+                )
+
+        with gr.Accordion(label="Settings", open=False, elem_id="accordion"):
+            with gr.Row(variant="compact"):
+                explainer_name = gr.Radio(
+                    label="Explainer",
+                    choices=list(EXPLAINERS.keys()),
+                    value=list(EXPLAINERS.keys())[0],
+                    container=True,
+                )
+            with gr.Row(variant="compact"):
+                upper_percentile = gr.Textbox(label="Upper", value="85", container=True)
+                middle_percentile = gr.Textbox(
+                    label="Middle", value="75", container=True
+                )
+                lower_percentile = gr.Textbox(label="Lower", value="10", container=True)
+
+            with gr.Row(variant="compact"):
+                model_name = gr.Radio(
+                    label="Model",
+                    choices=list(MODELS.keys()),
+                    value=list(MODELS.keys())[0],
+                    container=True,
+                )
+            with gr.Row(variant="compact"):
+                perturber_name = gr.Radio(
+                    label="Perturber",
+                    choices=list(PERTURBERS.keys()),
+                    value=list(PERTURBERS.keys())[0],
+                    container=True,
+                )
+            with gr.Row(variant="compact"):
+                comparator_name = gr.Radio(
+                    label="Comparator",
+                    choices=list(COMPARATORS.keys()),
+                    value=list(COMPARATORS.keys())[0],
+                    container=True,
+                )
+        with gr.Row(variant="compact"):
+            # passing "elem_id" to use a custom style for the component
+            # in the CSS passed.
+            submit_btn = gr.Button(
+                value="🛠 Submit",
+                variant="secondary",
+                elem_id="button",
+                interactive=True,
+            )
+
+        with gr.Row():
+            generator_vis = gr.HTML(label="Explanations")
+
+        with gr.Row():
+            system_response = gr.Textbox(
+                label="System Response",
+                container=True,
+                interactive=False,
+            )
+
+        return (
+            user_input,
+            system_response,
+            granularity,
+            upper_percentile,
+            middle_percentile,
+            lower_percentile,
+            explainer_name,
+            model_name,
+            perturber_name,
+            comparator_name,
+            generator_vis,
+            submit_btn,
+        )
+
+    def __visualize_explanations(
+        self,
+        user_input: str,
+        system_response: Optional[str],
+        generator_explanations: ExplanationDto,
+        upper_percentile: Optional[int],
+        middle_percentile: Optional[int],
+        lower_percentile: Optional[int],
+    ) -> str:
+        segregator = PercentileBasedSegregator(
+            upper_bound_percentile=upper_percentile,
+            middle_bound_percentile=middle_percentile,
+            lower_bound_percentile=lower_percentile,
+        )
+        return self.__visualizer.visualize(
+            segregator=segregator,
+            explanations=generator_explanations,
+            output_from_explanations=user_input,
+        )

requirements.txt

@@ -0,0 +1,3 @@
+gradio==4.17.0
+pydantic~=1.8.2
+numpy~=1.22.4

src/dto/dto.py

@@ -0,0 +1,27 @@
+from enum import Enum
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+
+class FeatureImportance(BaseModel):
+    feature: str
+    score: float
+    token_field: Optional[str] = None
+
+
+class ExplanationDto(BaseModel):
+    explanations: List[FeatureImportance]
+    input_text: str
+    output_text: str
+
+
+class ExplanationGranularity(str, Enum):
+    WORD_LEVEL = "word_level_granularity"
+    SENTENCE_LEVEL = "sentence_level_granularity"
+    PARAGRAPH_LEVEL = "paragraph_level_granularity"
+    PHRASE_LEVEL = "phrase_level_granularity"
+
+
+class SimilarityMetric(Enum):
+    COSINE = "cosine"

src/utils/registry.py

@@ -0,0 +1,22 @@
+COMPARATORS = {
+    "base_llm_based_comparator": ...,
+    "sentence_transformers_based_comparator": ...,
+    "levenshtein_comparator": ...,
+    "jaro_winkler_comparator": ...,
+    "n_gram_comparator": ...,
+}
+
+EXPLAINERS = {
+    "generic_explainer": "generic_explainer",
+}
+
+MODELS = {"flan-t5-xxl": "flan-t5-xxl"}
+
+PERTURBERS = {
+    "leave_one_out": ...,
+    "random_word_perturber": ...,
+    "reorder_perturber": ...,
+    "antonym_perturber": ...,
+    "synonym_perturber": ...,
+    "entity_perturber": ...,
+}

src/utils/segregate.py

@@ -0,0 +1,54 @@
+from abc import ABC, abstractmethod
+from typing import Tuple, List
+
+import numpy as np
+
+from src.dto.dto import ExplanationDto
+
+
+class Segregator(ABC):
+    @abstractmethod
+    def segregate(
+        self, explanations: ExplanationDto
+    ) -> Tuple[List[str], List[str], List[str]]:
+        ...
+
+
+class PercentileBasedSegregator(Segregator):
+    def __init__(
+        self,
+        upper_bound_percentile: int = 85,
+        middle_bound_percentile: int = 75,
+        lower_bound_percentile: int = 10,
+    ):
+        self.__upper_bound_percentile = upper_bound_percentile
+        self.__middle_bound_percentile = middle_bound_percentile
+        self.__lower_bound_percentile = lower_bound_percentile
+
+    def segregate(
+        self,
+        explanations: ExplanationDto,
+    ) -> Tuple[List[str], List[str], List[str]]:
+        scores = [explanation.score for explanation in explanations.explanations]
+        scores = np.asarray(scores)
+        upper_bound = np.percentile(scores, self.__upper_bound_percentile)
+        mid_bound = np.percentile(scores, self.__middle_bound_percentile)
+        lower_bound = np.percentile(scores, self.__lower_bound_percentile)
+
+        pos_features = [
+            explanation.feature
+            for explanation in explanations.explanations
+            if explanation.score >= upper_bound and explanation.score != 0
+        ]
+        mid_features = [
+            explanation.feature
+            for explanation in explanations.explanations
+            if upper_bound > explanation.score >= mid_bound > 0
+        ]
+        low_features = [
+            explanation.feature
+            for explanation in explanations.explanations
+            if mid_bound > explanation.score >= lower_bound > 0
+        ]
+
+        return pos_features, mid_features, low_features

src/utils/visualizer.py

@@ -0,0 +1,119 @@
+from xlm.demo.utils.segregate import Segregator
+from xlm.dto.dto import ExplanationDto
+
+
+UPPER_COLOR = "#D4EFDF"  # green
+MID_COLOR = "#FBFBB8BF"  # amber
+LOW_COLOR = "black"
+
+
+class Visualizer:
+    def __init__(self, show_mid_features: bool = True, show_low_features: bool = True):
+        self.__show_mid_features = show_mid_features
+        self.__show_low_features = show_low_features
+
+    def visualize(
+        self,
+        segregator: Segregator,
+        explanations: ExplanationDto,
+        output_from_explanations: str,
+        avoid_exp_label: bool = False,
+    ) -> str:
+        highlighted_text = output_from_explanations
+
+        pos_features, mid_features, low_features = segregator.segregate(
+            explanations=explanations
+        )
+
+        if not self.__show_mid_features:
+            mid_features = []
+
+        if not self.__show_low_features:
+            low_features = []
+
+        for explanation in explanations.explanations:
+            score = round(explanation.score, 2)
+
+            if explanation.feature in pos_features:
+                token_str = (
+                    '<span title="'
+                    + str(score)
+                    + '"style="font-weight:bold;background-color:'
+                    + UPPER_COLOR
+                    + '">'
+                    + explanation.feature
+                    + "</span>"
+                )
+            elif explanation.feature in mid_features:
+                token_str = (
+                    '<span title="'
+                    + str(score)
+                    + '"style="font-weight:bold;background-color:'
+                    + MID_COLOR
+                    + '">'
+                    + explanation.feature
+                    + "</span>"
+                )
+            else:
+                token_str = (
+                    '<span title="'
+                    + str(score)
+                    + '"style="color:'
+                    + LOW_COLOR
+                    + '">'
+                    + explanation.feature
+                    + "</span>"
+                )
+
+            highlighted_text = highlighted_text.replace(explanation.feature, token_str)
+
+        if avoid_exp_label:
+            vis = "<p>" + highlighted_text + "</p>"
+        else:
+            vis = "<p><b>Explanations:</b><br>" + highlighted_text + "</p>"
+        vis = vis.replace("\n", "<br>")
+
+        legend = "<p align='right'"
+
+        legend += (
+            '<span title="' + '"style="color:' + LOW_COLOR + '">' + "💡" + "</span>"
+        )
+
+        legend += "&emsp;"
+
+        legend += (
+            '<span title="'
+            + '"style="color:'
+            + LOW_COLOR
+            + '">'
+            + "not important"
+            + "</span>"
+        )
+
+        legend += "&emsp;⇢&emsp;"
+
+        legend += (
+            '<span title="'
+            + '"style="font-weight:bold;background-color:'
+            + MID_COLOR
+            + '">'
+            + " important "
+            + "</span>"
+        )
+
+        legend += "&emsp;⇢&emsp;"
+
+        legend += (
+            '<span title="'
+            + '"style="font-weight:bold;background-color:'
+            + UPPER_COLOR
+            + '">'
+            + " very important "
+            + "</span>"
+        )
+
+        legend += "</p>"
+
+        html_str = legend + vis
+
+        return html_str