added a demo to the app

.gitignore

@@ -0,0 +1 @@
+*.pyc

ShaderEval.py

@@ -158,7 +158,7 @@ class ReturnGenerationEvaluator(evaluate.TextGenerationEvaluator):
 class Suite(evaluate.EvaluationSuite):
 
 
-    def __init__(self, name):
+    def __init__(self, name, snippet: int = ""):
         super().__init__(name)
         self.preprocessor = lambda x: {"return_statement": x["return_statement"].split(";")[0]} #like this? refactored to RetrunGenerationEvaluator
         self.suite = [
@@ -167,7 +167,7 @@ class Suite(evaluate.EvaluationSuite):
                 task_type="text-generation", #this call an evaluator, but can you specify your own custom evaluator instead?
                 data="Vipitis/Shadertoys-fine",
                 subset="return_completion",
-                split="test[5:10]", #[5:10] is for testing to make it quick, and they got some easy examples, unless the first 5.
+                split=f"test[:{snippet}]", # use this to select a subset of the data during testing, perhaps remove later?
                 args_for_task={
                     # "metric": "exact_match",
                     "input_column": "body",

app.py

@@ -1,7 +1,38 @@
 import gradio as gr
+import evaluate
+from ShaderEval import Suite
 
-def greet(name):
-    return "Hello " + name + "!!\n This space hosts the ShaderEval Suite. more to follow soon."
+# suite = evaluate.EvaluationSuite.load("Vipitis/ShaderEval") #downloads it
+suite = Suite("Vipitis/ShaderEval") #local to the space?
 
-iface = gr.Interface(fn=greet, inputs="text", outputs="text")
-iface.launch()
+def run_suite(model_cp, snippet):
+    suite = Suite("Vipitis/ShaderEval", snippet)
+    results = suite.run(model_cp)
+    return results[0]["exact_match"]
+
+with gr.Blocks() as site:
+    text = gr.Markdown("""# Welcome to the ShaderEval Suite.
+            
+            This space hosts the ShaderEval Suite. more to follow soon.
+            
+            ## Instructions
+            ### Run the code yourself:.
+            ```python
+            import evaluate
+            suite = evaluate.EvaluationSuite.load("Vipitis/ShaderEval")
+            model_cp = "gpt2"
+            suite.run("model_cp")
+            ```
+            
+            ### try the demo below
+            - Select a **model checkpoint** from the "dropdown"
+            - Select how many **samples** to run (there us up to 100 from the test set)
+            - Click **Run** to run the suite
+            - The results will be displayed in the **Output** box
+            """)
+    model_cp = gr.Textbox(label="Model Checkpoint")
+    first_n = gr.Slider(minimum=1, maximum=100, default=10, label="num_samples", step=1.0)
+    output = gr.Textbox(label="Output")
+    run_button = gr.Button(label="Run")
+    run_button.click(fn=run_suite, inputs=[model_cp, first_n], outputs=output)
+site.launch()