try to avoid cuda OO error

app.py

@@ -90,18 +90,18 @@ def evaluate(
 ):
     content = process_webpage(url=url)
     # avoid GPU memory overflow
-    torch.cuda.empty_cache()
-    prompt = generate_prompt(instruction, content)
-    inputs = tokenizer(prompt, return_tensors="pt")
-    input_ids = inputs["input_ids"].to(device)
-    generation_config = GenerationConfig(
-        temperature=temperature,
-        top_p=top_p,
-        top_k=top_k,
-        num_beams=num_beams,
-        **kwargs,
-    )
     with torch.no_grad():
+        torch.cuda.empty_cache()
+        prompt = generate_prompt(instruction, content)
+        inputs = tokenizer(prompt, return_tensors="pt")
+        input_ids = inputs["input_ids"].to(device)
+        generation_config = GenerationConfig(
+            temperature=temperature,
+            top_p=top_p,
+            top_k=top_k,
+            num_beams=num_beams,
+            **kwargs,
+        )
         generation_output = model.generate(
             input_ids=input_ids,
             generation_config=generation_config,
@@ -109,8 +109,8 @@ def evaluate(
             output_scores=True,
             max_new_tokens=max_new_tokens,
         )
-    s = generation_output.sequences[0]
-    output = tokenizer.decode(s)
+        s = generation_output.sequences[0]
+        output = tokenizer.decode(s)
     # avoid GPU memory overflow
     torch.cuda.empty_cache()
     return output.split("### Response:")[1].strip()