add new "fast LLM" optimization

src/app/main.tsx

@@ -7,7 +7,6 @@ import { TopMenu } from "./interface/top-menu"
 import { fonts } from "@/lib/fonts"
 import { useStore } from "./store"
 import { Zoom } from "./interface/zoom"
-import { getStory } from "./queries/getStory"
 import { BottomBar } from "./interface/bottom-bar"
 import { Page } from "./interface/page"
 import { GeneratedPanel } from "@/types"
@@ -88,7 +87,8 @@ export default function Main() {
             nbPanelsToGenerate,
             existingPanels,
           })
-          console.log("LLM generated some new panels:", candidatePanels)
+          console.log("LLM generated some new panels:")
+          console.table(candidatePanels)
 
           existingPanels.push(...candidatePanels)
 
@@ -114,7 +114,7 @@ export default function Main() {
           }
 
           // update the frontend
-          console.log("updating the frontend..")
+          // console.log("updating the frontend..")
           setCaptions(newCaptions)
           setPanels(newPanelsPrompts)    
 

src/app/queries/getStory.ts

@@ -17,7 +17,7 @@ export const getStory = async ({
   prompt: string;
   nbTotalPanels: number;
 }): Promise<GeneratedPanels> => {
-  // throw new Error("Planned maintenance")
+  throw new Error("legacy, deprecated")
   
   // In case you need to quickly debug the RENDERING engine you can uncomment this:
   // return mockGeneratedPanels

src/app/queries/getStoryContinuation.ts

@@ -32,23 +32,22 @@ export const getStoryContinuation = async ({
       existingPanels,
     })
 
-    console.log("LLM responded with panels:", panelCandidates)
-
+    // console.log("LLM responded with panelCandidates:", panelCandidates)
 
     // we clean the output from the LLM
     // most importantly, we need to adjust the panel index,
     // to start from where we last finished
-    for (let p = startAt; p < endAt; p++) {
+    for (let i = 0; i < nbPanelsToGenerate; i++) {
       panels.push({
-        panel: p,
-        instructions: `${panelCandidates[p]?.instructions || ""}`,
-        caption: `${panelCandidates[p]?.caption || ""}`,
+        panel: startAt + i,
+        instructions: `${panelCandidates[i]?.instructions || ""}`,
+        caption: `${panelCandidates[i]?.caption || ""}`,
       })
     }
     
   } catch (err) {
-    console.log("LLM step failed due to:", err)
-    console.log("we are now switching to a degraded mode, using 4 similar panels")
+    // console.log("LLM step failed due to:", err)
+    // console.log("we are now switching to a degraded mode, using 4 similar panels")
     panels = []
     for (let p = startAt; p < endAt; p++) {
       panels.push({
@@ -61,7 +60,7 @@ export const getStoryContinuation = async ({
         caption: "(Sorry, LLM generation failed: using degraded mode)"
       })
     }
-    console.error(err)
+    // console.error(err)
   } finally {
     return panels
   }

src/app/queries/predictNextPanels.ts

@@ -18,6 +18,7 @@ export const predictNextPanels = async ({
   nbPanelsToGenerate: number;
   existingPanels: GeneratedPanel[];
 }): Promise<GeneratedPanel[]> => {
+  // console.log("predictNextPanels: ", { prompt, nbPanelsToGenerate })
   // throw new Error("Planned maintenance")
   
   // In case you need to quickly debug the RENDERING engine you can uncomment this:

src/app/queries/predictWithHuggingFace.ts

@@ -58,7 +58,7 @@ export async function predict(inputs: string, nbPanels: number): Promise<string>
       }
     })) {
       instructions += output.token.text
-      process.stdout.write(output.token.text)
+      // process.stdout.write(output.token.text)
       if (
         instructions.includes("</s>") || 
         instructions.includes("<s>") ||
@@ -78,7 +78,7 @@ export async function predict(inputs: string, nbPanels: number): Promise<string>
       }
     }
   } catch (err) {
-    console.error(`error during generation: ${err}`)
+    // console.error(`error during generation: ${err}`)
 
     // a common issue with Llama-2 might be that the model receives too many requests
     if (`${err}` === "Error: Model is overloaded") {

src/lib/dirtyLLMResponseCleaner.ts

@@ -2,12 +2,14 @@ export function dirtyLLMResponseCleaner(input: string): string {
   let str = (
     `${input || ""}`
     // a summary of all the weird hallucinations I saw it make..
+    .replaceAll(`"\n`, `",\n`) // fix missing commas at the end of a line
     .replaceAll(`"]`, `"}]`)
-    .replaceAll(`" ]`, `"}]`)
-    .replaceAll(`"  ]`, `"}]`)
-    .replaceAll(`"\n]`, `"}]`)
-    .replaceAll(`"\n ]`, `"}]`)
-    .replaceAll(`"\n  ]`, `"}]`)
+    .replaceAll(/"\S*,?\S*\]/gi, `"}]`)
+    .replaceAll(/"\S*,?\S*\}\S*]/gi, `"}]`)
+
+    // this removes the trailing commas (which are valid in JS but not JSON)
+    .replace(/,(?=\s*?[\}\]])/g, '')
+    
     .replaceAll("}}", "}")
     .replaceAll("]]", "]")
     .replaceAll("[[", "[")