update sdk

README.md

@@ -4,7 +4,7 @@ emoji: 🎼🎶
 colorFrom: red
 colorTo: indigo
 sdk: gradio
-sdk_version: 3.41.1
+sdk_version: 4.36.0
 app_file: app.py
 pinned: false
 license: apache-2.0

app.py

@@ -121,6 +121,8 @@ def run(model_name, tab, instruments, drum_kit, mid, midi_events, gen_events, te
         i = 0
         mid = [[tokenizer.bos_id] + [tokenizer.pad_id] * (tokenizer.max_token_seq - 1)]
         patches = {}
+        if instruments is None:
+            instruments = []
         for instr in instruments:
             patches[i] = patch2number[instr]
             i = (i + 1) if i != 8 else 10
@@ -187,20 +189,17 @@ def load_javascript(dir="javascript"):
 
     gr.routes.templates.TemplateResponse = template_response
 
+# JSMsgReceiver
+HTML_postprocess_ori = gr.HTML.postprocess
 
-class JSMsgReceiver(gr.HTML):
 
-    def __init__(self, **kwargs):
-        super().__init__(elem_id="msg_receiver", visible=False, **kwargs)
+def JSMsgReceiver_postprocess(self, y):
+    if self.elem_id == "msg_receiver" and y:
+        y = f"<p>{json.dumps(y)}</p>"
+    return HTML_postprocess_ori(self, y)
 
-    def postprocess(self, y):
-        if y:
-            y = f"<p>{json.dumps(y)}</p>"
-        return super().postprocess(y)
-
-    def get_block_name(self) -> str:
-        return "html"
 
+gr.HTML.postprocess = JSMsgReceiver_postprocess
 
 number2drum_kits = {-1: "None", 0: "Standard", 8: "Room", 16: "Power", 24: "Electric", 25: "TR-808", 32: "Jazz",
                     40: "Blush", 48: "Orchestra"}
@@ -216,7 +215,8 @@ if __name__ == "__main__":
     soundfont_path = hf_hub_download(repo_id="skytnt/midi-model", filename="soundfont.sf2")
     models_info = {"generic pretrain model": ["skytnt/midi-model", ""],
                    "j-pop finetune model": ["skytnt/midi-model-ft", "jpop/"],
-                   "touhou finetune model": ["skytnt/midi-model-ft", "touhou/"]}
+                   "touhou finetune model": ["skytnt/midi-model-ft", "touhou/"],
+                   }
     models = {}
     tokenizer = MIDITokenizer()
     providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']
@@ -238,10 +238,10 @@ if __name__ == "__main__":
                     "(https://colab.research.google.com/github/SkyTNT/midi-model/blob/main/demo.ipynb)"
                     " for faster running and longer generation"
                     )
-        js_msg = JSMsgReceiver()
+        js_msg = gr.HTML(elem_id="msg_receiver", visible=False)
         input_model = gr.Dropdown(label="select model", choices=list(models.keys()),
                                   type="value", value=list(models.keys())[0])
-        tab_select = gr.Variable(value=0)
+        tab_select = gr.State(value=0)
         with gr.Tabs():
             with gr.TabItem("instrument prompt") as tab1:
                 input_instruments = gr.Dropdown(label="instruments (auto if empty)", choices=list(patch2number.keys()),
@@ -279,7 +279,7 @@ if __name__ == "__main__":
             example3 = gr.Examples([[1, 0.98, 12], [1.2, 0.95, 8]], [input_temp, input_top_p, input_top_k])
         run_btn = gr.Button("generate", variant="primary")
         stop_btn = gr.Button("stop and output")
-        output_midi_seq = gr.Variable()
+        output_midi_seq = gr.State()
         output_midi_visualizer = gr.HTML(elem_id="midi_visualizer_container")
         output_audio = gr.Audio(label="output audio", format="mp3", elem_id="midi_audio")
         output_midi = gr.File(label="output midi", file_types=[".mid"])

javascript/app.js

@@ -1,3 +1,46 @@
+/**
+ * 自动绕过 shadowRoot 的 querySelector
+ * @param {string} selector - 要查询的 CSS 选择器
+ * @returns {Element|null} - 匹配的元素或 null 如果未找到
+ */
+function deepQuerySelector(selector) {
+  /**
+   * 在指定的根元素或文档对象下深度查询元素
+   * @param {Element|Document} root - 要开始搜索的根元素或文档对象
+   * @param {string} selector - 要查询的 CSS 选择器
+   * @returns {Element|null} - 匹配的元素或 null 如果未找到
+   */
+  function deepSearch(root, selector) {
+    // 在当前根元素下查找
+    let element = root.querySelector(selector);
+    if (element) {
+      return element;
+    }
+
+    // 如果未找到，递归检查 shadow DOM
+    const shadowHosts = root.querySelectorAll('*');
+
+    for (let i = 0; i < shadowHosts.length; i++) {
+      const host = shadowHosts[i];
+
+      // 检查当前元素是否有 shadowRoot
+      if (host.shadowRoot) {
+        element = deepSearch(host.shadowRoot, selector);
+        if (element) {
+          return element;
+        }
+      }
+    }
+    // 未找到元素
+    return null;
+  }
+
+  return deepSearch(this, selector);
+}
+
+Element.prototype.deepQuerySelector = deepQuerySelector;
+Document.prototype.deepQuerySelector = deepQuerySelector;
+
 function gradioApp() {
     const elems = document.getElementsByTagName('gradio-app')
     const gradioShadowRoot = elems.length == 0 ? null : elems[0].shadowRoot
@@ -98,6 +141,7 @@ class MidiVisualizer extends HTMLElement{
         this.timePreBeat = 16
         this.svgWidth = 0;
         this.t1 = 0;
+        this.totalTimeMs = 0
         this.playTime = 0
         this.playTimeMs = 0
         this.colorMap = new Map();
@@ -137,6 +181,7 @@ class MidiVisualizer extends HTMLElement{
         this.t1 = 0
         this.colorMap.clear()
         this.setPlayTime(0);
+        this.totalTimeMs = 0;
         this.playTimeMs = 0
         this.svgWidth = 0
         this.svg.innerHTML = ''
@@ -215,6 +260,9 @@ class MidiVisualizer extends HTMLElement{
                 tempo = (60 / midiEvent[3]) * 10 ** 3
                 this.midiTimes.push({ms:ms, t: t, tempo: tempo})
             }
+            if(midiEvent[0]==="note"){
+                this.totalTimeMs = ms + (midiEvent[3]/ this.timePreBeat)*tempo
+            }
             lastT = t
         })
     }
@@ -277,16 +325,10 @@ class MidiVisualizer extends HTMLElement{
 
     play(){
         this.playing = true;
-        this.timer = setInterval(() => {
-            this.setPlayTimeMs(this.playTimeMs + 10)
-        }, 10);
     }
 
     pause(){
-        if(!!this.timer)
-            clearInterval(this.timer)
         this.removeActiveNotes(this.activeNotes)
-        this.timer = null;
         this.playing = false;
     }
 
@@ -299,9 +341,25 @@ class MidiVisualizer extends HTMLElement{
         audio.addEventListener("pause", (event)=>{
             this.pause()
         })
-        audio.addEventListener("timeupdate", (event)=>{
-            this.setPlayTimeMs(event.target.currentTime*10**3)
-        })
+    }
+
+    bindWaveformCursor(cursor){
+        let self = this;
+        const callback = function(mutationsList, observer) {
+            for(let mutation of mutationsList) {
+                if (mutation.type === 'attributes' && mutation.attributeName === 'style') {
+                    let progress = parseFloat(mutation.target.style.left.slice(0,-1))*0.01;
+                    if(!isNaN(progress)){
+                        self.setPlayTimeMs(progress*self.totalTimeMs);
+                    }
+                }
+            }
+        };
+        const observer = new MutationObserver(callback);
+        observer.observe(cursor, {
+            attributes: true,
+            attributeFilter: ['style']
+        });
     }
 }
 
@@ -309,7 +367,8 @@ customElements.define('midi-visualizer', MidiVisualizer);
 
 (()=>{
     let midi_visualizer_container_inited = null
-    let midi_audio_inited = null;
+    let midi_audio_audio_inited = null;
+    let midi_audio_cursor_inited = null;
     let midi_visualizer = document.createElement('midi-visualizer')
     onUiUpdate((m)=>{
         let app = gradioApp()
@@ -318,10 +377,18 @@ customElements.define('midi-visualizer', MidiVisualizer);
             midi_visualizer_container.appendChild(midi_visualizer)
             midi_visualizer_container_inited = midi_visualizer_container;
         }
-        let midi_audio = app.querySelector("#midi_audio > audio");
-        if(!!midi_audio && midi_audio_inited!==midi_audio){
-            midi_visualizer.bindAudioPlayer(midi_audio)
-            midi_audio_inited = midi_audio
+        let midi_audio = app.querySelector("#midi_audio");
+        if (!!midi_audio){
+            let midi_audio_cursor = midi_audio.deepQuerySelector(".cursor");
+            if(!!midi_audio_cursor && midi_audio_cursor_inited!==midi_audio_cursor){
+                midi_visualizer.bindWaveformCursor(midi_audio_cursor)
+                midi_audio_cursor_inited = midi_audio_cursor
+            }
+            let midi_audio_audio = midi_audio.deepQuerySelector("audio");
+            if(!!midi_audio_audio && midi_audio_audio_inited!==midi_audio_audio){
+                midi_visualizer.bindAudioPlayer(midi_audio_audio)
+                midi_audio_audio_inited = midi_audio_audio
+            }
         }
     })