import { __internal_XetBlob as XetBlob, __internal_sha256, } from "@huggingface/hub"; const lfsContent = "0123456789".repeat(1_000_000); const XetBlobs = { // "hub-unbundled": XetBlobUnbundled, "hub-esm": XetBlobEsm, hub: XetBlob, }; async function test(id) { const blob = new Blob([lfsContent]); try { const blob = new XetBlobs[id]({ repo: { type: "model", name: "celinah/xet-experiments", }, hash: "7b3b6d07673a88cf467e67c1f7edef1a8c268cbf66e9dd9b0366322d4ab56d9b", size: 5_234_139_343, }); const text = await blob.slice(10, 22).text(); document.getElementById(id).textContent = text; } catch (err) { document.getElementById(id).textContent = err.message; } } async function checkRange(n) { // Get selected option const select = document.getElementById("file"); const option = select.options[select.selectedIndex]; // hash is in data-xet-hash attribute const xetHash = option.getAttribute("data-xet-hash"); const sha = option.getAttribute("data-sha"); const size = +option.getAttribute("data-size"); const path = `https://huggingface.co/celinah/xet-experiments/resolve/main/${option.value}`; const output = document.getElementById("output"); output.textContent = ""; const blob = new XetBlob({ repo: { type: "model", name: "celinah/xet-experiments", }, hash: xetHash, size, }); const [own, bridge] = await Promise.all([ blob.slice(0, n).arrayBuffer(), fetch(path, { headers: { Range: `bytes=0-${n - 1}` } }).then((res) => res.arrayBuffer() ), ]); output.textContent += `Own: ${own.byteLength}\nBridge: ${bridge.byteLength}\n`; // Check if the first n bytes are the same const array1 = new Uint8Array(own); const array2 = new Uint8Array(bridge); let different = false; for (let i = 0; i < n; i++) { if (array1[i] !== array2[i]) { output.textContent += `Difference at ${i}\n`; different = true; break; } } if (!different) { output.textContent += "No differences\n"; } } async function download() { const select = document.getElementById("file"); const option = select.options[select.selectedIndex]; const xetHash = option.getAttribute("data-xet-hash"); const size = +option.getAttribute("data-size"); const output = document.getElementById("output"); output.textContent = ""; const blob = new XetBlob({ repo: { type: "model", name: "celinah/xet-experiments", }, hash: xetHash, size, listener: (progress) => { output.textContent += JSON.stringify(progress) + "\n"; output.textContent = output.textContent.split("\n").slice(-1000).join("\n") + "\n"; }, }); const arrayBuffer = await blob.arrayBuffer(); const blob2 = new Blob([arrayBuffer]); const url = URL.createObjectURL(blob2); const a = document.createElement("a"); a.href = url; a.download = option.value; a.click(); } async function checkSha() { const select = document.getElementById("file"); const option = select.options[select.selectedIndex]; const xetHash = option.getAttribute("data-xet-hash"); const sha = option.getAttribute("data-sha"); const size = +option.getAttribute("data-size"); const output = document.getElementById("output"); output.textContent = ""; const blob = new XetBlob({ repo: { type: "model", name: "celinah/xet-experiments", }, hash: xetHash, size, }); const iterator = __internal_sha256(blob); while (1) { const result = await iterator.next(); if (result.done) { const hash = result.value; output.textContent += `SHA: ${hash}\n`; output.textContent += `Expected: ${sha}\n`; output.textContent += `Match: ${hash === sha}\n`; break; } output.textContent = `Progress: ${result.value * 100}%\n`; } } // On loaded, add event listeners document.addEventListener( "DOMContentLoaded", () => { document.getElementById("10kB").addEventListener("click", () => { checkRange(10_000); }); document.getElementById("100kB").addEventListener("click", () => { checkRange(100_000); }); document.getElementById("1MB").addEventListener("click", () => { checkRange(1_000_000); }); document.getElementById("sequential").addEventListener("click", () => { checkRange(100_000).then(() => checkRange(1_000_000)); }); document.getElementById("download").addEventListener("click", download); document.getElementById("sha").addEventListener("click", checkSha); }, { once: true } );