cleanup

package.json

@@ -9,9 +9,14 @@
     "lint": "next lint"
   },
   "dependencies": {
+    "@duckdb/duckdb-wasm": "^1.28.1-dev106.0",
+    "@emotion/react": "^11.13.0",
+    "@emotion/styled": "^11.13.0",
+    "@mui/material": "^5.16.6",
+    "next": "14.2.5",
     "react": "^18",
-    "react-dom": "^18",
-    "next": "14.2.5"
+    "react-activity-calendar": "^2.2.11",
+    "react-dom": "^18"
   },
   "devDependencies": {
     "typescript": "^5",

src/pages/api/hello.ts

@@ -1,13 +0,0 @@
-// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
-import type { NextApiRequest, NextApiResponse } from "next";
-
-type Data = {
-  name: string;
-};
-
-export default function handler(
-  req: NextApiRequest,
-  res: NextApiResponse<Data>,
-) {
-  res.status(200).json({ name: "John Doe" });
-}

src/pages/index.tsx

@@ -1,118 +1,189 @@
-import Image from "next/image";
+"use client";
+
 import { Inter } from "next/font/google";
+import ActivityCalendar from "react-activity-calendar";
+import * as duckdb from "@duckdb/duckdb-wasm"
+import { useState, useEffect } from "react";
+import { Tooltip as MuiTooltip } from '@mui/material';
 
 const inter = Inter({ subsets: ["latin"] });
 
+interface CustomActivity {
+  date: string;
+  count: number;
+  level: number;
+  details: Array<{ provider: string; count: number }>;
+}
+
+type ProviderKey = "mistralai" | "meta-llama" | "openai" | "anthropic" | "google";
+
 export default function Home() {
-  return (
-    <main
-      className={`flex min-h-screen flex-col items-center justify-between p-24 ${inter.className}`}
-    >
-      <div className="z-10 max-w-5xl w-full items-center justify-between font-mono text-sm lg:flex">
-        <p className="fixed left-0 top-0 flex w-full justify-center border-b border-gray-300 bg-gradient-to-b from-zinc-200 pb-6 pt-8 backdrop-blur-2xl dark:border-neutral-800 dark:bg-zinc-800/30 dark:from-inherit lg:static lg:w-auto  lg:rounded-xl lg:border lg:bg-gray-200 lg:p-4 lg:dark:bg-zinc-800/30">
-          Get started by editing&nbsp;
-          <code className="font-mono font-bold">src/pages/index.tsx</code>
-        </p>
-        <div className="fixed bottom-0 left-0 flex h-48 w-full items-end justify-center bg-gradient-to-t from-white via-white dark:from-black dark:via-black lg:static lg:h-auto lg:w-auto lg:bg-none">
-          <a
-            className="pointer-events-none flex place-items-center gap-2 p-8 lg:pointer-events-auto lg:p-0"
-            href="https://vercel.com?utm_source=create-next-app&utm_medium=default-template-tw&utm_campaign=create-next-app"
-            target="_blank"
-            rel="noopener noreferrer"
-          >
-            By{" "}
-            <Image
-              src="/vercel.svg"
-              alt="Vercel Logo"
-              className="dark:invert"
-              width={100}
-              height={24}
-              priority
-            />
-          </a>
-        </div>
-      </div>
+  const [calendarData, setCalendarData] = useState<Record<ProviderKey, CustomActivity[]>>({} as Record<ProviderKey, CustomActivity[]>);
+  const [isLoading, setIsLoading] = useState(true);
 
-      <div className="relative flex place-items-center before:absolute before:h-[300px] before:w-full sm:before:w-[480px] before:-translate-x-1/2 before:rounded-full before:bg-gradient-radial before:from-white before:to-transparent before:blur-2xl before:content-[''] after:absolute after:-z-20 after:h-[180px] after:w-full sm:after:w-[240px] after:translate-x-1/3 after:bg-gradient-conic after:from-sky-200 after:via-blue-200 after:blur-2xl after:content-[''] before:dark:bg-gradient-to-br before:dark:from-transparent before:dark:to-blue-700/10 after:dark:from-sky-900 after:dark:via-[#0141ff]/40 before:lg:h-[360px]">
-        <Image
-          className="relative dark:drop-shadow-[0_0_0.3rem_#ffffff70] dark:invert"
-          src="/next.svg"
-          alt="Next.js Logo"
-          width={180}
-          height={37}
-          priority
-        />
-      </div>
+  const PROVIDERS_MAP: Record<ProviderKey, { name: string; color: string }> = {
+    "mistralai": {"name": "Mistral AI", "color": "#ff7000"},
+    "meta-llama": {"name": "Meta", "color": "#0668E1"},
+    "openai": {"name": "OpenAI", "color": "#10A37F"},
+    "anthropic": {"name": "Anthropic", "color": "#cc785c"},
+    "google": {"name": "Google", "color": "#4285F4"},
+  }
+
+  const getModelData = async (conn: duckdb.AsyncDuckDBConnection) => {
+    const result = await conn.query(`
+      SELECT 
+        STRFTIME(DATE_TRUNC('day', CAST(createdAt AS DATE)), '%Y-%m-%d') AS date,
+        CASE 
+          WHEN SPLIT_PART(id, '/', 1) IN ('meta-llama', 'facebook') THEN 'meta-llama'
+          ELSE SPLIT_PART(id, '/', 1)
+        END AS provider,
+        COUNT(*) AS count
+      FROM models
+      WHERE CAST(createdAt AS DATE) >= DATE_TRUNC('year', CURRENT_DATE)
+      AND (
+        SPLIT_PART(id, '/', 1) IN (${Object.keys(PROVIDERS_MAP).map(provider => `'${provider}'`).join(', ')})
+        OR SPLIT_PART(id, '/', 1) = 'facebook'
+      )
+      GROUP BY DATE_TRUNC('day', CAST(createdAt AS DATE)), 
+        CASE 
+          WHEN SPLIT_PART(id, '/', 1) IN ('meta-llama', 'facebook') THEN 'meta-llama'
+          ELSE SPLIT_PART(id, '/', 1)
+        END
+      ORDER BY date
+    `);
+    return result.toArray().map((row: any) => ({
+      date: row.date,
+      provider: row.provider,
+      count: Number(row.count)
+    }));
+  }
+
+  const generateCalendarData = (modelData: any[]) => {
+    const data: Record<ProviderKey, CustomActivity[]> = Object.keys(PROVIDERS_MAP).reduce((acc, provider) => {
+      acc[provider as ProviderKey] = [];
+      return acc;
+    }, {} as Record<ProviderKey, CustomActivity[]>);
+
+    const today = new Date();
+    const startOfYear = new Date(today.getFullYear(), 0, 1);
+
+    for (let d = new Date(startOfYear); d <= today; d.setDate(d.getDate() + 1)) {
+      const dateString = d.toISOString().split('T')[0];
+      
+      Object.keys(PROVIDERS_MAP).forEach((provider) => {
+        const dayData = modelData.filter(item => item.date === dateString && item.provider === provider);
+        const count = dayData.reduce((sum, item) => sum + item.count, 0);
+        
+        data[provider as ProviderKey].push({
+          date: dateString,
+          count,
+          level: 0,   
+          details: dayData,
+        });
+      });
+    }
+
+    const avgCounts: Record<ProviderKey, number> = Object.fromEntries(
+      Object.keys(PROVIDERS_MAP).map(provider => [
+        provider,
+        data[provider as ProviderKey].reduce((sum, day) => sum + day.count, 0) / data[provider as ProviderKey].length || 0
+      ])
+    ) as Record<ProviderKey, number>;
+
+    Object.entries(data).forEach(([provider, days]) => {
+      const avgCount = avgCounts[provider as ProviderKey];
+      days.forEach(day => {
+        day.level = day.count === 0 ? 0 :
+                    day.count <= avgCount * 0.5 ? 1 :
+                    day.count <= avgCount ? 2 :
+                    day.count <= avgCount * 1.5 ? 3 : 4;
+      });
+    });
 
-      <div className="mb-32 grid text-center lg:max-w-5xl lg:w-full lg:mb-0 lg:grid-cols-4 lg:text-left">
-        <a
-          href="https://nextjs.org/docs?utm_source=create-next-app&utm_medium=default-template-tw&utm_campaign=create-next-app"
-          className="group rounded-lg border border-transparent px-5 py-4 transition-colors hover:border-gray-300 hover:bg-gray-100 hover:dark:border-neutral-700 hover:dark:bg-neutral-800/30"
-          target="_blank"
-          rel="noopener noreferrer"
-        >
-          <h2 className={`mb-3 text-2xl font-semibold`}>
-            Docs{" "}
-            <span className="inline-block transition-transform group-hover:translate-x-1 motion-reduce:transform-none">
-              -&gt;
-            </span>
-          </h2>
-          <p className={`m-0 max-w-[30ch] text-sm opacity-50`}>
-            Find in-depth information about Next.js features and API.
-          </p>
-        </a>
-
-        <a
-          href="https://nextjs.org/learn?utm_source=create-next-app&utm_medium=default-template-tw&utm_campaign=create-next-app"
-          className="group rounded-lg border border-transparent px-5 py-4 transition-colors hover:border-gray-300 hover:bg-gray-100 hover:dark:border-neutral-700 hover:dark:bg-neutral-800/30"
-          target="_blank"
-          rel="noopener noreferrer"
-        >
-          <h2 className={`mb-3 text-2xl font-semibold`}>
-            Learn{" "}
-            <span className="inline-block transition-transform group-hover:translate-x-1 motion-reduce:transform-none">
-              -&gt;
-            </span>
-          </h2>
-          <p className={`m-0 max-w-[30ch] text-sm opacity-50`}>
-            Learn about Next.js in an interactive course with&nbsp;quizzes!
-          </p>
-        </a>
-
-        <a
-          href="https://vercel.com/templates?framework=next.js&utm_source=create-next-app&utm_medium=default-template-tw&utm_campaign=create-next-app"
-          className="group rounded-lg border border-transparent px-5 py-4 transition-colors hover:border-gray-300 hover:bg-gray-100 hover:dark:border-neutral-700 hover:dark:bg-neutral-800/30"
-          target="_blank"
-          rel="noopener noreferrer"
-        >
-          <h2 className={`mb-3 text-2xl font-semibold`}>
-            Templates{" "}
-            <span className="inline-block transition-transform group-hover:translate-x-1 motion-reduce:transform-none">
-              -&gt;
-            </span>
-          </h2>
-          <p className={`m-0 max-w-[30ch] text-sm opacity-50`}>
-            Discover and deploy boilerplate example Next.js&nbsp;projects.
-          </p>
-        </a>
-
-        <a
-          href="https://vercel.com/new?utm_source=create-next-app&utm_medium=default-template-tw&utm_campaign=create-next-app"
-          className="group rounded-lg border border-transparent px-5 py-4 transition-colors hover:border-gray-300 hover:bg-gray-100 hover:dark:border-neutral-700 hover:dark:bg-neutral-800/30"
-          target="_blank"
-          rel="noopener noreferrer"
-        >
-          <h2 className={`mb-3 text-2xl font-semibold`}>
-            Deploy{" "}
-            <span className="inline-block transition-transform group-hover:translate-x-1 motion-reduce:transform-none">
-              -&gt;
-            </span>
-          </h2>
-          <p className={`m-0 max-w-[30ch] text-sm opacity-50 text-balance`}>
-            Instantly deploy your Next.js site to a shareable URL with Vercel.
-          </p>
-        </a>
+    return data;
+  }
+
+  const initDB = async () => {
+    const CDN_BASE = `https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@next`
+
+    const JSDELIVR_BUNDLES = {
+      mvp: {
+        mainModule: `${CDN_BASE}/dist/duckdb-mvp.wasm`,
+        mainWorker: `${CDN_BASE}/dist/duckdb-browser-mvp.worker.js`,
+      },
+      eh: {
+        mainModule: `${CDN_BASE}/dist/duckdb-eh.wasm`,
+        mainWorker: `${CDN_BASE}/dist/duckdb-browser-eh.worker.js`,
+      },
+    }
+
+    const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES)
+    const worker_url = URL.createObjectURL(
+      new Blob([`importScripts("${bundle.mainWorker}");`], {
+        type: "text/javascript",
+      })
+    )
+
+    const worker = new Worker(worker_url)
+    const logger = new duckdb.ConsoleLogger()
+    const db = new duckdb.AsyncDuckDB(logger, worker)
+    await db.instantiate(bundle.mainModule)
+
+    const connection = await db.connect()
+
+    await connection.query(`
+        CREATE VIEW models AS SELECT * FROM read_parquet('https://huggingface.co/datasets/cfahlgren1/hub-stats/resolve/refs%2Fconvert%2Fparquet/models/train/0000.parquet?download=true');
+    `);
+
+    const modelData = await getModelData(connection);
+    const calendarData = generateCalendarData(modelData);
+    setCalendarData(calendarData);
+    setIsLoading(false);
+
+    await connection.close();
+  }
+
+  useEffect(() => {
+    initDB();
+  }, []);
+
+  return (
+    <main className={`flex flex-col items-center justify-center min-h-screen mx-auto p-24 ${inter.className}`}>
+      <h1 className="text-5xl font-bold text-center">Open Source Calendar</h1>
+      <p className="text-center mt-2 text-sm">A calendar for open source model releases.</p>
+      <div className="mt-16">
+        {isLoading ? (
+          <p>Loading...</p>
+        ) : (
+          <>
+            {Object.entries(PROVIDERS_MAP)
+              .sort(([keyA], [keyB]) => 
+                calendarData[keyB as ProviderKey].reduce((sum, day) => sum + day.count, 0) -
+                calendarData[keyA as ProviderKey].reduce((sum, day) => sum + day.count, 0)
+              )
+              .map(([key, value]) => (
+                <div key={key} className="mb-8">
+                  <h2 className="text-2xl font-bold mb-2">{value.name}</h2>
+                  <ActivityCalendar 
+                    data={calendarData[key as ProviderKey]}
+                    theme={{
+                      dark: ['#161b22', value.color],
+                    }}
+                    colorScheme="dark"
+                    renderBlock={(block, activity) => (
+                      <MuiTooltip
+                        title={`${activity.count} activities on ${activity.date}`}
+                      >
+                        {block}
+                      </MuiTooltip>
+                    )}
+                  />
+                </div>
+              ))
+            }
+          </>
+        )}
       </div>
     </main>
   );
-}
+}