Add Dockerfile and fix requirements.txt typo

Dockerfile

@@ -0,0 +1,38 @@
+FROM nvidia/cuda:11.7.1-devel-ubuntu20.04
+
+WORKDIR /workspace
+
+# Basic installs
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ='America/Detroit'
+RUN apt-get update -qq \
+    && apt-get -y --no-install-recommends install \
+       build-essential software-properties-common wget git tar rsync \
+    && apt-get clean all \
+    && rm -r /var/lib/apt/lists/*
+
+# Install Miniconda3 4.12.0
+ENV PATH="/root/.local/miniconda3/bin:$PATH"
+RUN mkdir -p /root/.local \
+    && wget https://repo.anaconda.com/miniconda/Miniconda3-py39_23.3.1-0-Linux-x86_64.sh \
+    && mkdir /root/.conda \
+    && bash Miniconda3-py39_23.3.1-0-Linux-x86_64.sh -b -p /root/.local/miniconda3 \
+    && rm -f Miniconda3-py39_23.3.1-0-Linux-x86_64.sh \
+    && ln -sf /root/.local/miniconda3/etc/profile.d/conda.sh /etc/profile.d/conda.sh
+
+# Install PyTorch
+RUN pip install torch==2.0.1
+
+# Install the HEAD commit of Zeus (for ZeusMonitor)
+RUN git clone https://github.com/SymbioticLab/Zeus.git zeus \
+      && cd zeus \
+      && pip install -e . \
+      && cd ..
+
+# Install requirements for benchmarking
+ADD . /workspace/leaderboard
+RUN cd leaderboard \
+      && pip install -r requirements.txt \
+      && cd ..
+
+ENV TRANSFORMERS_CACHE=/data/leaderboard/hfcache

README.md

@@ -24,6 +24,10 @@ export TRANSFORMERS_CACHE=/data/leaderboard/hfcache
 Run benchmarks like this:
 
 ```console
-$ python benchmark.py --model-path /data/leaderboard/weights/lmsys/vicuna-7B --input-file /data/leaderboard/sharegpt/sg_90k_part1_html_cleaned_lang_first_sampled.json
-$ python benchmark.py --model-path databricks/dolly-v2-12b --input-file /data/leaderboard/sharegpt/sg_90k_part1_html_cleaned_lang_first_sampled.json
+$ docker build -t leaderboard:latest .
+$ docker run -it --name jw-leaderboard --gpus all --cap-add SYS_ADMIN -v /data/leaderboard:/data/leaderboard -v $HOME/workspace/leaderboard:/workspace/leaderboard leaderboard:latest bash
+
+# cd leaderboard
+# python benchmark.py --model-path /data/leaderboard/weights/lmsys/vicuna-7B --input-file /data/leaderboard/sharegpt/sg_90k_part1_html_cleaned_lang_first_sampled.json
+# python benchmark.py --model-path databricks/dolly-v2-12b --input-file /data/leaderboard/sharegpt/sg_90k_part1_html_cleaned_lang_first_sampled.json
 ```

benchmark.py

@@ -70,6 +70,8 @@ def main(
         raise ValueError("ChatGLM is not supported.")
 
     # Print out what we're about to do.
+    if model_path.endswith("/"):
+        model_path = model_path[:-1]
     model_name_cleaned = "--".join(model_path.split("/")[-2:])
     output_dir = f"data/{task}/{model_name_cleaned}"
     output_csv_path = f"{output_dir}/benchmark.json"

requirements.txt

@@ -1,7 +1,5 @@
-zeus-ml==0.4.0
-fastchat==0.2.14
+zeus-ml
+fschat==0.2.14
 rwkv==0.7.5
 einops
 tyro
-plotpy
-gradio