# MODEL=deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
# ValueError: User-specified max_model_len (32768) is greater than the derived max_model_len (max_position_embeddings=2048 or model_max_length=None in model's config.json). This may lead
# to incorrect model outputs or CUDA errors. To allow overriding this maximum, set the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
# Only needed for MicroLlama V1
# export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1

NUM_GPUS=4
MODEL="/root/open-r1/data/meta-llama/Llama-3.2-1B-Instruct/checkpoint-900"
MODEL_ARGS="pretrained=$MODEL,dtype=float16,data_parallel_size=$NUM_GPUS,max_model_length=32768,gpu_memory_utilisation=0.8"
TASK=aime24
OUTPUT_DIR=data/evals/$MODEL

lighteval vllm $MODEL_ARGS "custom|$TASK|0|0" \
    --custom-tasks src/open_r1/evaluate.py \
    --use-chat-template \
    --system-prompt="Please reason step by step, and put your final answer within \boxed{}." \
    --output-dir $OUTPUT_DIR