Upload folder using huggingface_hub
Browse files- added_tokens.json +3 -0
- arguments.json +53 -0
- arguments.pkl +3 -0
- config.json +30 -0
- environ.txt +213 -0
- pytorch_model.bin +3 -0
- script.sh +155 -0
- special_tokens_map.json +30 -0
- stderr.log +585 -0
- stdout.log +303 -0
- tokenizer.json +0 -0
- tokenizer.model +3 -0
- tokenizer_config.json +55 -0
- wandb/debug-internal.log +16 -0
- wandb/debug.log +47 -0
- wandb/run-20250527_211254-n7vbaf0c/files/config.yaml +125 -0
- wandb/run-20250527_211254-n7vbaf0c/files/output.log +13 -0
- wandb/run-20250527_211254-n7vbaf0c/files/requirements.txt +219 -0
- wandb/run-20250527_211254-n7vbaf0c/files/wandb-metadata.json +130 -0
- wandb/run-20250527_211254-n7vbaf0c/files/wandb-summary.json +1 -0
- wandb/run-20250527_211254-n7vbaf0c/logs/debug-core.log +14 -0
- wandb/run-20250527_211254-n7vbaf0c/logs/debug-internal.log +16 -0
- wandb/run-20250527_211254-n7vbaf0c/logs/debug.log +47 -0
- wandb/run-20250527_211254-n7vbaf0c/run-n7vbaf0c.wandb +0 -0
added_tokens.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"<pad>": 32000
|
| 3 |
+
}
|
arguments.json
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_name_or_path": "/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000",
|
| 3 |
+
"max_length": 512,
|
| 4 |
+
"trust_remote_code": true,
|
| 5 |
+
"train_datasets": [
|
| 6 |
+
[
|
| 7 |
+
"inverse-json",
|
| 8 |
+
{
|
| 9 |
+
"proportion": 1.0,
|
| 10 |
+
"path": "/home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json"
|
| 11 |
+
}
|
| 12 |
+
]
|
| 13 |
+
],
|
| 14 |
+
"eval_datasets": null,
|
| 15 |
+
"epochs": 1,
|
| 16 |
+
"per_device_train_batch_size": 1,
|
| 17 |
+
"per_device_eval_batch_size": 4,
|
| 18 |
+
"gradient_accumulation_steps": 8,
|
| 19 |
+
"gradient_checkpointing": true,
|
| 20 |
+
"lr": 1e-05,
|
| 21 |
+
"lr_scheduler_type": "constant",
|
| 22 |
+
"lr_warmup_ratio": 0.0,
|
| 23 |
+
"weight_decay": 0.0,
|
| 24 |
+
"seed": 42,
|
| 25 |
+
"fp16": false,
|
| 26 |
+
"bf16": true,
|
| 27 |
+
"tf32": true,
|
| 28 |
+
"eval_strategy": "epoch",
|
| 29 |
+
"eval_interval": 1000000,
|
| 30 |
+
"need_eval": false,
|
| 31 |
+
"eval_split_ratio": null,
|
| 32 |
+
"output_dir": "/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500",
|
| 33 |
+
"log_type": "wandb",
|
| 34 |
+
"log_dir": "/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500",
|
| 35 |
+
"log_project": "Inverse_Alignment_IMDb",
|
| 36 |
+
"log_run_name": "imdb-tinyllama-3T-s3-Q1-5000-Q2-500",
|
| 37 |
+
"save_16bit": true,
|
| 38 |
+
"save_interval": 1000000,
|
| 39 |
+
"local_rank": 0,
|
| 40 |
+
"zero_stage": 3,
|
| 41 |
+
"offload": "none",
|
| 42 |
+
"deepspeed": false,
|
| 43 |
+
"deepspeed_config": null,
|
| 44 |
+
"deepscale": false,
|
| 45 |
+
"deepscale_config": null,
|
| 46 |
+
"global_rank": 0,
|
| 47 |
+
"device": {
|
| 48 |
+
"type": "torch.device",
|
| 49 |
+
"repr": "device(type='cuda', index=0)"
|
| 50 |
+
},
|
| 51 |
+
"num_update_steps_per_epoch": 8,
|
| 52 |
+
"total_training_steps": 8
|
| 53 |
+
}
|
arguments.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:35c5c78abb419ecb328997428818b73040016fb704f17333a62a699d7041621b
|
| 3 |
+
size 1215
|
config.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"LlamaForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"attention_bias": false,
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"bos_token_id": 1,
|
| 8 |
+
"eos_token_id": 2,
|
| 9 |
+
"head_dim": 64,
|
| 10 |
+
"hidden_act": "silu",
|
| 11 |
+
"hidden_size": 2048,
|
| 12 |
+
"initializer_range": 0.02,
|
| 13 |
+
"intermediate_size": 5632,
|
| 14 |
+
"max_position_embeddings": 2048,
|
| 15 |
+
"mlp_bias": false,
|
| 16 |
+
"model_type": "llama",
|
| 17 |
+
"num_attention_heads": 32,
|
| 18 |
+
"num_hidden_layers": 22,
|
| 19 |
+
"num_key_value_heads": 4,
|
| 20 |
+
"pad_token_id": 32000,
|
| 21 |
+
"pretraining_tp": 1,
|
| 22 |
+
"rms_norm_eps": 1e-05,
|
| 23 |
+
"rope_scaling": null,
|
| 24 |
+
"rope_theta": 10000.0,
|
| 25 |
+
"tie_word_embeddings": false,
|
| 26 |
+
"torch_dtype": "float32",
|
| 27 |
+
"transformers_version": "4.52.1",
|
| 28 |
+
"use_cache": true,
|
| 29 |
+
"vocab_size": 32001
|
| 30 |
+
}
|
environ.txt
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
ADDR2LINE=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-addr2line
|
| 2 |
+
AR=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-ar
|
| 3 |
+
AS=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-as
|
| 4 |
+
BASH_FUNC__module_raw%%=() { unset _mlshdbg;
|
| 5 |
+
if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
|
| 6 |
+
case "$-" in
|
| 7 |
+
*v*x*)
|
| 8 |
+
set +vx;
|
| 9 |
+
_mlshdbg='vx'
|
| 10 |
+
;;
|
| 11 |
+
*v*)
|
| 12 |
+
set +v;
|
| 13 |
+
_mlshdbg='v'
|
| 14 |
+
;;
|
| 15 |
+
*x*)
|
| 16 |
+
set +x;
|
| 17 |
+
_mlshdbg='x'
|
| 18 |
+
;;
|
| 19 |
+
*)
|
| 20 |
+
_mlshdbg=''
|
| 21 |
+
;;
|
| 22 |
+
esac;
|
| 23 |
+
fi;
|
| 24 |
+
unset _mlre _mlIFS;
|
| 25 |
+
if [ -n "${IFS+x}" ]; then
|
| 26 |
+
_mlIFS=$IFS;
|
| 27 |
+
fi;
|
| 28 |
+
IFS=' ';
|
| 29 |
+
for _mlv in ${MODULES_RUN_QUARANTINE:-};
|
| 30 |
+
do
|
| 31 |
+
if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
|
| 32 |
+
if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
|
| 33 |
+
_mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
|
| 34 |
+
fi;
|
| 35 |
+
_mlrv="MODULES_RUNENV_${_mlv}";
|
| 36 |
+
_mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
|
| 37 |
+
fi;
|
| 38 |
+
done;
|
| 39 |
+
if [ -n "${_mlre:-}" ]; then
|
| 40 |
+
eval `eval ${_mlre} /usr/bin/tclsh /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl bash '"$@"'`;
|
| 41 |
+
else
|
| 42 |
+
eval `/usr/bin/tclsh /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl bash "$@"`;
|
| 43 |
+
fi;
|
| 44 |
+
_mlstatus=$?;
|
| 45 |
+
if [ -n "${_mlIFS+x}" ]; then
|
| 46 |
+
IFS=$_mlIFS;
|
| 47 |
+
else
|
| 48 |
+
unset IFS;
|
| 49 |
+
fi;
|
| 50 |
+
unset _mlre _mlv _mlrv _mlIFS;
|
| 51 |
+
if [ -n "${_mlshdbg:-}" ]; then
|
| 52 |
+
set -$_mlshdbg;
|
| 53 |
+
fi;
|
| 54 |
+
unset _mlshdbg;
|
| 55 |
+
return $_mlstatus
|
| 56 |
+
}
|
| 57 |
+
BASH_FUNC_ml%%=() { module ml "$@"
|
| 58 |
+
}
|
| 59 |
+
BASH_FUNC_module%%=() { _module_raw "$@" 2>&1
|
| 60 |
+
}
|
| 61 |
+
BASH_FUNC_switchml%%=() { typeset swfound=1;
|
| 62 |
+
if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
|
| 63 |
+
typeset swname='main';
|
| 64 |
+
if [ -e /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl ]; then
|
| 65 |
+
typeset swfound=0;
|
| 66 |
+
unset MODULES_USE_COMPAT_VERSION;
|
| 67 |
+
fi;
|
| 68 |
+
else
|
| 69 |
+
typeset swname='compatibility';
|
| 70 |
+
if [ -e /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd-compat ]; then
|
| 71 |
+
typeset swfound=0;
|
| 72 |
+
MODULES_USE_COMPAT_VERSION=1;
|
| 73 |
+
export MODULES_USE_COMPAT_VERSION;
|
| 74 |
+
fi;
|
| 75 |
+
fi;
|
| 76 |
+
if [ $swfound -eq 0 ]; then
|
| 77 |
+
echo "Switching to Modules $swname version";
|
| 78 |
+
source /cm/local/apps/environment-modules/4.5.3/init/bash;
|
| 79 |
+
else
|
| 80 |
+
echo "Cannot switch to Modules $swname version, command not found";
|
| 81 |
+
return 1;
|
| 82 |
+
fi
|
| 83 |
+
}
|
| 84 |
+
BUILD=x86_64-conda-linux-gnu
|
| 85 |
+
CC=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-cc
|
| 86 |
+
CC_FOR_BUILD=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-cc
|
| 87 |
+
CFLAGS=-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /aifs4su/hansirui_1st/miniconda3/envs/jy-resist/include -I/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/targets/x86_64-linux/include -L/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/targets/x86_64-linux/lib -L/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/targets/x86_64-linux/lib/stubs
|
| 88 |
+
CMAKE_ARGS=-DCMAKE_AR=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-ar -DCMAKE_RANLIB=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-ranlib -DCMAKE_LINKER=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-ld -DCMAKE_STRIP=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-strip -DCMAKE_BUILD_TYPE=Release
|
| 89 |
+
CMAKE_PREFIX_PATH=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist:/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/x86_64-conda-linux-gnu/sysroot/usr
|
| 90 |
+
CMD_WLM_CLUSTER_NAME=slurm
|
| 91 |
+
CONDA_BUILD_SYSROOT=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/x86_64-conda-linux-gnu/sysroot
|
| 92 |
+
CONDA_DEFAULT_ENV=jy-resist
|
| 93 |
+
CONDA_EXE=/aifs4su/hansirui_1st/miniconda3/bin/conda
|
| 94 |
+
CONDA_PREFIX=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist
|
| 95 |
+
CONDA_PREFIX_1=/aifs4su/hansirui_1st/miniconda3
|
| 96 |
+
CONDA_PREFIX_2=/aifs4su/hansirui_1st/miniconda3/envs/by-verl
|
| 97 |
+
CONDA_PREFIX_3=/aifs4su/hansirui_1st/miniconda3
|
| 98 |
+
CONDA_PROMPT_MODIFIER=(jy-resist)
|
| 99 |
+
CONDA_PYTHON_EXE=/aifs4su/hansirui_1st/miniconda3/bin/python
|
| 100 |
+
CONDA_SHLVL=4
|
| 101 |
+
CPATH=/cm/shared/apps/slurm/current/include
|
| 102 |
+
CPATH_modshare=/cm/shared/apps/slurm/current/include:1
|
| 103 |
+
CPP=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-cpp
|
| 104 |
+
CPPFLAGS=-DNDEBUG -D_FORTIFY_SOURCE=2 -O2 -isystem /aifs4su/hansirui_1st/miniconda3/envs/jy-resist/include -I/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/targets/x86_64-linux/include -L/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/targets/x86_64-linux/lib -L/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/targets/x86_64-linux/lib/stubs
|
| 105 |
+
CROSS_RANK=0
|
| 106 |
+
CROSS_SIZE=1
|
| 107 |
+
CUDA_MODULE_LOADING=LAZY
|
| 108 |
+
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 109 |
+
CXX=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-c++
|
| 110 |
+
CXXFILT=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-c++filt
|
| 111 |
+
CXXFLAGS=-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /aifs4su/hansirui_1st/miniconda3/envs/jy-resist/include -I/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/targets/x86_64-linux/include -L/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/targets/x86_64-linux/lib -L/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/targets/x86_64-linux/lib/stubs
|
| 112 |
+
CXX_FOR_BUILD=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-c++
|
| 113 |
+
DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1081/bus
|
| 114 |
+
DEBUG_CFLAGS=-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem /aifs4su/hansirui_1st/miniconda3/envs/jy-resist/include
|
| 115 |
+
DEBUG_CPPFLAGS=-D_DEBUG -D_FORTIFY_SOURCE=2 -Og -isystem /aifs4su/hansirui_1st/miniconda3/envs/jy-resist/include
|
| 116 |
+
DEBUG_CXXFLAGS=-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem /aifs4su/hansirui_1st/miniconda3/envs/jy-resist/include
|
| 117 |
+
ELFEDIT=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-elfedit
|
| 118 |
+
ENABLE_LMOD=0
|
| 119 |
+
GCC=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-gcc
|
| 120 |
+
GCC_AR=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-gcc-ar
|
| 121 |
+
GCC_NM=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-gcc-nm
|
| 122 |
+
GCC_RANLIB=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-gcc-ranlib
|
| 123 |
+
GPROF=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-gprof
|
| 124 |
+
GSETTINGS_SCHEMA_DIR=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/share/glib-2.0/schemas
|
| 125 |
+
GSETTINGS_SCHEMA_DIR_CONDA_BACKUP=
|
| 126 |
+
GXX=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-g++
|
| 127 |
+
HF_DATASETS_CACHE=/aifs4su/hansirui_1st/.cache/huggingface/datasets
|
| 128 |
+
HF_HOME=/aifs4su/hansirui_1st/.cache/huggingface
|
| 129 |
+
HISTTIMEFORMAT=%y/%m/%d %T
|
| 130 |
+
HOME=/home/hansirui_1st
|
| 131 |
+
HOST=x86_64-conda-linux-gnu
|
| 132 |
+
IMDb=True
|
| 133 |
+
KMP_DUPLICATE_LIB_OK=True
|
| 134 |
+
KMP_INIT_AT_FORK=FALSE
|
| 135 |
+
LANG=C.UTF-8
|
| 136 |
+
LD=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-ld
|
| 137 |
+
LDFLAGS=-Wl,-O2 -Wl,--sort-common -Wl,--as-needed -Wl,-z,relro -Wl,-z,now -Wl,--disable-new-dtags -Wl,--gc-sections -Wl,-rpath,/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib -Wl,-rpath-link,/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib -L/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib -L/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/targets/x86_64-linux/lib -L/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/targets/x86_64-linux/lib/stubs
|
| 138 |
+
LD_GOLD=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-ld.gold
|
| 139 |
+
LD_LIBRARY_PATH=/usr/mpi/gcc/openmpi-4.1.7a1/lib:/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64
|
| 140 |
+
LD_LIBRARY_PATH_modshare=/cm/shared/apps/slurm/current/lib64:1:/usr/mpi/gcc/openmpi-4.1.7a1/lib:1:/cm/shared/apps/slurm/current/lib64/slurm:1
|
| 141 |
+
LD_RUN_PATH=/usr/mpi/gcc/openmpi-4.1.7a1/lib
|
| 142 |
+
LD_RUN_PATH_modshare=/usr/mpi/gcc/openmpi-4.1.7a1/lib:1
|
| 143 |
+
LESSCLOSE=/usr/bin/lesspipe %s %s
|
| 144 |
+
LESSOPEN=| /usr/bin/lesspipe %s
|
| 145 |
+
LIBRARY_PATH=/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64
|
| 146 |
+
LIBRARY_PATH_modshare=/cm/shared/apps/slurm/current/lib64:1:/cm/shared/apps/slurm/current/lib64/slurm:1
|
| 147 |
+
LOADEDMODULES=slurm/slurm/23.02.6:gcc/64/4.1.7a1
|
| 148 |
+
LOADEDMODULES_modshare=slurm/slurm/23.02.6:1:gcc/64/4.1.7a1:1
|
| 149 |
+
LOCAL_RANK=0
|
| 150 |
+
LOCAL_SIZE=8
|
| 151 |
+
LOGLEVEL=INFO
|
| 152 |
+
LOGNAME=hansirui_1st
|
| 153 |
+
LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.webp=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
|
| 154 |
+
MANPATH=/usr/mpi/gcc/openmpi-4.1.7a1/share/man:/cm/shared/apps/slurm/current/man:/cm/local/apps/environment-modules/4.5.3/share/man:/usr/local/man:/usr/local/share/man:/usr/share/man:/cm/local/apps/environment-modules/current/share/man:/cm/local/apps/environment-modules/current/share/man
|
| 155 |
+
MANPATH_modshare=/usr/local/share/man:1:/usr/mpi/gcc/openmpi-4.1.7a1/share/man:1:/cm/local/apps/environment-modules/current/share/man:1:/cm/local/apps/environment-modules/4.5.3/share/man:1:/usr/local/man:1:/usr/share/man:1:/cm/shared/apps/slurm/current/man:1
|
| 156 |
+
MASTER_ADDR=127.0.0.1
|
| 157 |
+
MASTER_PORT=23603
|
| 158 |
+
MIG_PARTED_CHECKPOINT_FILE=/var/lib/nvidia-mig-manager/checkpoint.json
|
| 159 |
+
MIG_PARTED_CONFIG_FILE=/etc/nvidia-mig-manager/config.yaml
|
| 160 |
+
MIG_PARTED_HOOKS_FILE=/etc/nvidia-mig-manager/hooks.yaml
|
| 161 |
+
MODELSCOPE_CACHE=/aifs4su/hansirui_1st/.cache/modelscope/hub
|
| 162 |
+
MODULEPATH=/cm/local/modulefiles:/cm/shared/modulefiles
|
| 163 |
+
MODULESHOME=/cm/local/apps/environment-modules/4.5.3
|
| 164 |
+
MODULES_CMD=/cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl
|
| 165 |
+
MODULES_SET_SHELL_STARTUP=0
|
| 166 |
+
MOTD_SHOWN=pam
|
| 167 |
+
MPI_HOME=/usr/mpi/gcc/openmpi-4.1.7a1
|
| 168 |
+
MPI_RUN=/usr/mpi/gcc/openmpi-4.1.7a1/bin/mpirun
|
| 169 |
+
NM=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-nm
|
| 170 |
+
NVCC_PREPEND_FLAGS= -ccbin=/aifs4su/hansirui_1st/miniconda3/envs/by-verl/bin/x86_64-conda-linux-gnu-c++ -ccbin=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-c++
|
| 171 |
+
NVCC_PREPEND_FLAGS_BACKUP= -ccbin=/aifs4su/hansirui_1st/miniconda3/envs/by-verl/bin/x86_64-conda-linux-gnu-c++
|
| 172 |
+
OBJCOPY=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-objcopy
|
| 173 |
+
OBJDUMP=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-objdump
|
| 174 |
+
OLDPWD=/home/hansirui_1st/jiayi/resist/align-anything/scripts
|
| 175 |
+
PATH=/home/hansirui_1st/.local/bin:/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin:/usr/lpp/mmfs/bin:/usr/local/cuda/bin:/opt/bin:/home/hansirui_1st/.cursor-server/cli/servers/Stable-0781e811de386a0c5bcb07ceb259df8ff8246a50/server/bin/remote-cli:/home/hansirui_1st/.local/bin:/home/hansirui_1st/.local/bin:/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin:/aifs4su/hansirui_1st/miniconda3/condabin:/usr/mpi/gcc/openmpi-4.1.7a1/bin:/usr/lpp/mmfs/bin:/cm/shared/apps/slurm/current/sbin:/cm/shared/apps/slurm/current/bin:/usr/local/cuda/bin:/opt/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/sbin:/usr/sbin:/cm/local/apps/environment-modules/4.5.3/bin:/home/hansirui_1st/.cursor-server/cli/servers/Stable-0781e811de386a0c5bcb07ceb259df8ff8246a50/server/bin/remote-cli:/home/hansirui_1st/.local/bin:/home/hansirui_1st/.local/bin:/aifs4su/hansirui_1st/miniconda3/bin:/aifs4su/hansirui_1st/miniconda3/condabin:/usr/mpi/gcc/openmpi-4.1.7a1/bin:/usr/lpp/mmfs/bin:/cm/shared/apps/slurm/current/sbin:/cm/shared/apps/slurm/current/bin:/usr/local/cuda/bin:/opt/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/sbin:/usr/sbin:/cm/local/apps/environment-modules/4.5.3/bin:/home/hansirui_1st/.cursor-server/cli/servers/Stable-0781e811de386a0c5bcb07ceb259df8ff8246a50/server/bin/remote-cli:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
|
| 176 |
+
PATH_modshare=/usr/mpi/gcc/openmpi-4.1.7a1/bin:1:/opt/bin/:1:/usr/bin:1:/usr/local/bin:1:/cm/shared/apps/slurm/current/bin:1:/cm/shared/apps/slurm/current/sbin:1:/bin:1:/snap/bin:1:/sbin:1:/usr/sbin:1:/cm/local/apps/environment-modules/4.5.3/bin:1:/usr/games:1:/usr/local/sbin:1:/usr/lpp/mmfs/bin:1:/usr/local/cuda/bin:1:/usr/local/games:1
|
| 177 |
+
PWD=/home/hansirui_1st/jiayi/resist/setting3/scripts
|
| 178 |
+
PYTHONHASHSEED=42
|
| 179 |
+
PYTHONPATH=/home/hansirui_1st/jiayi/resist/setting3
|
| 180 |
+
RANK=0
|
| 181 |
+
RANLIB=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-ranlib
|
| 182 |
+
READELF=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-readelf
|
| 183 |
+
RUNTIME_ENV=./verl/trainer/runtime_env.yaml
|
| 184 |
+
SHELL=/bin/bash
|
| 185 |
+
SHLVL=4
|
| 186 |
+
SIZE=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-size
|
| 187 |
+
SLURM_CONF=/cm/shared/apps/slurm/var/etc/slurm/slurm.conf
|
| 188 |
+
SSH_CLIENT=10.33.4.76 48492 22
|
| 189 |
+
SSH_CONNECTION=10.33.4.51 50244 10.33.4.77 22
|
| 190 |
+
SSH_TTY=/dev/pts/56
|
| 191 |
+
STRINGS=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-strings
|
| 192 |
+
STRIP=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/x86_64-conda-linux-gnu-strip
|
| 193 |
+
TERM=screen
|
| 194 |
+
TERM_PROGRAM=tmux
|
| 195 |
+
TERM_PROGRAM_VERSION=3.2a
|
| 196 |
+
TMUX=/tmp/tmux-1081/default,531470,15
|
| 197 |
+
TMUX_PANE=%17
|
| 198 |
+
USER=hansirui_1st
|
| 199 |
+
WANDB_API_KEY=0e77f7c02e33b86269ca2123964b9fefcf9c1a7a
|
| 200 |
+
WANDB_SERVICE=2-2755439-tcp-localhost-34757
|
| 201 |
+
WORKING_DIR=/home/hansirui_1st/boyuan/mm_r1_workspace/xhs_workspace/multimodal_agent_rl/verl
|
| 202 |
+
WORLD_SIZE=8
|
| 203 |
+
XDG_DATA_DIRS=/usr/local/share:/usr/share:/var/lib/snapd/desktop
|
| 204 |
+
XDG_RUNTIME_DIR=/run/user/1081
|
| 205 |
+
XDG_SESSION_CLASS=user
|
| 206 |
+
XDG_SESSION_ID=71669
|
| 207 |
+
XDG_SESSION_TYPE=tty
|
| 208 |
+
_=/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/deepspeed
|
| 209 |
+
_CONDA_PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_x86_64_conda_cos7_linux_gnu
|
| 210 |
+
_LMFILES_=/cm/local/modulefiles/slurm/slurm/23.02.6:/cm/local/modulefiles/gcc/64/4.1.7a1
|
| 211 |
+
_LMFILES__modshare=/cm/local/modulefiles/slurm/slurm/23.02.6:1:/cm/local/modulefiles/gcc/64/4.1.7a1:1
|
| 212 |
+
build_alias=x86_64-conda-linux-gnu
|
| 213 |
+
host_alias=x86_64-conda-linux-gnu
|
pytorch_model.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2479f1692b0e7d63d21f568e8bbf56fbc863863292f82bc52db33048081b1926
|
| 3 |
+
size 2200170094
|
script.sh
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
#
|
| 3 |
+
# Copyright 2024 PKU-Alignment Team. All Rights Reserved.
|
| 4 |
+
#
|
| 5 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 6 |
+
# you may not use this file except in compliance with the License.
|
| 7 |
+
# You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 12 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 13 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 14 |
+
# See the License for the specific language governing permissions and
|
| 15 |
+
# limitations under the License.
|
| 16 |
+
# ==============================================================================
|
| 17 |
+
|
| 18 |
+
if [ -z "${BASH_VERSION}" ]; then
|
| 19 |
+
echo "Please use bash to run this script." >&2
|
| 20 |
+
exit 1
|
| 21 |
+
fi
|
| 22 |
+
|
| 23 |
+
set -x
|
| 24 |
+
|
| 25 |
+
SCRIPT_DIR="$(cd "$(dirname "$0")" &>/dev/null && pwd)"
|
| 26 |
+
ROOT_DIR="$(dirname "${SCRIPT_DIR}")"
|
| 27 |
+
export PYTHONPATH="${ROOT_DIR}${PYTHONPATH:+:${PYTHONPATH}}"
|
| 28 |
+
export LOGLEVEL="${LOGLEVEL:-WARNING}"
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
# -------------------need to change-------------
|
| 33 |
+
export LOGLEVEL="INFO"
|
| 34 |
+
export WANDB_API_KEY="0e77f7c02e33b86269ca2123964b9fefcf9c1a7a"
|
| 35 |
+
# MODEL_NAME_OR_PATH="/mnt/fl/models/reproduced/alpaca-7b-reproduced"
|
| 36 |
+
# OUTPUT_DIR="/mnt/fl/projects/unalignment/models/alpaca-reproduced-s3-Q1"
|
| 37 |
+
# DATASET="/mnt/fl/projects/unalignment/boyuan/workspace/inverse_alignment/ablation/setting2/dataset/training/safe/safe_train_20k.json"
|
| 38 |
+
# -------------------need to change------------
|
| 39 |
+
|
| 40 |
+
unset HOSTFILE
|
| 41 |
+
ZERO_STAGE=3
|
| 42 |
+
OFFLOAD="none"
|
| 43 |
+
LOG_RUN_NAME='setting3-default'
|
| 44 |
+
while [[ "$#" -gt 0 ]]; do
|
| 45 |
+
arg="$1"
|
| 46 |
+
shift
|
| 47 |
+
case "${arg}" in
|
| 48 |
+
--train_datasets)
|
| 49 |
+
DATASET="$1"
|
| 50 |
+
shift
|
| 51 |
+
;;
|
| 52 |
+
--train_datasets=*)
|
| 53 |
+
DATASET="${arg#*=}"
|
| 54 |
+
;;
|
| 55 |
+
--model_name_or_path)
|
| 56 |
+
MODEL_NAME_OR_PATH="$1"
|
| 57 |
+
shift
|
| 58 |
+
;;
|
| 59 |
+
--model_name_or_path=*)
|
| 60 |
+
MODEL_NAME_OR_PATH="${arg#*=}"
|
| 61 |
+
;;
|
| 62 |
+
--output_dir)
|
| 63 |
+
OUTPUT_DIR="$1"
|
| 64 |
+
shift
|
| 65 |
+
;;
|
| 66 |
+
--output_dir=*)
|
| 67 |
+
OUTPUT_DIR="${arg#*=}"
|
| 68 |
+
;;
|
| 69 |
+
--log_run_name)
|
| 70 |
+
LOG_RUN_NAME="$1"
|
| 71 |
+
shift
|
| 72 |
+
;;
|
| 73 |
+
--log_run_name=*)
|
| 74 |
+
LOG_RUN_NAME="${arg#*=}"
|
| 75 |
+
;;
|
| 76 |
+
--hostfile)
|
| 77 |
+
HOSTFILE="$1"
|
| 78 |
+
shift
|
| 79 |
+
;;
|
| 80 |
+
--hostfile=*)
|
| 81 |
+
HOSTFILE="${arg#*=}"
|
| 82 |
+
;;
|
| 83 |
+
--zero_stage)
|
| 84 |
+
ZERO_STAGE="$1"
|
| 85 |
+
shift
|
| 86 |
+
;;
|
| 87 |
+
--zero_stage=*)
|
| 88 |
+
ZERO_STAGE="${arg#*=}"
|
| 89 |
+
;;
|
| 90 |
+
--offload)
|
| 91 |
+
OFFLOAD="$1"
|
| 92 |
+
shift
|
| 93 |
+
;;
|
| 94 |
+
--offload=*)
|
| 95 |
+
OFFLOAD="${arg#*=}"
|
| 96 |
+
;;
|
| 97 |
+
*)
|
| 98 |
+
echo "Unknown parameter passed: '${arg}'" >&2
|
| 99 |
+
exit 1
|
| 100 |
+
;;
|
| 101 |
+
esac
|
| 102 |
+
done
|
| 103 |
+
|
| 104 |
+
mkdir -p "${OUTPUT_DIR}"
|
| 105 |
+
OUTPUT_DIR="$(cd "${OUTPUT_DIR}" &>/dev/null && pwd)"
|
| 106 |
+
if [[ ! -f "${OUTPUT_DIR}/.gitignore" ]]; then
|
| 107 |
+
echo '*' >"${OUTPUT_DIR}/.gitignore"
|
| 108 |
+
fi
|
| 109 |
+
|
| 110 |
+
cp -f "$0" "${OUTPUT_DIR}/script.sh"
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
MASTER_PORT_START=10000
|
| 114 |
+
MASTER_PORT_END=65535
|
| 115 |
+
MASTER_PORT="$(
|
| 116 |
+
comm -23 \
|
| 117 |
+
<(seq "${MASTER_PORT_START}" "${MASTER_PORT_END}" | sort) \
|
| 118 |
+
<(ss -Htan | awk '{ print $4 }' | awk -F ':' '{ print $NF }' | sort -u) |
|
| 119 |
+
shuf | head -n 1
|
| 120 |
+
)"
|
| 121 |
+
|
| 122 |
+
DEEPSPEED_ARGS=()
|
| 123 |
+
if [[ -n "${HOSTFILE+x}" ]]; then
|
| 124 |
+
DEEPSPEED_ARGS+=("--hostfile" "${HOSTFILE}")
|
| 125 |
+
fi
|
| 126 |
+
DEEPSPEED_ARGS+=("--master_port" "${MASTER_PORT}")
|
| 127 |
+
|
| 128 |
+
exec 1> >(tee "${OUTPUT_DIR}/stdout.log" >&1) 2> >(tee "${OUTPUT_DIR}/stderr.log" >&2)
|
| 129 |
+
|
| 130 |
+
deepspeed "${DEEPSPEED_ARGS[@]}" \
|
| 131 |
+
--module safe_rlhf.finetune \
|
| 132 |
+
--train_datasets inverse-json::${DATASET} \
|
| 133 |
+
--model_name_or_path "${MODEL_NAME_OR_PATH}" \
|
| 134 |
+
--max_length 512 \
|
| 135 |
+
--trust_remote_code True \
|
| 136 |
+
--epochs 1 \
|
| 137 |
+
--per_device_train_batch_size 1 \
|
| 138 |
+
--per_device_eval_batch_size 4 \
|
| 139 |
+
--gradient_accumulation_steps 8 \
|
| 140 |
+
--gradient_checkpointing \
|
| 141 |
+
--learning_rate 1e-5 \
|
| 142 |
+
--lr_warmup_ratio 0 \
|
| 143 |
+
--weight_decay 0.0 \
|
| 144 |
+
--lr_scheduler_type constant \
|
| 145 |
+
--weight_decay 0.0 \
|
| 146 |
+
--seed 42 \
|
| 147 |
+
--output_dir "${OUTPUT_DIR}" \
|
| 148 |
+
--log_type wandb \
|
| 149 |
+
--log_run_name "${LOG_RUN_NAME}" \
|
| 150 |
+
--log_project Inverse_Alignment_IMDb \
|
| 151 |
+
--zero_stage "${ZERO_STAGE}" \
|
| 152 |
+
--offload "${OFFLOAD}" \
|
| 153 |
+
--bf16 True \
|
| 154 |
+
--tf32 True \
|
| 155 |
+
--save_16bit
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<s>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "</s>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": {
|
| 17 |
+
"content": "<pad>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
},
|
| 23 |
+
"unk_token": {
|
| 24 |
+
"content": "<unk>",
|
| 25 |
+
"lstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"rstrip": false,
|
| 28 |
+
"single_word": false
|
| 29 |
+
}
|
| 30 |
+
}
|
stderr.log
ADDED
|
@@ -0,0 +1,585 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
+ deepspeed --master_port 23603 --module safe_rlhf.finetune --train_datasets inverse-json::/home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json --model_name_or_path /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000 --max_length 512 --trust_remote_code True --epochs 1 --per_device_train_batch_size 1 --per_device_eval_batch_size 4 --gradient_accumulation_steps 8 --gradient_checkpointing --learning_rate 1e-5 --lr_warmup_ratio 0 --weight_decay 0.0 --lr_scheduler_type constant --weight_decay 0.0 --seed 42 --output_dir /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500 --log_type wandb --log_run_name imdb-tinyllama-3T-s3-Q1-5000-Q2-500 --log_project Inverse_Alignment_IMDb --zero_stage 3 --offload none --bf16 True --tf32 True --save_16bit
|
| 2 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 3 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 4 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 5 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 6 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 7 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 8 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 9 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 10 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 11 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 12 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 13 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 14 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 15 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 16 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 17 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 18 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 19 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 20 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 21 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 22 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 23 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 24 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 25 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 26 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 27 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 28 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 29 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 30 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 31 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 32 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 33 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 34 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 35 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 36 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 37 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 38 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 39 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 40 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 41 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 42 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 43 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 44 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 45 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 46 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 47 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 48 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 49 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 50 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 51 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 52 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 53 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 54 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 55 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 56 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 57 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 58 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 59 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 60 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 61 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 62 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 63 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 64 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 65 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 66 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 67 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 68 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 69 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 70 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 71 |
+
nvcc warning : incompatible redefinition for option 'compiler-bindir', the last value of this option was used
|
| 72 |
+
[rank1]:[W527 21:12:38.622731692 ProcessGroupNCCL.cpp:4561] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
|
| 73 |
+
[rank3]:[W527 21:12:38.637424153 ProcessGroupNCCL.cpp:4561] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
|
| 74 |
+
[rank4]:[W527 21:12:38.639646167 ProcessGroupNCCL.cpp:4561] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
|
| 75 |
+
[rank5]:[W527 21:12:38.639688636 ProcessGroupNCCL.cpp:4561] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
|
| 76 |
+
[rank7]:[W527 21:12:38.639701574 ProcessGroupNCCL.cpp:4561] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
|
| 77 |
+
[rank2]:[W527 21:12:38.643489361 ProcessGroupNCCL.cpp:4561] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
|
| 78 |
+
[rank0]:[W527 21:12:39.656231963 ProcessGroupNCCL.cpp:4561] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
|
| 79 |
+
[rank6]:[W527 21:12:39.712441946 ProcessGroupNCCL.cpp:4561] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
|
| 80 |
+
loading configuration file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/config.json
|
| 81 |
+
loading configuration file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/config.json
|
| 82 |
+
loading configuration file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/config.json
|
| 83 |
+
loading configuration file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/config.json
|
| 84 |
+
loading configuration file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/config.json
|
| 85 |
+
loading configuration file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/config.json
|
| 86 |
+
loading configuration file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/config.json
|
| 87 |
+
loading configuration file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/config.json
|
| 88 |
+
Model config LlamaConfig {
|
| 89 |
+
"architectures": [
|
| 90 |
+
"LlamaForCausalLM"
|
| 91 |
+
],
|
| 92 |
+
"attention_bias": false,
|
| 93 |
+
"attention_dropout": 0.0,
|
| 94 |
+
"bos_token_id": 1,
|
| 95 |
+
"eos_token_id": 2,
|
| 96 |
+
"head_dim": 64,
|
| 97 |
+
"hidden_act": "silu",
|
| 98 |
+
"hidden_size": 2048,
|
| 99 |
+
"initializer_range": 0.02,
|
| 100 |
+
"intermediate_size": 5632,
|
| 101 |
+
"max_position_embeddings": 2048,
|
| 102 |
+
"mlp_bias": false,
|
| 103 |
+
"model_type": "llama",
|
| 104 |
+
"num_attention_heads": 32,
|
| 105 |
+
"num_hidden_layers": 22,
|
| 106 |
+
"num_key_value_heads": 4,
|
| 107 |
+
"pad_token_id": 32000,
|
| 108 |
+
"pretraining_tp": 1,
|
| 109 |
+
"rms_norm_eps": 1e-05,
|
| 110 |
+
"rope_scaling": null,
|
| 111 |
+
"rope_theta": 10000.0,
|
| 112 |
+
"tie_word_embeddings": false,
|
| 113 |
+
"torch_dtype": "float32",
|
| 114 |
+
"transformers_version": "4.52.1",
|
| 115 |
+
"use_cache": true,
|
| 116 |
+
"vocab_size": 32001
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
Model config LlamaConfig {
|
| 120 |
+
"architectures": [
|
| 121 |
+
"LlamaForCausalLM"
|
| 122 |
+
],
|
| 123 |
+
"attention_bias": false,
|
| 124 |
+
"attention_dropout": 0.0,
|
| 125 |
+
"bos_token_id": 1,
|
| 126 |
+
"eos_token_id": 2,
|
| 127 |
+
"head_dim": 64,
|
| 128 |
+
"hidden_act": "silu",
|
| 129 |
+
"hidden_size": 2048,
|
| 130 |
+
"initializer_range": 0.02,
|
| 131 |
+
"intermediate_size": 5632,
|
| 132 |
+
"max_position_embeddings": 2048,
|
| 133 |
+
"mlp_bias": false,
|
| 134 |
+
"model_type": "llama",
|
| 135 |
+
"num_attention_heads": 32,
|
| 136 |
+
"num_hidden_layers": 22,
|
| 137 |
+
"num_key_value_heads": 4,
|
| 138 |
+
"pad_token_id": 32000,
|
| 139 |
+
"pretraining_tp": 1,
|
| 140 |
+
"rms_norm_eps": 1e-05,
|
| 141 |
+
"rope_scaling": null,
|
| 142 |
+
"rope_theta": 10000.0,
|
| 143 |
+
"tie_word_embeddings": false,
|
| 144 |
+
"torch_dtype": "float32",
|
| 145 |
+
"transformers_version": "4.52.1",
|
| 146 |
+
"use_cache": true,
|
| 147 |
+
"vocab_size": 32001
|
| 148 |
+
}
|
| 149 |
+
|
| 150 |
+
Model config LlamaConfig {
|
| 151 |
+
"architectures": [
|
| 152 |
+
"LlamaForCausalLM"
|
| 153 |
+
],
|
| 154 |
+
"attention_bias": false,
|
| 155 |
+
"attention_dropout": 0.0,
|
| 156 |
+
"bos_token_id": 1,
|
| 157 |
+
"eos_token_id": 2,
|
| 158 |
+
"head_dim": 64,
|
| 159 |
+
"hidden_act": "silu",
|
| 160 |
+
"hidden_size": 2048,
|
| 161 |
+
"initializer_range": 0.02,
|
| 162 |
+
"intermediate_size": 5632,
|
| 163 |
+
"max_position_embeddings": 2048,
|
| 164 |
+
"mlp_bias": false,
|
| 165 |
+
"model_type": "llama",
|
| 166 |
+
"num_attention_heads": 32,
|
| 167 |
+
"num_hidden_layers": 22,
|
| 168 |
+
"num_key_value_heads": 4,
|
| 169 |
+
"pad_token_id": 32000,
|
| 170 |
+
"pretraining_tp": 1,
|
| 171 |
+
"rms_norm_eps": 1e-05,
|
| 172 |
+
"rope_scaling": null,
|
| 173 |
+
"rope_theta": 10000.0,
|
| 174 |
+
"tie_word_embeddings": false,
|
| 175 |
+
"torch_dtype": "float32",
|
| 176 |
+
"transformers_version": "4.52.1",
|
| 177 |
+
"use_cache": true,
|
| 178 |
+
"vocab_size": 32001
|
| 179 |
+
}
|
| 180 |
+
|
| 181 |
+
Model config LlamaConfig {
|
| 182 |
+
"architectures": [
|
| 183 |
+
"LlamaForCausalLM"
|
| 184 |
+
],
|
| 185 |
+
"attention_bias": false,
|
| 186 |
+
"attention_dropout": 0.0,
|
| 187 |
+
"bos_token_id": 1,
|
| 188 |
+
"eos_token_id": 2,
|
| 189 |
+
"head_dim": 64,
|
| 190 |
+
"hidden_act": "silu",
|
| 191 |
+
"hidden_size": 2048,
|
| 192 |
+
"initializer_range": 0.02,
|
| 193 |
+
"intermediate_size": 5632,
|
| 194 |
+
"max_position_embeddings": 2048,
|
| 195 |
+
"mlp_bias": false,
|
| 196 |
+
"model_type": "llama",
|
| 197 |
+
"num_attention_heads": 32,
|
| 198 |
+
"num_hidden_layers": 22,
|
| 199 |
+
"num_key_value_heads": 4,
|
| 200 |
+
"pad_token_id": 32000,
|
| 201 |
+
"pretraining_tp": 1,
|
| 202 |
+
"rms_norm_eps": 1e-05,
|
| 203 |
+
"rope_scaling": null,
|
| 204 |
+
"rope_theta": 10000.0,
|
| 205 |
+
"tie_word_embeddings": false,
|
| 206 |
+
"torch_dtype": "float32",
|
| 207 |
+
"transformers_version": "4.52.1",
|
| 208 |
+
"use_cache": true,
|
| 209 |
+
"vocab_size": 32001
|
| 210 |
+
}
|
| 211 |
+
|
| 212 |
+
Model config LlamaConfig {
|
| 213 |
+
"architectures": [
|
| 214 |
+
"LlamaForCausalLM"
|
| 215 |
+
],
|
| 216 |
+
"attention_bias": false,
|
| 217 |
+
"attention_dropout": 0.0,
|
| 218 |
+
"bos_token_id": 1,
|
| 219 |
+
"eos_token_id": 2,
|
| 220 |
+
"head_dim": 64,
|
| 221 |
+
"hidden_act": "silu",
|
| 222 |
+
"hidden_size": 2048,
|
| 223 |
+
"initializer_range": 0.02,
|
| 224 |
+
"intermediate_size": 5632,
|
| 225 |
+
"max_position_embeddings": 2048,
|
| 226 |
+
"mlp_bias": false,
|
| 227 |
+
"model_type": "llama",
|
| 228 |
+
"num_attention_heads": 32,
|
| 229 |
+
"num_hidden_layers": 22,
|
| 230 |
+
"num_key_value_heads": 4,
|
| 231 |
+
"pad_token_id": 32000,
|
| 232 |
+
"pretraining_tp": 1,
|
| 233 |
+
"rms_norm_eps": 1e-05,
|
| 234 |
+
"rope_scaling": null,
|
| 235 |
+
"rope_theta": 10000.0,
|
| 236 |
+
"tie_word_embeddings": false,
|
| 237 |
+
"torch_dtype": "float32",
|
| 238 |
+
"transformers_version": "4.52.1",
|
| 239 |
+
"use_cache": true,
|
| 240 |
+
"vocab_size": 32001
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
Model config LlamaConfig {
|
| 244 |
+
"architectures": [
|
| 245 |
+
"LlamaForCausalLM"
|
| 246 |
+
],
|
| 247 |
+
"attention_bias": false,
|
| 248 |
+
"attention_dropout": 0.0,
|
| 249 |
+
"bos_token_id": 1,
|
| 250 |
+
"eos_token_id": 2,
|
| 251 |
+
"head_dim": 64,
|
| 252 |
+
"hidden_act": "silu",
|
| 253 |
+
"hidden_size": 2048,
|
| 254 |
+
"initializer_range": 0.02,
|
| 255 |
+
"intermediate_size": 5632,
|
| 256 |
+
"max_position_embeddings": 2048,
|
| 257 |
+
"mlp_bias": false,
|
| 258 |
+
"model_type": "llama",
|
| 259 |
+
"num_attention_heads": 32,
|
| 260 |
+
"num_hidden_layers": 22,
|
| 261 |
+
"num_key_value_heads": 4,
|
| 262 |
+
"pad_token_id": 32000,
|
| 263 |
+
"pretraining_tp": 1,
|
| 264 |
+
"rms_norm_eps": 1e-05,
|
| 265 |
+
"rope_scaling": null,
|
| 266 |
+
"rope_theta": 10000.0,
|
| 267 |
+
"tie_word_embeddings": false,
|
| 268 |
+
"torch_dtype": "float32",
|
| 269 |
+
"transformers_version": "4.52.1",
|
| 270 |
+
"use_cache": true,
|
| 271 |
+
"vocab_size": 32001
|
| 272 |
+
}
|
| 273 |
+
|
| 274 |
+
Model config LlamaConfig {
|
| 275 |
+
"architectures": [
|
| 276 |
+
"LlamaForCausalLM"
|
| 277 |
+
],
|
| 278 |
+
"attention_bias": false,
|
| 279 |
+
"attention_dropout": 0.0,
|
| 280 |
+
"bos_token_id": 1,
|
| 281 |
+
"eos_token_id": 2,
|
| 282 |
+
"head_dim": 64,
|
| 283 |
+
"hidden_act": "silu",
|
| 284 |
+
"hidden_size": 2048,
|
| 285 |
+
"initializer_range": 0.02,
|
| 286 |
+
"intermediate_size": 5632,
|
| 287 |
+
"max_position_embeddings": 2048,
|
| 288 |
+
"mlp_bias": false,
|
| 289 |
+
"model_type": "llama",
|
| 290 |
+
"num_attention_heads": 32,
|
| 291 |
+
"num_hidden_layers": 22,
|
| 292 |
+
"num_key_value_heads": 4,
|
| 293 |
+
"pad_token_id": 32000,
|
| 294 |
+
"pretraining_tp": 1,
|
| 295 |
+
"rms_norm_eps": 1e-05,
|
| 296 |
+
"rope_scaling": null,
|
| 297 |
+
"rope_theta": 10000.0,
|
| 298 |
+
"tie_word_embeddings": false,
|
| 299 |
+
"torch_dtype": "float32",
|
| 300 |
+
"transformers_version": "4.52.1",
|
| 301 |
+
"use_cache": true,
|
| 302 |
+
"vocab_size": 32001
|
| 303 |
+
}
|
| 304 |
+
|
| 305 |
+
Model config LlamaConfig {
|
| 306 |
+
"architectures": [
|
| 307 |
+
"LlamaForCausalLM"
|
| 308 |
+
],
|
| 309 |
+
"attention_bias": false,
|
| 310 |
+
"attention_dropout": 0.0,
|
| 311 |
+
"bos_token_id": 1,
|
| 312 |
+
"eos_token_id": 2,
|
| 313 |
+
"head_dim": 64,
|
| 314 |
+
"hidden_act": "silu",
|
| 315 |
+
"hidden_size": 2048,
|
| 316 |
+
"initializer_range": 0.02,
|
| 317 |
+
"intermediate_size": 5632,
|
| 318 |
+
"max_position_embeddings": 2048,
|
| 319 |
+
"mlp_bias": false,
|
| 320 |
+
"model_type": "llama",
|
| 321 |
+
"num_attention_heads": 32,
|
| 322 |
+
"num_hidden_layers": 22,
|
| 323 |
+
"num_key_value_heads": 4,
|
| 324 |
+
"pad_token_id": 32000,
|
| 325 |
+
"pretraining_tp": 1,
|
| 326 |
+
"rms_norm_eps": 1e-05,
|
| 327 |
+
"rope_scaling": null,
|
| 328 |
+
"rope_theta": 10000.0,
|
| 329 |
+
"tie_word_embeddings": false,
|
| 330 |
+
"torch_dtype": "float32",
|
| 331 |
+
"transformers_version": "4.52.1",
|
| 332 |
+
"use_cache": true,
|
| 333 |
+
"vocab_size": 32001
|
| 334 |
+
}
|
| 335 |
+
|
| 336 |
+
loading weights file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/pytorch_model.bin
|
| 337 |
+
Will use torch_dtype=torch.float32 as defined in model's config object
|
| 338 |
+
Instantiating LlamaForCausalLM model under default dtype torch.float32.
|
| 339 |
+
loading weights file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/pytorch_model.bin
|
| 340 |
+
Detected DeepSpeed ZeRO-3: activating zero.init() for this model
|
| 341 |
+
Will use torch_dtype=torch.float32 as defined in model's config object
|
| 342 |
+
Instantiating LlamaForCausalLM model under default dtype torch.float32.
|
| 343 |
+
Detected DeepSpeed ZeRO-3: activating zero.init() for this model
|
| 344 |
+
loading weights file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/pytorch_model.bin
|
| 345 |
+
Will use torch_dtype=torch.float32 as defined in model's config object
|
| 346 |
+
Instantiating LlamaForCausalLM model under default dtype torch.float32.
|
| 347 |
+
Detected DeepSpeed ZeRO-3: activating zero.init() for this model
|
| 348 |
+
loading weights file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/pytorch_model.bin
|
| 349 |
+
Will use torch_dtype=torch.float32 as defined in model's config object
|
| 350 |
+
Instantiating LlamaForCausalLM model under default dtype torch.float32.
|
| 351 |
+
Detected DeepSpeed ZeRO-3: activating zero.init() for this model
|
| 352 |
+
loading weights file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/pytorch_model.bin
|
| 353 |
+
Will use torch_dtype=torch.float32 as defined in model's config object
|
| 354 |
+
Instantiating LlamaForCausalLM model under default dtype torch.float32.
|
| 355 |
+
Detected DeepSpeed ZeRO-3: activating zero.init() for this model
|
| 356 |
+
loading weights file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/pytorch_model.bin
|
| 357 |
+
Will use torch_dtype=torch.float32 as defined in model's config object
|
| 358 |
+
Instantiating LlamaForCausalLM model under default dtype torch.float32.
|
| 359 |
+
Detected DeepSpeed ZeRO-3: activating zero.init() for this model
|
| 360 |
+
loading weights file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/pytorch_model.bin
|
| 361 |
+
Will use torch_dtype=torch.float32 as defined in model's config object
|
| 362 |
+
Instantiating LlamaForCausalLM model under default dtype torch.float32.
|
| 363 |
+
Detected DeepSpeed ZeRO-3: activating zero.init() for this model
|
| 364 |
+
Generate config GenerationConfig {
|
| 365 |
+
"bos_token_id": 1,
|
| 366 |
+
"eos_token_id": 2,
|
| 367 |
+
"pad_token_id": 32000
|
| 368 |
+
}
|
| 369 |
+
|
| 370 |
+
Generate config GenerationConfig {
|
| 371 |
+
"bos_token_id": 1,
|
| 372 |
+
"eos_token_id": 2,
|
| 373 |
+
"pad_token_id": 32000
|
| 374 |
+
}
|
| 375 |
+
|
| 376 |
+
Generate config GenerationConfig {
|
| 377 |
+
"bos_token_id": 1,
|
| 378 |
+
"eos_token_id": 2,
|
| 379 |
+
"pad_token_id": 32000
|
| 380 |
+
}
|
| 381 |
+
|
| 382 |
+
Generate config GenerationConfig {
|
| 383 |
+
"bos_token_id": 1,
|
| 384 |
+
"eos_token_id": 2,
|
| 385 |
+
"pad_token_id": 32000
|
| 386 |
+
}
|
| 387 |
+
|
| 388 |
+
Generate config GenerationConfig {
|
| 389 |
+
"bos_token_id": 1,
|
| 390 |
+
"eos_token_id": 2,
|
| 391 |
+
"pad_token_id": 32000
|
| 392 |
+
}
|
| 393 |
+
|
| 394 |
+
Generate config GenerationConfig {
|
| 395 |
+
"bos_token_id": 1,
|
| 396 |
+
"eos_token_id": 2,
|
| 397 |
+
"pad_token_id": 32000
|
| 398 |
+
}
|
| 399 |
+
|
| 400 |
+
Generate config GenerationConfig {
|
| 401 |
+
"bos_token_id": 1,
|
| 402 |
+
"eos_token_id": 2,
|
| 403 |
+
"pad_token_id": 32000
|
| 404 |
+
}
|
| 405 |
+
|
| 406 |
+
loading weights file /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000/pytorch_model.bin
|
| 407 |
+
Will use torch_dtype=torch.float32 as defined in model's config object
|
| 408 |
+
Instantiating LlamaForCausalLM model under default dtype torch.float32.
|
| 409 |
+
Detected DeepSpeed ZeRO-3: activating zero.init() for this model
|
| 410 |
+
Generate config GenerationConfig {
|
| 411 |
+
"bos_token_id": 1,
|
| 412 |
+
"eos_token_id": 2,
|
| 413 |
+
"pad_token_id": 32000
|
| 414 |
+
}
|
| 415 |
+
|
| 416 |
+
All model checkpoint weights were used when initializing LlamaForCausalLM.
|
| 417 |
+
|
| 418 |
+
All model checkpoint weights were used when initializing LlamaForCausalLM.
|
| 419 |
+
|
| 420 |
+
All the weights of LlamaForCausalLM were initialized from the model checkpoint at /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000.
|
| 421 |
+
If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
|
| 422 |
+
All model checkpoint weights were used when initializing LlamaForCausalLM.
|
| 423 |
+
|
| 424 |
+
All the weights of LlamaForCausalLM were initialized from the model checkpoint at /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000.
|
| 425 |
+
If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
|
| 426 |
+
All the weights of LlamaForCausalLM were initialized from the model checkpoint at /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000.
|
| 427 |
+
If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
|
| 428 |
+
All model checkpoint weights were used when initializing LlamaForCausalLM.
|
| 429 |
+
|
| 430 |
+
All the weights of LlamaForCausalLM were initialized from the model checkpoint at /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000.
|
| 431 |
+
If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
|
| 432 |
+
All model checkpoint weights were used when initializing LlamaForCausalLM.
|
| 433 |
+
|
| 434 |
+
All the weights of LlamaForCausalLM were initialized from the model checkpoint at /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000.
|
| 435 |
+
If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
|
| 436 |
+
All model checkpoint weights were used when initializing LlamaForCausalLM.
|
| 437 |
+
|
| 438 |
+
All the weights of LlamaForCausalLM were initialized from the model checkpoint at /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000.
|
| 439 |
+
If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
|
| 440 |
+
All model checkpoint weights were used when initializing LlamaForCausalLM.
|
| 441 |
+
|
| 442 |
+
All the weights of LlamaForCausalLM were initialized from the model checkpoint at /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000.
|
| 443 |
+
If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
|
| 444 |
+
Generation config file not found, using a generation config created from the model config.
|
| 445 |
+
Generation config file not found, using a generation config created from the model config.
|
| 446 |
+
Generation config file not found, using a generation config created from the model config.
|
| 447 |
+
Generation config file not found, using a generation config created from the model config.
|
| 448 |
+
Generation config file not found, using a generation config created from the model config.
|
| 449 |
+
Generation config file not found, using a generation config created from the model config.
|
| 450 |
+
Generation config file not found, using a generation config created from the model config.
|
| 451 |
+
loading file tokenizer.model
|
| 452 |
+
loading file tokenizer.json
|
| 453 |
+
loading file added_tokens.json
|
| 454 |
+
loading file special_tokens_map.json
|
| 455 |
+
loading file tokenizer_config.json
|
| 456 |
+
loading file chat_template.jinja
|
| 457 |
+
loading file tokenizer.model
|
| 458 |
+
loading file tokenizer.json
|
| 459 |
+
loading file added_tokens.json
|
| 460 |
+
loading file special_tokens_map.json
|
| 461 |
+
loading file tokenizer_config.json
|
| 462 |
+
loading file chat_template.jinja
|
| 463 |
+
loading file tokenizer.model
|
| 464 |
+
loading file tokenizer.json
|
| 465 |
+
loading file added_tokens.json
|
| 466 |
+
loading file special_tokens_map.json
|
| 467 |
+
loading file tokenizer_config.json
|
| 468 |
+
loading file chat_template.jinja
|
| 469 |
+
loading file tokenizer.model
|
| 470 |
+
loading file tokenizer.json
|
| 471 |
+
loading file added_tokens.json
|
| 472 |
+
loading file special_tokens_map.json
|
| 473 |
+
loading file tokenizer_config.json
|
| 474 |
+
loading file chat_template.jinja
|
| 475 |
+
loading file tokenizer.model
|
| 476 |
+
loading file tokenizer.model
|
| 477 |
+
loading file tokenizer.json
|
| 478 |
+
loading file tokenizer.json
|
| 479 |
+
loading file added_tokens.json
|
| 480 |
+
loading file added_tokens.json
|
| 481 |
+
loading file special_tokens_map.json
|
| 482 |
+
loading file tokenizer_config.json
|
| 483 |
+
loading file special_tokens_map.json
|
| 484 |
+
loading file chat_template.jinja
|
| 485 |
+
loading file tokenizer_config.json
|
| 486 |
+
loading file chat_template.jinja
|
| 487 |
+
loading file tokenizer.model
|
| 488 |
+
loading file tokenizer.json
|
| 489 |
+
loading file added_tokens.json
|
| 490 |
+
loading file special_tokens_map.json
|
| 491 |
+
loading file tokenizer_config.json
|
| 492 |
+
loading file chat_template.jinja
|
| 493 |
+
All model checkpoint weights were used when initializing LlamaForCausalLM.
|
| 494 |
+
|
| 495 |
+
All the weights of LlamaForCausalLM were initialized from the model checkpoint at /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000.
|
| 496 |
+
If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
|
| 497 |
+
Generation config file not found, using a generation config created from the model config.
|
| 498 |
+
loading file tokenizer.model
|
| 499 |
+
loading file tokenizer.json
|
| 500 |
+
loading file added_tokens.json
|
| 501 |
+
loading file special_tokens_map.json
|
| 502 |
+
loading file tokenizer_config.json
|
| 503 |
+
loading file chat_template.jinja
|
| 504 |
+
Using /home/hansirui_1st/.cache/torch_extensions/py311_cu124 as PyTorch extensions root...
|
| 505 |
+
Using /home/hansirui_1st/.cache/torch_extensions/py311_cu124 as PyTorch extensions root...
|
| 506 |
+
Using /home/hansirui_1st/.cache/torch_extensions/py311_cu124 as PyTorch extensions root...
|
| 507 |
+
Using /home/hansirui_1st/.cache/torch_extensions/py311_cu124 as PyTorch extensions root...
|
| 508 |
+
Using /home/hansirui_1st/.cache/torch_extensions/py311_cu124 as PyTorch extensions root...
|
| 509 |
+
Using /home/hansirui_1st/.cache/torch_extensions/py311_cu124 as PyTorch extensions root...
|
| 510 |
+
Using /home/hansirui_1st/.cache/torch_extensions/py311_cu124 as PyTorch extensions root...
|
| 511 |
+
Using /home/hansirui_1st/.cache/torch_extensions/py311_cu124 as PyTorch extensions root...
|
| 512 |
+
Detected CUDA files, patching ldflags
|
| 513 |
+
Emitting ninja build file /home/hansirui_1st/.cache/torch_extensions/py311_cu124/fused_adam/build.ninja...
|
| 514 |
+
/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/torch/utils/cpp_extension.py:2059: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
|
| 515 |
+
If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
|
| 516 |
+
warnings.warn(
|
| 517 |
+
Building extension module fused_adam...
|
| 518 |
+
Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
|
| 519 |
+
Loading extension module fused_adam...
|
| 520 |
+
Loading extension module fused_adam...
|
| 521 |
+
Loading extension module fused_adam...
|
| 522 |
+
Loading extension module fused_adam...
|
| 523 |
+
Loading extension module fused_adam...
|
| 524 |
+
Loading extension module fused_adam...
|
| 525 |
+
Loading extension module fused_adam...
|
| 526 |
+
Loading extension module fused_adam...
|
| 527 |
+
`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
|
| 528 |
+
`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
|
| 529 |
+
`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
|
| 530 |
+
`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
|
| 531 |
+
`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
|
| 532 |
+
`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
|
| 533 |
+
`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
|
| 534 |
+
wandb: Currently logged in as: xtom to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 535 |
+
wandb: Tracking run with wandb version 0.19.11
|
| 536 |
+
wandb: Run data is saved locally in /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/wandb/run-20250527_211254-n7vbaf0c
|
| 537 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 538 |
+
wandb: Syncing run imdb-tinyllama-3T-s3-Q1-5000-Q2-500
|
| 539 |
+
wandb: ⭐️ View project at https://wandb.ai/xtom/Inverse_Alignment_IMDb
|
| 540 |
+
wandb: 🚀 View run at https://wandb.ai/xtom/Inverse_Alignment_IMDb/runs/n7vbaf0c
|
| 541 |
+
|
| 542 |
+
|
| 543 |
+
tokenizer config file saved in /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/tokenizer_config.json
|
| 544 |
+
Special tokens file saved in /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/special_tokens_map.json
|
| 545 |
+
wandb: ERROR Problem finishing run
|
| 546 |
+
Exception ignored in atexit callback: <bound method rank_zero_only.<locals>.wrapper of <safe_rlhf.logger.Logger object at 0x1550cc97c5d0>>
|
| 547 |
+
Traceback (most recent call last):
|
| 548 |
+
File "/home/hansirui_1st/jiayi/resist/setting3/safe_rlhf/utils.py", line 212, in wrapper
|
| 549 |
+
return func(*args, **kwargs)
|
| 550 |
+
^^^^^^^^^^^^^^^^^^^^^
|
| 551 |
+
File "/home/hansirui_1st/jiayi/resist/setting3/safe_rlhf/logger.py", line 183, in close
|
| 552 |
+
self.wandb.finish()
|
| 553 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 406, in wrapper
|
| 554 |
+
return func(self, *args, **kwargs)
|
| 555 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 556 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 503, in wrapper
|
| 557 |
+
return func(self, *args, **kwargs)
|
| 558 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 559 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 451, in wrapper
|
| 560 |
+
return func(self, *args, **kwargs)
|
| 561 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 562 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2309, in finish
|
| 563 |
+
return self._finish(exit_code)
|
| 564 |
+
^^^^^^^^^^^^^^^^^^^^^^^
|
| 565 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 406, in wrapper
|
| 566 |
+
return func(self, *args, **kwargs)
|
| 567 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 568 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2337, in _finish
|
| 569 |
+
self._atexit_cleanup(exit_code=exit_code)
|
| 570 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2550, in _atexit_cleanup
|
| 571 |
+
self._on_finish()
|
| 572 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2806, in _on_finish
|
| 573 |
+
wait_with_progress(
|
| 574 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/mailbox/wait_with_progress.py", line 24, in wait_with_progress
|
| 575 |
+
return wait_all_with_progress(
|
| 576 |
+
^^^^^^^^^^^^^^^^^^^^^^^
|
| 577 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/mailbox/wait_with_progress.py", line 87, in wait_all_with_progress
|
| 578 |
+
return asyncio_compat.run(progress_loop_with_timeout)
|
| 579 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 580 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/lib/asyncio_compat.py", line 27, in run
|
| 581 |
+
future = executor.submit(runner.run, fn)
|
| 582 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 583 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/concurrent/futures/thread.py", line 169, in submit
|
| 584 |
+
raise RuntimeError('cannot schedule new futures after '
|
| 585 |
+
RuntimeError: cannot schedule new futures after interpreter shutdown
|
stdout.log
ADDED
|
@@ -0,0 +1,303 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2025-05-27 21:12:11,001] [INFO] [real_accelerator.py:239:get_accelerator] Setting ds_accelerator to cuda (auto detect)
|
| 2 |
+
Warning: The cache directory for DeepSpeed Triton autotune, /home/hansirui_1st/.triton/autotune, appears to be on an NFS system. While this is generally acceptable, if you experience slowdowns or hanging when DeepSpeed exits, it is recommended to set the TRITON_CACHE_DIR environment variable to a non-NFS path.
|
| 3 |
+
[2025-05-27 21:12:15,999] [WARNING] [runner.py:215:fetch_hostfile] Unable to find hostfile, will proceed with training with local resources only.
|
| 4 |
+
[2025-05-27 21:12:15,999] [INFO] [runner.py:605:main] cmd = /aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/python3.11 -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbMCwgMSwgMiwgMywgNCwgNSwgNiwgN119 --master_addr=127.0.0.1 --master_port=23603 --module --enable_each_rank_log=None safe_rlhf.finetune --train_datasets inverse-json::/home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json --model_name_or_path /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000 --max_length 512 --trust_remote_code True --epochs 1 --per_device_train_batch_size 1 --per_device_eval_batch_size 4 --gradient_accumulation_steps 8 --gradient_checkpointing --learning_rate 1e-5 --lr_warmup_ratio 0 --weight_decay 0.0 --lr_scheduler_type constant --weight_decay 0.0 --seed 42 --output_dir /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500 --log_type wandb --log_run_name imdb-tinyllama-3T-s3-Q1-5000-Q2-500 --log_project Inverse_Alignment_IMDb --zero_stage 3 --offload none --bf16 True --tf32 True --save_16bit
|
| 5 |
+
[2025-05-27 21:12:18,181] [INFO] [real_accelerator.py:239:get_accelerator] Setting ds_accelerator to cuda (auto detect)
|
| 6 |
+
Warning: The cache directory for DeepSpeed Triton autotune, /home/hansirui_1st/.triton/autotune, appears to be on an NFS system. While this is generally acceptable, if you experience slowdowns or hanging when DeepSpeed exits, it is recommended to set the TRITON_CACHE_DIR environment variable to a non-NFS path.
|
| 7 |
+
[2025-05-27 21:12:23,173] [INFO] [launch.py:146:main] WORLD INFO DICT: {'localhost': [0, 1, 2, 3, 4, 5, 6, 7]}
|
| 8 |
+
[2025-05-27 21:12:23,173] [INFO] [launch.py:152:main] nnodes=1, num_local_procs=8, node_rank=0
|
| 9 |
+
[2025-05-27 21:12:23,173] [INFO] [launch.py:163:main] global_rank_mapping=defaultdict(<class 'list'>, {'localhost': [0, 1, 2, 3, 4, 5, 6, 7]})
|
| 10 |
+
[2025-05-27 21:12:23,173] [INFO] [launch.py:164:main] dist_world_size=8
|
| 11 |
+
[2025-05-27 21:12:23,173] [INFO] [launch.py:168:main] Setting CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 12 |
+
[2025-05-27 21:12:23,174] [INFO] [launch.py:256:main] process 2755439 spawned with command: ['/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/python3.11', '-u', '-m', 'safe_rlhf.finetune', '--local_rank=0', '--train_datasets', 'inverse-json::/home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json', '--model_name_or_path', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000', '--max_length', '512', '--trust_remote_code', 'True', '--epochs', '1', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '8', '--gradient_checkpointing', '--learning_rate', '1e-5', '--lr_warmup_ratio', '0', '--weight_decay', '0.0', '--lr_scheduler_type', 'constant', '--weight_decay', '0.0', '--seed', '42', '--output_dir', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500', '--log_type', 'wandb', '--log_run_name', 'imdb-tinyllama-3T-s3-Q1-5000-Q2-500', '--log_project', 'Inverse_Alignment_IMDb', '--zero_stage', '3', '--offload', 'none', '--bf16', 'True', '--tf32', 'True', '--save_16bit']
|
| 13 |
+
[2025-05-27 21:12:23,175] [INFO] [launch.py:256:main] process 2755440 spawned with command: ['/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/python3.11', '-u', '-m', 'safe_rlhf.finetune', '--local_rank=1', '--train_datasets', 'inverse-json::/home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json', '--model_name_or_path', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000', '--max_length', '512', '--trust_remote_code', 'True', '--epochs', '1', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '8', '--gradient_checkpointing', '--learning_rate', '1e-5', '--lr_warmup_ratio', '0', '--weight_decay', '0.0', '--lr_scheduler_type', 'constant', '--weight_decay', '0.0', '--seed', '42', '--output_dir', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500', '--log_type', 'wandb', '--log_run_name', 'imdb-tinyllama-3T-s3-Q1-5000-Q2-500', '--log_project', 'Inverse_Alignment_IMDb', '--zero_stage', '3', '--offload', 'none', '--bf16', 'True', '--tf32', 'True', '--save_16bit']
|
| 14 |
+
[2025-05-27 21:12:23,175] [INFO] [launch.py:256:main] process 2755441 spawned with command: ['/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/python3.11', '-u', '-m', 'safe_rlhf.finetune', '--local_rank=2', '--train_datasets', 'inverse-json::/home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json', '--model_name_or_path', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000', '--max_length', '512', '--trust_remote_code', 'True', '--epochs', '1', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '8', '--gradient_checkpointing', '--learning_rate', '1e-5', '--lr_warmup_ratio', '0', '--weight_decay', '0.0', '--lr_scheduler_type', 'constant', '--weight_decay', '0.0', '--seed', '42', '--output_dir', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500', '--log_type', 'wandb', '--log_run_name', 'imdb-tinyllama-3T-s3-Q1-5000-Q2-500', '--log_project', 'Inverse_Alignment_IMDb', '--zero_stage', '3', '--offload', 'none', '--bf16', 'True', '--tf32', 'True', '--save_16bit']
|
| 15 |
+
[2025-05-27 21:12:23,176] [INFO] [launch.py:256:main] process 2755442 spawned with command: ['/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/python3.11', '-u', '-m', 'safe_rlhf.finetune', '--local_rank=3', '--train_datasets', 'inverse-json::/home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json', '--model_name_or_path', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000', '--max_length', '512', '--trust_remote_code', 'True', '--epochs', '1', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '8', '--gradient_checkpointing', '--learning_rate', '1e-5', '--lr_warmup_ratio', '0', '--weight_decay', '0.0', '--lr_scheduler_type', 'constant', '--weight_decay', '0.0', '--seed', '42', '--output_dir', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500', '--log_type', 'wandb', '--log_run_name', 'imdb-tinyllama-3T-s3-Q1-5000-Q2-500', '--log_project', 'Inverse_Alignment_IMDb', '--zero_stage', '3', '--offload', 'none', '--bf16', 'True', '--tf32', 'True', '--save_16bit']
|
| 16 |
+
[2025-05-27 21:12:23,176] [INFO] [launch.py:256:main] process 2755443 spawned with command: ['/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/python3.11', '-u', '-m', 'safe_rlhf.finetune', '--local_rank=4', '--train_datasets', 'inverse-json::/home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json', '--model_name_or_path', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000', '--max_length', '512', '--trust_remote_code', 'True', '--epochs', '1', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '8', '--gradient_checkpointing', '--learning_rate', '1e-5', '--lr_warmup_ratio', '0', '--weight_decay', '0.0', '--lr_scheduler_type', 'constant', '--weight_decay', '0.0', '--seed', '42', '--output_dir', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500', '--log_type', 'wandb', '--log_run_name', 'imdb-tinyllama-3T-s3-Q1-5000-Q2-500', '--log_project', 'Inverse_Alignment_IMDb', '--zero_stage', '3', '--offload', 'none', '--bf16', 'True', '--tf32', 'True', '--save_16bit']
|
| 17 |
+
[2025-05-27 21:12:23,177] [INFO] [launch.py:256:main] process 2755444 spawned with command: ['/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/python3.11', '-u', '-m', 'safe_rlhf.finetune', '--local_rank=5', '--train_datasets', 'inverse-json::/home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json', '--model_name_or_path', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000', '--max_length', '512', '--trust_remote_code', 'True', '--epochs', '1', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '8', '--gradient_checkpointing', '--learning_rate', '1e-5', '--lr_warmup_ratio', '0', '--weight_decay', '0.0', '--lr_scheduler_type', 'constant', '--weight_decay', '0.0', '--seed', '42', '--output_dir', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500', '--log_type', 'wandb', '--log_run_name', 'imdb-tinyllama-3T-s3-Q1-5000-Q2-500', '--log_project', 'Inverse_Alignment_IMDb', '--zero_stage', '3', '--offload', 'none', '--bf16', 'True', '--tf32', 'True', '--save_16bit']
|
| 18 |
+
[2025-05-27 21:12:23,177] [INFO] [launch.py:256:main] process 2755445 spawned with command: ['/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/python3.11', '-u', '-m', 'safe_rlhf.finetune', '--local_rank=6', '--train_datasets', 'inverse-json::/home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json', '--model_name_or_path', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000', '--max_length', '512', '--trust_remote_code', 'True', '--epochs', '1', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '8', '--gradient_checkpointing', '--learning_rate', '1e-5', '--lr_warmup_ratio', '0', '--weight_decay', '0.0', '--lr_scheduler_type', 'constant', '--weight_decay', '0.0', '--seed', '42', '--output_dir', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500', '--log_type', 'wandb', '--log_run_name', 'imdb-tinyllama-3T-s3-Q1-5000-Q2-500', '--log_project', 'Inverse_Alignment_IMDb', '--zero_stage', '3', '--offload', 'none', '--bf16', 'True', '--tf32', 'True', '--save_16bit']
|
| 19 |
+
[2025-05-27 21:12:23,178] [INFO] [launch.py:256:main] process 2755446 spawned with command: ['/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/python3.11', '-u', '-m', 'safe_rlhf.finetune', '--local_rank=7', '--train_datasets', 'inverse-json::/home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json', '--model_name_or_path', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000', '--max_length', '512', '--trust_remote_code', 'True', '--epochs', '1', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '4', '--gradient_accumulation_steps', '8', '--gradient_checkpointing', '--learning_rate', '1e-5', '--lr_warmup_ratio', '0', '--weight_decay', '0.0', '--lr_scheduler_type', 'constant', '--weight_decay', '0.0', '--seed', '42', '--output_dir', '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500', '--log_type', 'wandb', '--log_run_name', 'imdb-tinyllama-3T-s3-Q1-5000-Q2-500', '--log_project', 'Inverse_Alignment_IMDb', '--zero_stage', '3', '--offload', 'none', '--bf16', 'True', '--tf32', 'True', '--save_16bit']
|
| 20 |
+
[2025-05-27 21:12:27,840] [INFO] [real_accelerator.py:239:get_accelerator] Setting ds_accelerator to cuda (auto detect)
|
| 21 |
+
[2025-05-27 21:12:27,897] [INFO] [real_accelerator.py:239:get_accelerator] Setting ds_accelerator to cuda (auto detect)
|
| 22 |
+
[2025-05-27 21:12:27,913] [INFO] [real_accelerator.py:239:get_accelerator] Setting ds_accelerator to cuda (auto detect)
|
| 23 |
+
[2025-05-27 21:12:28,016] [INFO] [real_accelerator.py:239:get_accelerator] Setting ds_accelerator to cuda (auto detect)
|
| 24 |
+
Warning: The cache directory for DeepSpeed Triton autotune, /home/hansirui_1st/.triton/autotune, appears to be on an NFS system. While this is generally acceptable, if you experience slowdowns or hanging when DeepSpeed exits, it is recommended to set the TRITON_CACHE_DIR environment variable to a non-NFS path.
|
| 25 |
+
[2025-05-27 21:12:28,063] [INFO] [real_accelerator.py:239:get_accelerator] Setting ds_accelerator to cuda (auto detect)
|
| 26 |
+
[2025-05-27 21:12:28,076] [INFO] [real_accelerator.py:239:get_accelerator] Setting ds_accelerator to cuda (auto detect)
|
| 27 |
+
[2025-05-27 21:12:28,083] [INFO] [real_accelerator.py:239:get_accelerator] Setting ds_accelerator to cuda (auto detect)
|
| 28 |
+
[2025-05-27 21:12:28,084] [INFO] [real_accelerator.py:239:get_accelerator] Setting ds_accelerator to cuda (auto detect)
|
| 29 |
+
Warning: The cache directory for DeepSpeed Triton autotune, /home/hansirui_1st/.triton/autotune, appears to be on an NFS system. While this is generally acceptable, if you experience slowdowns or hanging when DeepSpeed exits, it is recommended to set the TRITON_CACHE_DIR environment variable to a non-NFS path.
|
| 30 |
+
Warning: The cache directory for DeepSpeed Triton autotune, /home/hansirui_1st/.triton/autotune, appears to be on an NFS system. While this is generally acceptable, if you experience slowdowns or hanging when DeepSpeed exits, it is recommended to set the TRITON_CACHE_DIR environment variable to a non-NFS path.
|
| 31 |
+
Warning: The cache directory for DeepSpeed Triton autotune, /home/hansirui_1st/.triton/autotune, appears to be on an NFS system. While this is generally acceptable, if you experience slowdowns or hanging when DeepSpeed exits, it is recommended to set the TRITON_CACHE_DIR environment variable to a non-NFS path.
|
| 32 |
+
Warning: The cache directory for DeepSpeed Triton autotune, /home/hansirui_1st/.triton/autotune, appears to be on an NFS system. While this is generally acceptable, if you experience slowdowns or hanging when DeepSpeed exits, it is recommended to set the TRITON_CACHE_DIR environment variable to a non-NFS path.
|
| 33 |
+
Warning: The cache directory for DeepSpeed Triton autotune, /home/hansirui_1st/.triton/autotune, appears to be on an NFS system. While this is generally acceptable, if you experience slowdowns or hanging when DeepSpeed exits, it is recommended to set the TRITON_CACHE_DIR environment variable to a non-NFS path.
|
| 34 |
+
Warning: The cache directory for DeepSpeed Triton autotune, /home/hansirui_1st/.triton/autotune, appears to be on an NFS system. While this is generally acceptable, if you experience slowdowns or hanging when DeepSpeed exits, it is recommended to set the TRITON_CACHE_DIR environment variable to a non-NFS path.
|
| 35 |
+
Warning: The cache directory for DeepSpeed Triton autotune, /home/hansirui_1st/.triton/autotune, appears to be on an NFS system. While this is generally acceptable, if you experience slowdowns or hanging when DeepSpeed exits, it is recommended to set the TRITON_CACHE_DIR environment variable to a non-NFS path.
|
| 36 |
+
[2025-05-27 21:12:35,553] [INFO] [comm.py:669:init_distributed] cdb=None
|
| 37 |
+
[2025-05-27 21:12:35,553] [INFO] [comm.py:669:init_distributed] cdb=None
|
| 38 |
+
[2025-05-27 21:12:35,553] [INFO] [comm.py:700:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
|
| 39 |
+
[2025-05-27 21:12:35,723] [INFO] [comm.py:669:init_distributed] cdb=None
|
| 40 |
+
[2025-05-27 21:12:35,761] [INFO] [comm.py:669:init_distributed] cdb=None
|
| 41 |
+
[2025-05-27 21:12:35,762] [INFO] [comm.py:669:init_distributed] cdb=None
|
| 42 |
+
[2025-05-27 21:12:35,762] [INFO] [comm.py:669:init_distributed] cdb=None
|
| 43 |
+
[2025-05-27 21:12:35,885] [INFO] [comm.py:669:init_distributed] cdb=None
|
| 44 |
+
[2025-05-27 21:12:36,059] [INFO] [comm.py:669:init_distributed] cdb=None
|
| 45 |
+
Set logger level to INFO.
|
| 46 |
+
[2025-05-27 21:12:47,704] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 47 |
+
[2025-05-27 21:12:47,704] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 48 |
+
[2025-05-27 21:12:47,705] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 49 |
+
[2025-05-27 21:12:47,706] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 50 |
+
[2025-05-27 21:12:47,707] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 51 |
+
[2025-05-27 21:12:47,707] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 52 |
+
[2025-05-27 21:12:47,707] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 53 |
+
[2025-05-27 21:12:47,743] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 54 |
+
[2025-05-27 21:12:48,003] [INFO] [partition_parameters.py:348:__exit__] finished initializing model - num_params = 201, num_elems = 1.10B
|
| 55 |
+
ninja: no work to do.
|
| 56 |
+
Time to load fused_adam op: 0.09676456451416016 seconds
|
| 57 |
+
[2025-05-27 21:12:52,008] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 58 |
+
Time to load fused_adam op: 0.10324668884277344 seconds
|
| 59 |
+
Time to load fused_adam op: 0.10322880744934082 seconds
|
| 60 |
+
Time to load fused_adam op: 0.10318541526794434 seconds
|
| 61 |
+
Time to load fused_adam op: 0.10344648361206055 seconds
|
| 62 |
+
Time to load fused_adam op: 0.1032407283782959 seconds
|
| 63 |
+
Time to load fused_adam op: 0.10383749008178711 seconds
|
| 64 |
+
[2025-05-27 21:12:52,014] [INFO] [logging.py:107:log_dist] [Rank 0] DeepSpeed info: version=0.16.8, git-hash=unknown, git-branch=unknown
|
| 65 |
+
[2025-05-27 21:12:52,014] [INFO] [comm.py:694:init_distributed] Distributed backend already initialized
|
| 66 |
+
[2025-05-27 21:12:52,014] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 67 |
+
[2025-05-27 21:12:52,014] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 68 |
+
Time to load fused_adam op: 0.10416865348815918 seconds
|
| 69 |
+
[2025-05-27 21:12:52,015] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 70 |
+
[2025-05-27 21:12:52,016] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 71 |
+
[2025-05-27 21:12:52,016] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 72 |
+
[2025-05-27 21:12:52,017] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 73 |
+
[2025-05-27 21:12:52,017] [INFO] [config.py:735:__init__] Config mesh_device None world_size = 8
|
| 74 |
+
[2025-05-27 21:12:52,019] [INFO] [logging.py:107:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False
|
| 75 |
+
[2025-05-27 21:12:52,019] [INFO] [logging.py:107:log_dist] [Rank 0] Using client Optimizer as basic optimizer
|
| 76 |
+
[2025-05-27 21:12:52,019] [INFO] [logging.py:107:log_dist] [Rank 0] Removing param_group that has no 'params' in the basic Optimizer
|
| 77 |
+
[2025-05-27 21:12:52,021] [INFO] [logging.py:107:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam
|
| 78 |
+
[2025-05-27 21:12:52,021] [INFO] [utils.py:59:is_zero_supported_optimizer] Checking ZeRO support for optimizer=FusedAdam type=<class 'deepspeed.ops.adam.fused_adam.FusedAdam'>
|
| 79 |
+
[2025-05-27 21:12:52,022] [INFO] [logging.py:107:log_dist] [Rank 0] Creating fp16 ZeRO stage 3 optimizer, MiCS is enabled False, Hierarchical params gather False
|
| 80 |
+
[2025-05-27 21:12:52,022] [INFO] [logging.py:107:log_dist] [Rank 0] Creating torch.bfloat16 ZeRO stage 3 optimizer
|
| 81 |
+
[2025-05-27 21:12:52,288] [INFO] [utils.py:781:see_memory_usage] Stage 3 initialize beginning
|
| 82 |
+
[2025-05-27 21:12:52,289] [INFO] [utils.py:782:see_memory_usage] MA 0.26 GB Max_MA 0.61 GB CA 0.67 GB Max_CA 1 GB
|
| 83 |
+
[2025-05-27 21:12:52,289] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory: used = 192.5 GB, percent = 9.6%
|
| 84 |
+
[2025-05-27 21:12:52,290] [INFO] [stage3.py:170:__init__] Reduce bucket size 500000000
|
| 85 |
+
[2025-05-27 21:12:52,290] [INFO] [stage3.py:171:__init__] Prefetch bucket size 30000000
|
| 86 |
+
[2025-05-27 21:12:52,481] [INFO] [utils.py:781:see_memory_usage] DeepSpeedZeRoOffload initialize [begin]
|
| 87 |
+
[2025-05-27 21:12:52,481] [INFO] [utils.py:782:see_memory_usage] MA 0.26 GB Max_MA 0.26 GB CA 0.67 GB Max_CA 1 GB
|
| 88 |
+
[2025-05-27 21:12:52,481] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory: used = 192.52 GB, percent = 9.6%
|
| 89 |
+
Parameter Offload: Total persistent parameters: 92160 in 45 params
|
| 90 |
+
[2025-05-27 21:12:52,665] [INFO] [utils.py:781:see_memory_usage] DeepSpeedZeRoOffload initialize [end]
|
| 91 |
+
[2025-05-27 21:12:52,666] [INFO] [utils.py:782:see_memory_usage] MA 0.26 GB Max_MA 0.26 GB CA 0.67 GB Max_CA 1 GB
|
| 92 |
+
[2025-05-27 21:12:52,666] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory: used = 192.52 GB, percent = 9.6%
|
| 93 |
+
[2025-05-27 21:12:52,778] [INFO] [utils.py:781:see_memory_usage] Before creating fp16 partitions
|
| 94 |
+
[2025-05-27 21:12:52,778] [INFO] [utils.py:782:see_memory_usage] MA 0.26 GB Max_MA 0.26 GB CA 0.67 GB Max_CA 1 GB
|
| 95 |
+
[2025-05-27 21:12:52,779] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory: used = 192.53 GB, percent = 9.6%
|
| 96 |
+
[2025-05-27 21:12:53,421] [INFO] [utils.py:781:see_memory_usage] After creating fp16 partitions: 2
|
| 97 |
+
[2025-05-27 21:12:53,421] [INFO] [utils.py:782:see_memory_usage] MA 0.26 GB Max_MA 0.26 GB CA 0.26 GB Max_CA 1 GB
|
| 98 |
+
[2025-05-27 21:12:53,421] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory: used = 192.56 GB, percent = 9.6%
|
| 99 |
+
[2025-05-27 21:12:53,599] [INFO] [utils.py:781:see_memory_usage] Before creating fp32 partitions
|
| 100 |
+
[2025-05-27 21:12:53,599] [INFO] [utils.py:782:see_memory_usage] MA 0.26 GB Max_MA 0.26 GB CA 0.26 GB Max_CA 0 GB
|
| 101 |
+
[2025-05-27 21:12:53,600] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory: used = 192.56 GB, percent = 9.6%
|
| 102 |
+
[2025-05-27 21:12:53,713] [INFO] [utils.py:781:see_memory_usage] After creating fp32 partitions
|
| 103 |
+
[2025-05-27 21:12:53,713] [INFO] [utils.py:782:see_memory_usage] MA 0.77 GB Max_MA 1.02 GB CA 1.03 GB Max_CA 1 GB
|
| 104 |
+
[2025-05-27 21:12:53,713] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory: used = 192.56 GB, percent = 9.6%
|
| 105 |
+
[2025-05-27 21:12:53,825] [INFO] [utils.py:781:see_memory_usage] Before initializing optimizer states
|
| 106 |
+
[2025-05-27 21:12:53,825] [INFO] [utils.py:782:see_memory_usage] MA 0.77 GB Max_MA 0.77 GB CA 1.03 GB Max_CA 1 GB
|
| 107 |
+
[2025-05-27 21:12:53,825] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory: used = 192.56 GB, percent = 9.6%
|
| 108 |
+
[2025-05-27 21:12:53,937] [INFO] [utils.py:781:see_memory_usage] After initializing optimizer states
|
| 109 |
+
[2025-05-27 21:12:53,937] [INFO] [utils.py:782:see_memory_usage] MA 0.77 GB Max_MA 1.28 GB CA 1.54 GB Max_CA 2 GB
|
| 110 |
+
[2025-05-27 21:12:53,937] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory: used = 192.56 GB, percent = 9.6%
|
| 111 |
+
[2025-05-27 21:12:53,939] [INFO] [stage3.py:534:_setup_for_real_optimizer] optimizer state initialized
|
| 112 |
+
[2025-05-27 21:12:54,174] [INFO] [utils.py:781:see_memory_usage] After initializing ZeRO optimizer
|
| 113 |
+
[2025-05-27 21:12:54,175] [INFO] [utils.py:782:see_memory_usage] MA 1.96 GB Max_MA 2.2 GB CA 2.48 GB Max_CA 2 GB
|
| 114 |
+
[2025-05-27 21:12:54,175] [INFO] [utils.py:789:see_memory_usage] CPU Virtual Memory: used = 192.56 GB, percent = 9.6%
|
| 115 |
+
[2025-05-27 21:12:54,175] [INFO] [logging.py:107:log_dist] [Rank 0] DeepSpeed Final Optimizer = DeepSpeedZeroOptimizer_Stage3
|
| 116 |
+
[2025-05-27 21:12:54,175] [INFO] [logging.py:107:log_dist] [Rank 0] DeepSpeed using client LR scheduler
|
| 117 |
+
[2025-05-27 21:12:54,175] [INFO] [logging.py:107:log_dist] [Rank 0] DeepSpeed LR Scheduler = <torch.optim.lr_scheduler.LambdaLR object at 0x15512c12f290>
|
| 118 |
+
[2025-05-27 21:12:54,175] [INFO] [logging.py:107:log_dist] [Rank 0] step=0, skipped=0, lr=[1e-05, 1e-05], mom=[(0.9, 0.95), (0.9, 0.95)]
|
| 119 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1003:print] DeepSpeedEngine configuration:
|
| 120 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] activation_checkpointing_config {
|
| 121 |
+
"partition_activations": false,
|
| 122 |
+
"contiguous_memory_optimization": false,
|
| 123 |
+
"cpu_checkpointing": false,
|
| 124 |
+
"number_checkpoints": null,
|
| 125 |
+
"synchronize_checkpoint_boundary": false,
|
| 126 |
+
"profile": false
|
| 127 |
+
}
|
| 128 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'intra_op_parallelism': 1, 'single_submit': False, 'overlap_events': True, 'use_gds': False}
|
| 129 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] amp_enabled .................. False
|
| 130 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] amp_params ................... False
|
| 131 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] autotuning_config ............ {
|
| 132 |
+
"enabled": false,
|
| 133 |
+
"start_step": null,
|
| 134 |
+
"end_step": null,
|
| 135 |
+
"metric_path": null,
|
| 136 |
+
"arg_mappings": null,
|
| 137 |
+
"metric": "throughput",
|
| 138 |
+
"model_info": null,
|
| 139 |
+
"results_dir": "autotuning_results",
|
| 140 |
+
"exps_dir": "autotuning_exps",
|
| 141 |
+
"overwrite": true,
|
| 142 |
+
"fast": true,
|
| 143 |
+
"start_profile_step": 3,
|
| 144 |
+
"end_profile_step": 5,
|
| 145 |
+
"tuner_type": "gridsearch",
|
| 146 |
+
"tuner_early_stopping": 5,
|
| 147 |
+
"tuner_num_trials": 50,
|
| 148 |
+
"model_info_path": null,
|
| 149 |
+
"mp_size": 1,
|
| 150 |
+
"max_train_batch_size": null,
|
| 151 |
+
"min_train_batch_size": 1,
|
| 152 |
+
"max_train_micro_batch_size_per_gpu": 1.024000e+03,
|
| 153 |
+
"min_train_micro_batch_size_per_gpu": 1,
|
| 154 |
+
"num_tuning_micro_batch_sizes": 3
|
| 155 |
+
}
|
| 156 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] bfloat16_enabled ............. True
|
| 157 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] bfloat16_immediate_grad_update True
|
| 158 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] checkpoint_parallel_write_pipeline False
|
| 159 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] checkpoint_tag_validation_enabled True
|
| 160 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] checkpoint_tag_validation_fail False
|
| 161 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] comms_config ................. <deepspeed.comm.config.DeepSpeedCommsConfig object at 0x1551f1027210>
|
| 162 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] communication_data_type ...... None
|
| 163 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] compile_config ............... deepcompile=False free_activation=False offload_activation=False offload_opt_states=False double_buffer=True symmetric_memory=False debug_log=False offload_parameters=False sync_before_reduce=False sync_after_reduce=False sync_before_allgather=False sync_after_allgather=False
|
| 164 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}}
|
| 165 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] curriculum_enabled_legacy .... False
|
| 166 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] curriculum_params_legacy ..... False
|
| 167 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] data_efficiency_config ....... {'enabled': False, 'seed': 1234, 'data_sampling': {'enabled': False, 'num_epochs': 1000, 'num_workers': 0, 'pin_memory': False, 'curriculum_learning': {'enabled': False}, 'dynamic_batching': {'enabled': False, 'lr_scaling_method': 'linear', 'min_batch_size': 1, 'max_batch_size': None, 'sequence_picking_order': 'dataloader', 'verbose': False}}, 'data_routing': {'enabled': False, 'random_ltd': {'enabled': False, 'layer_token_lr_schedule': {'enabled': False}}}}
|
| 168 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] data_efficiency_enabled ...... False
|
| 169 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] dataloader_drop_last ......... False
|
| 170 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] disable_allgather ............ False
|
| 171 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] dump_state ................... False
|
| 172 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] dynamic_loss_scale_args ...... None
|
| 173 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] eigenvalue_enabled ........... False
|
| 174 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] eigenvalue_gas_boundary_resolution 1
|
| 175 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] eigenvalue_layer_name ........ bert.encoder.layer
|
| 176 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] eigenvalue_layer_num ......... 0
|
| 177 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] eigenvalue_max_iter .......... 100
|
| 178 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] eigenvalue_stability ......... 1e-06
|
| 179 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] eigenvalue_tol ............... 0.01
|
| 180 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] eigenvalue_verbose ........... False
|
| 181 |
+
[2025-05-27 21:12:54,176] [INFO] [config.py:1007:print] elasticity_enabled ........... False
|
| 182 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] flops_profiler_config ........ {
|
| 183 |
+
"enabled": false,
|
| 184 |
+
"recompute_fwd_factor": 0.0,
|
| 185 |
+
"profile_step": 1,
|
| 186 |
+
"module_depth": -1,
|
| 187 |
+
"top_modules": 1,
|
| 188 |
+
"detailed": true,
|
| 189 |
+
"output_file": null
|
| 190 |
+
}
|
| 191 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] fp16_auto_cast ............... None
|
| 192 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] fp16_enabled ................. False
|
| 193 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] fp16_master_weights_and_gradients False
|
| 194 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] global_rank .................. 0
|
| 195 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] grad_accum_dtype ............. None
|
| 196 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] gradient_accumulation_steps .. 8
|
| 197 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] gradient_clipping ............ 1.0
|
| 198 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] gradient_predivide_factor .... 1.0
|
| 199 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] graph_harvesting ............. False
|
| 200 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] hybrid_engine ................ enabled=False max_out_tokens=512 inference_tp_size=1 release_inference_cache=False pin_parameters=True tp_gather_partition_size=8
|
| 201 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] initial_dynamic_scale ........ 1
|
| 202 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] load_universal_checkpoint .... False
|
| 203 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] loss_scale ................... 1.0
|
| 204 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] memory_breakdown ............. False
|
| 205 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] mics_hierarchial_params_gather False
|
| 206 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] mics_shard_size .............. -1
|
| 207 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] monitor_config ............... tensorboard=TensorBoardConfig(enabled=False, output_path='', job_name='DeepSpeedJobName') comet=CometConfig(enabled=False, samples_log_interval=100, project=None, workspace=None, api_key=None, experiment_name=None, experiment_key=None, online=None, mode=None) wandb=WandbConfig(enabled=False, group=None, team=None, project='deepspeed') csv_monitor=CSVConfig(enabled=False, output_path='', job_name='DeepSpeedJobName')
|
| 208 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] nebula_config ................ {
|
| 209 |
+
"enabled": false,
|
| 210 |
+
"persistent_storage_path": null,
|
| 211 |
+
"persistent_time_interval": 100,
|
| 212 |
+
"num_of_version_in_retention": 2,
|
| 213 |
+
"enable_nebula_load": true,
|
| 214 |
+
"load_path": null
|
| 215 |
+
}
|
| 216 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] optimizer_legacy_fusion ...... False
|
| 217 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] optimizer_name ............... None
|
| 218 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] optimizer_params ............. None
|
| 219 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0, 'pipe_partitioned': True, 'grad_partitioned': True}
|
| 220 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] pld_enabled .................. False
|
| 221 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] pld_params ................... False
|
| 222 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] prescale_gradients ........... False
|
| 223 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] scheduler_name ............... None
|
| 224 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] scheduler_params ............. None
|
| 225 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] seq_parallel_communication_data_type torch.float32
|
| 226 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] sparse_attention ............. None
|
| 227 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] sparse_gradients_enabled ..... False
|
| 228 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] steps_per_print .............. 10
|
| 229 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] tensor_parallel_config ....... dtype=torch.float16 autotp_size=0 tp_overlap_comm=False tensor_parallel=TPConfig(tp_size=1, tp_grain_size=1, mpu=None, tp_group=None) injection_policy_tuple=None keep_module_on_host=False replace_with_kernel_inject=False
|
| 230 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] timers_config ................ enabled=True synchronized=True
|
| 231 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] train_batch_size ............. 64
|
| 232 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] train_micro_batch_size_per_gpu 1
|
| 233 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] use_data_before_expert_parallel_ False
|
| 234 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] use_node_local_storage ....... False
|
| 235 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] wall_clock_breakdown ......... False
|
| 236 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] weight_quantization_config ... None
|
| 237 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] world_size ................... 8
|
| 238 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] zero_allow_untested_optimizer False
|
| 239 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] zero_config .................. stage=3 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 use_multi_rank_bucket_allreduce=True allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=True load_from_fp32_weights=True elastic_checkpoint=False offload_param=DeepSpeedZeroOffloadParamConfig(device='none', nvme_path=None, buffer_count=5, buffer_size=100000000, max_in_cpu=1000000000, pin_memory=False) offload_optimizer=DeepSpeedZeroOffloadOptimizerConfig(device='none', nvme_path=None, buffer_count=4, pin_memory=False, pipeline_read=False, pipeline_write=False, fast_init=False, ratio=1.0) sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=30000000 param_persistence_threshold=10000 model_persistence_threshold=9223372036854775807 max_live_parameters=30000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=True module_granularity_threshold=0 use_all_reduce_for_fetch_params=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False zero_hpz_partition_size=1 zero_quantized_weights=False zero_quantized_nontrainable_weights=False zero_quantized_gradients=False zeropp_loco_param=None mics_shard_size=-1 mics_hierarchical_params_gather=False memory_efficient_linear=False pipeline_loading_checkpoint=False override_module_apply=True log_trace_cache_warnings=False
|
| 240 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] zero_enabled ................. True
|
| 241 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] zero_force_ds_cpu_optimizer .. True
|
| 242 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:1007:print] zero_optimization_stage ...... 3
|
| 243 |
+
[2025-05-27 21:12:54,177] [INFO] [config.py:993:print_user_config] json = {
|
| 244 |
+
"train_batch_size": 64,
|
| 245 |
+
"train_micro_batch_size_per_gpu": 1,
|
| 246 |
+
"gradient_accumulation_steps": 8,
|
| 247 |
+
"steps_per_print": 10,
|
| 248 |
+
"zero_optimization": {
|
| 249 |
+
"stage": 3,
|
| 250 |
+
"offload_param": {
|
| 251 |
+
"device": "none"
|
| 252 |
+
},
|
| 253 |
+
"offload_optimizer": {
|
| 254 |
+
"device": "none"
|
| 255 |
+
},
|
| 256 |
+
"param_persistence_threshold": 1.000000e+04,
|
| 257 |
+
"max_live_parameters": 3.000000e+07,
|
| 258 |
+
"prefetch_bucket_size": 3.000000e+07,
|
| 259 |
+
"memory_efficient_linear": false,
|
| 260 |
+
"gather_16bit_weights_on_model_save": true
|
| 261 |
+
},
|
| 262 |
+
"gradient_clipping": 1.0,
|
| 263 |
+
"prescale_gradients": false,
|
| 264 |
+
"wall_clock_breakdown": false,
|
| 265 |
+
"hybrid_engine": {
|
| 266 |
+
"enabled": false,
|
| 267 |
+
"max_out_tokens": 512,
|
| 268 |
+
"inference_tp_size": 1,
|
| 269 |
+
"release_inference_cache": false,
|
| 270 |
+
"pin_parameters": true,
|
| 271 |
+
"tp_gather_partition_size": 8
|
| 272 |
+
},
|
| 273 |
+
"bf16": {
|
| 274 |
+
"enabled": true
|
| 275 |
+
}
|
| 276 |
+
}
|
| 277 |
+
***** Running training *****
|
| 278 |
+
Saving model to "/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500" ...
|
| 279 |
+
Saving 16-bit model...
|
| 280 |
+
[2025-05-27 21:14:37,428] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step7 is ready now!
|
| 281 |
+
[2025-05-27 21:14:37,428] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step7 is ready now!
|
| 282 |
+
[2025-05-27 21:14:37,431] [INFO] [logging.py:107:log_dist] [Rank 0] [Torch] Checkpoint global_step7 is about to be saved!
|
| 283 |
+
[2025-05-27 21:14:37,431] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step7 is ready now!
|
| 284 |
+
[2025-05-27 21:14:37,431] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step7 is ready now!
|
| 285 |
+
[2025-05-27 21:14:37,431] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step7 is ready now!
|
| 286 |
+
[2025-05-27 21:14:37,431] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step7 is ready now!
|
| 287 |
+
[2025-05-27 21:14:37,431] [INFO] [engine.py:3860:save_16bit_model] Saving model weights to /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/pytorch_model.bin, tag: global_step7
|
| 288 |
+
[2025-05-27 21:14:37,431] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/pytorch_model.bin...
|
| 289 |
+
[2025-05-27 21:14:37,431] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step7 is ready now!
|
| 290 |
+
[2025-05-27 21:14:39,649] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/pytorch_model.bin.
|
| 291 |
+
[2025-05-27 21:14:39,650] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step7 is ready now!
|
| 292 |
+
Model saved!
|
| 293 |
+
[2025-05-27 21:14:40,193] [INFO] [launch.py:351:main] Process 2755443 exits successfully.
|
| 294 |
+
[2025-05-27 21:14:40,193] [INFO] [launch.py:351:main] Process 2755440 exits successfully.
|
| 295 |
+
[2025-05-27 21:14:40,193] [INFO] [launch.py:351:main] Process 2755444 exits successfully.
|
| 296 |
+
[2025-05-27 21:14:40,193] [INFO] [launch.py:351:main] Process 2755446 exits successfully.
|
| 297 |
+
[2025-05-27 21:14:41,193] [INFO] [launch.py:351:main] Process 2755445 exits successfully.
|
| 298 |
+
[2025-05-27 21:14:41,194] [INFO] [launch.py:351:main] Process 2755442 exits successfully.
|
| 299 |
+
[2025-05-27 21:14:41,194] [INFO] [launch.py:351:main] Process 2755441 exits successfully.
|
| 300 |
+
[1;34mwandb[0m:
|
| 301 |
+
[1;34mwandb[0m: 🚀 View run [33mimdb-tinyllama-3T-s3-Q1-5000-Q2-500[0m at: [34mhttps://wandb.ai/xtom/Inverse_Alignment_IMDb/runs/n7vbaf0c[0m
|
| 302 |
+
[1;34mwandb[0m: Find logs at: [1;35m../../../../../../aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/wandb/run-20250527_211254-n7vbaf0c/logs[0m
|
| 303 |
+
[2025-05-27 21:14:43,194] [INFO] [launch.py:351:main] Process 2755439 exits successfully.
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer.model
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
|
| 3 |
+
size 499723
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": true,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"add_prefix_space": null,
|
| 5 |
+
"added_tokens_decoder": {
|
| 6 |
+
"0": {
|
| 7 |
+
"content": "<unk>",
|
| 8 |
+
"lstrip": false,
|
| 9 |
+
"normalized": false,
|
| 10 |
+
"rstrip": false,
|
| 11 |
+
"single_word": false,
|
| 12 |
+
"special": true
|
| 13 |
+
},
|
| 14 |
+
"1": {
|
| 15 |
+
"content": "<s>",
|
| 16 |
+
"lstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"rstrip": false,
|
| 19 |
+
"single_word": false,
|
| 20 |
+
"special": true
|
| 21 |
+
},
|
| 22 |
+
"2": {
|
| 23 |
+
"content": "</s>",
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"normalized": false,
|
| 26 |
+
"rstrip": false,
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"special": true
|
| 29 |
+
},
|
| 30 |
+
"32000": {
|
| 31 |
+
"content": "<pad>",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": false,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false,
|
| 36 |
+
"special": true
|
| 37 |
+
}
|
| 38 |
+
},
|
| 39 |
+
"bos_token": "<s>",
|
| 40 |
+
"clean_up_tokenization_spaces": false,
|
| 41 |
+
"eos_token": "</s>",
|
| 42 |
+
"extra_special_tokens": {},
|
| 43 |
+
"legacy": false,
|
| 44 |
+
"max_length": 512,
|
| 45 |
+
"model_max_length": 512,
|
| 46 |
+
"pad_token": "<pad>",
|
| 47 |
+
"padding_side": "right",
|
| 48 |
+
"sp_model_kwargs": {},
|
| 49 |
+
"stride": 0,
|
| 50 |
+
"tokenizer_class": "LlamaTokenizer",
|
| 51 |
+
"truncation_side": "right",
|
| 52 |
+
"truncation_strategy": "longest_first",
|
| 53 |
+
"unk_token": "<unk>",
|
| 54 |
+
"use_default_system_prompt": false
|
| 55 |
+
}
|
wandb/debug-internal.log
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2025-05-27T21:12:54.813037185+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/wandb/run-20250527_211254-n7vbaf0c/logs/debug-core.log"}
|
| 2 |
+
{"time":"2025-05-27T21:12:55.218185006+08:00","level":"INFO","msg":"created new stream","id":"n7vbaf0c"}
|
| 3 |
+
{"time":"2025-05-27T21:12:55.218245882+08:00","level":"INFO","msg":"stream: started","id":"n7vbaf0c"}
|
| 4 |
+
{"time":"2025-05-27T21:12:55.218266256+08:00","level":"INFO","msg":"handler: started","stream_id":"n7vbaf0c"}
|
| 5 |
+
{"time":"2025-05-27T21:12:55.218268456+08:00","level":"INFO","msg":"sender: started","stream_id":"n7vbaf0c"}
|
| 6 |
+
{"time":"2025-05-27T21:12:55.21829514+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"n7vbaf0c"}
|
| 7 |
+
{"time":"2025-05-27T21:12:55.632716292+08:00","level":"INFO","msg":"Starting system monitor"}
|
| 8 |
+
{"time":"2025-05-27T21:14:39.658094795+08:00","level":"INFO","msg":"Stopping system monitor"}
|
| 9 |
+
{"time":"2025-05-27T21:14:39.658189877+08:00","level":"INFO","msg":"Stopped system monitor"}
|
| 10 |
+
{"time":"2025-05-27T21:14:40.905025188+08:00","level":"INFO","msg":"stream: closing","id":"n7vbaf0c"}
|
| 11 |
+
{"time":"2025-05-27T21:14:40.905084792+08:00","level":"WARN","msg":"sender: received Exit record more than once, ignoring"}
|
| 12 |
+
{"time":"2025-05-27T21:14:41.038913661+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 13 |
+
{"time":"2025-05-27T21:14:41.364806087+08:00","level":"INFO","msg":"handler: closed","stream_id":"n7vbaf0c"}
|
| 14 |
+
{"time":"2025-05-27T21:14:41.364896902+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"n7vbaf0c"}
|
| 15 |
+
{"time":"2025-05-27T21:14:41.364917542+08:00","level":"INFO","msg":"sender: closed","stream_id":"n7vbaf0c"}
|
| 16 |
+
{"time":"2025-05-27T21:14:41.365210048+08:00","level":"INFO","msg":"stream: closed","id":"n7vbaf0c"}
|
wandb/debug.log
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2025-05-27 21:12:54,805 INFO MainThread:2755439 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
|
| 2 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_setup.py:_flush():70] Configure stats pid to 2755439
|
| 3 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_setup.py:_flush():70] Loading settings from /home/hansirui_1st/.config/wandb/settings
|
| 4 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_setup.py:_flush():70] Loading settings from /home/hansirui_1st/jiayi/resist/setting3/scripts/wandb/settings
|
| 5 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_setup.py:_flush():70] Loading settings from environment variables
|
| 6 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_init.py:setup_run_log_directory():724] Logging user logs to /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/wandb/run-20250527_211254-n7vbaf0c/logs/debug.log
|
| 7 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/wandb/run-20250527_211254-n7vbaf0c/logs/debug-internal.log
|
| 8 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_init.py:init():852] calling init triggers
|
| 9 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
|
| 10 |
+
config: {'model_name_or_path': '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000', 'max_length': 512, 'trust_remote_code': True, 'train_datasets': [('inverse-json', {'proportion': 1.0, 'path': '/home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json'})], 'eval_datasets': None, 'epochs': 1, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 4, 'gradient_accumulation_steps': 8, 'gradient_checkpointing': True, 'lr': 1e-05, 'lr_scheduler_type': <SchedulerType.CONSTANT: 'constant'>, 'lr_warmup_ratio': 0.0, 'weight_decay': 0.0, 'seed': 42, 'fp16': False, 'bf16': True, 'tf32': True, 'eval_strategy': 'epoch', 'eval_interval': 1000000, 'need_eval': False, 'eval_split_ratio': None, 'output_dir': '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500', 'log_type': 'wandb', 'log_dir': '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500', 'log_project': 'Inverse_Alignment_IMDb', 'log_run_name': 'imdb-tinyllama-3T-s3-Q1-5000-Q2-500', 'save_16bit': True, 'save_interval': 1000000, 'local_rank': 0, 'zero_stage': 3, 'offload': 'none', 'deepspeed': False, 'deepspeed_config': None, 'deepscale': False, 'deepscale_config': None, 'global_rank': 0, 'device': device(type='cuda', index=0), 'num_update_steps_per_epoch': 8, 'total_training_steps': 8, '_wandb': {}}
|
| 11 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_init.py:init():893] starting backend
|
| 12 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_init.py:init():897] sending inform_init request
|
| 13 |
+
2025-05-27 21:12:54,810 INFO MainThread:2755439 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
| 14 |
+
2025-05-27 21:12:54,810 INFO MainThread:2755439 [wandb_init.py:init():907] backend started and connected
|
| 15 |
+
2025-05-27 21:12:54,811 INFO MainThread:2755439 [wandb_init.py:init():1005] updated telemetry
|
| 16 |
+
2025-05-27 21:12:54,812 INFO MainThread:2755439 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
|
| 17 |
+
2025-05-27 21:12:55,628 INFO MainThread:2755439 [wandb_init.py:init():1104] starting run threads in backend
|
| 18 |
+
2025-05-27 21:12:55,897 INFO MainThread:2755439 [wandb_run.py:_console_start():2573] atexit reg
|
| 19 |
+
2025-05-27 21:12:55,897 INFO MainThread:2755439 [wandb_run.py:_redirect():2421] redirect: wrap_raw
|
| 20 |
+
2025-05-27 21:12:55,897 INFO MainThread:2755439 [wandb_run.py:_redirect():2490] Wrapping output streams.
|
| 21 |
+
2025-05-27 21:12:55,897 INFO MainThread:2755439 [wandb_run.py:_redirect():2513] Redirects installed.
|
| 22 |
+
2025-05-27 21:12:55,899 INFO MainThread:2755439 [wandb_init.py:init():1150] run started, returning control to user process
|
| 23 |
+
2025-05-27 21:14:39,656 INFO MainThread:2755439 [wandb_run.py:_finish():2321] finishing run xtom/Inverse_Alignment_IMDb/n7vbaf0c
|
| 24 |
+
2025-05-27 21:14:39,657 INFO MainThread:2755439 [wandb_run.py:_atexit_cleanup():2538] got exitcode: 0
|
| 25 |
+
2025-05-27 21:14:39,657 INFO MainThread:2755439 [wandb_run.py:_restore():2520] restore
|
| 26 |
+
2025-05-27 21:14:39,657 INFO MainThread:2755439 [wandb_run.py:_restore():2526] restore done
|
| 27 |
+
2025-05-27 21:14:40,658 INFO MainThread:2755439 [wandb_run.py:_restore():2520] restore
|
| 28 |
+
2025-05-27 21:14:40,658 INFO MainThread:2755439 [wandb_run.py:_restore():2526] restore done
|
| 29 |
+
2025-05-27 21:14:40,658 ERROR MainThread:2755439 [wandb_run.py:_atexit_cleanup():2559] Problem finishing run
|
| 30 |
+
Traceback (most recent call last):
|
| 31 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2550, in _atexit_cleanup
|
| 32 |
+
self._on_finish()
|
| 33 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2806, in _on_finish
|
| 34 |
+
wait_with_progress(
|
| 35 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/mailbox/wait_with_progress.py", line 24, in wait_with_progress
|
| 36 |
+
return wait_all_with_progress(
|
| 37 |
+
^^^^^^^^^^^^^^^^^^^^^^^
|
| 38 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/mailbox/wait_with_progress.py", line 87, in wait_all_with_progress
|
| 39 |
+
return asyncio_compat.run(progress_loop_with_timeout)
|
| 40 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 41 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/lib/asyncio_compat.py", line 27, in run
|
| 42 |
+
future = executor.submit(runner.run, fn)
|
| 43 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 44 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/concurrent/futures/thread.py", line 169, in submit
|
| 45 |
+
raise RuntimeError('cannot schedule new futures after '
|
| 46 |
+
RuntimeError: cannot schedule new futures after interpreter shutdown
|
| 47 |
+
2025-05-27 21:14:40,904 INFO MsgRouterThr:2755439 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.
|
wandb/run-20250527_211254-n7vbaf0c/files/config.yaml
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.19.11
|
| 4 |
+
m: []
|
| 5 |
+
python_version: 3.11.11
|
| 6 |
+
t:
|
| 7 |
+
"1":
|
| 8 |
+
- 1
|
| 9 |
+
- 5
|
| 10 |
+
- 11
|
| 11 |
+
- 41
|
| 12 |
+
- 49
|
| 13 |
+
- 51
|
| 14 |
+
- 53
|
| 15 |
+
- 55
|
| 16 |
+
- 71
|
| 17 |
+
- 105
|
| 18 |
+
"2":
|
| 19 |
+
- 1
|
| 20 |
+
- 5
|
| 21 |
+
- 11
|
| 22 |
+
- 41
|
| 23 |
+
- 49
|
| 24 |
+
- 51
|
| 25 |
+
- 53
|
| 26 |
+
- 55
|
| 27 |
+
- 71
|
| 28 |
+
- 105
|
| 29 |
+
"3":
|
| 30 |
+
- 2
|
| 31 |
+
- 13
|
| 32 |
+
- 16
|
| 33 |
+
- 23
|
| 34 |
+
- 55
|
| 35 |
+
- 61
|
| 36 |
+
"4": 3.11.11
|
| 37 |
+
"5": 0.19.11
|
| 38 |
+
"6": 4.52.1
|
| 39 |
+
"8":
|
| 40 |
+
- 5
|
| 41 |
+
"12": 0.19.11
|
| 42 |
+
"13": linux-x86_64
|
| 43 |
+
bf16:
|
| 44 |
+
value: true
|
| 45 |
+
deepscale:
|
| 46 |
+
value: false
|
| 47 |
+
deepscale_config:
|
| 48 |
+
value: null
|
| 49 |
+
deepspeed:
|
| 50 |
+
value: false
|
| 51 |
+
deepspeed_config:
|
| 52 |
+
value: null
|
| 53 |
+
device:
|
| 54 |
+
value: cuda:0
|
| 55 |
+
epochs:
|
| 56 |
+
value: 1
|
| 57 |
+
eval_datasets:
|
| 58 |
+
value: null
|
| 59 |
+
eval_interval:
|
| 60 |
+
value: 1000000
|
| 61 |
+
eval_split_ratio:
|
| 62 |
+
value: null
|
| 63 |
+
eval_strategy:
|
| 64 |
+
value: epoch
|
| 65 |
+
fp16:
|
| 66 |
+
value: false
|
| 67 |
+
global_rank:
|
| 68 |
+
value: 0
|
| 69 |
+
gradient_accumulation_steps:
|
| 70 |
+
value: 8
|
| 71 |
+
gradient_checkpointing:
|
| 72 |
+
value: true
|
| 73 |
+
local_rank:
|
| 74 |
+
value: 0
|
| 75 |
+
log_dir:
|
| 76 |
+
value: /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500
|
| 77 |
+
log_project:
|
| 78 |
+
value: Inverse_Alignment_IMDb
|
| 79 |
+
log_run_name:
|
| 80 |
+
value: imdb-tinyllama-3T-s3-Q1-5000-Q2-500
|
| 81 |
+
log_type:
|
| 82 |
+
value: wandb
|
| 83 |
+
lr:
|
| 84 |
+
value: 1e-05
|
| 85 |
+
lr_scheduler_type:
|
| 86 |
+
value: CONSTANT
|
| 87 |
+
lr_warmup_ratio:
|
| 88 |
+
value: 0
|
| 89 |
+
max_length:
|
| 90 |
+
value: 512
|
| 91 |
+
model_name_or_path:
|
| 92 |
+
value: /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000
|
| 93 |
+
need_eval:
|
| 94 |
+
value: false
|
| 95 |
+
num_update_steps_per_epoch:
|
| 96 |
+
value: 8
|
| 97 |
+
offload:
|
| 98 |
+
value: none
|
| 99 |
+
output_dir:
|
| 100 |
+
value: /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500
|
| 101 |
+
per_device_eval_batch_size:
|
| 102 |
+
value: 4
|
| 103 |
+
per_device_train_batch_size:
|
| 104 |
+
value: 1
|
| 105 |
+
save_16bit:
|
| 106 |
+
value: true
|
| 107 |
+
save_interval:
|
| 108 |
+
value: 1000000
|
| 109 |
+
seed:
|
| 110 |
+
value: 42
|
| 111 |
+
tf32:
|
| 112 |
+
value: true
|
| 113 |
+
total_training_steps:
|
| 114 |
+
value: 8
|
| 115 |
+
train_datasets:
|
| 116 |
+
value:
|
| 117 |
+
- - inverse-json
|
| 118 |
+
- path: /home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json
|
| 119 |
+
proportion: 1
|
| 120 |
+
trust_remote_code:
|
| 121 |
+
value: true
|
| 122 |
+
weight_decay:
|
| 123 |
+
value: 0
|
| 124 |
+
zero_stage:
|
| 125 |
+
value: 3
|
wandb/run-20250527_211254-n7vbaf0c/files/output.log
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
***** Running training *****
|
| 2 |
+
Training 1/1 epoch: 0%| | 0/63 [00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
|
| 3 |
+
Training 1/1 epoch (loss 2.5375): 100%|██████████| 63/63 [01:38<00:00, 1.57s/it]
|
| 4 |
+
Saving model to "/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500" ...
|
| 5 |
+
tokenizer config file saved in /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/tokenizer_config.json
|
| 6 |
+
Special tokens file saved in /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/special_tokens_map.json
|
| 7 |
+
Saving 16-bit model...
|
| 8 |
+
[2025-05-27 21:14:37,431] [INFO] [logging.py:107:log_dist] [Rank 0] [Torch] Checkpoint global_step7 is about to be saved!
|
| 9 |
+
[2025-05-27 21:14:37,431] [INFO] [engine.py:3860:save_16bit_model] Saving model weights to /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/pytorch_model.bin, tag: global_step7
|
| 10 |
+
[2025-05-27 21:14:37,431] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/pytorch_model.bin...
|
| 11 |
+
[2025-05-27 21:14:39,649] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/pytorch_model.bin.
|
| 12 |
+
[2025-05-27 21:14:39,650] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step7 is ready now!
|
| 13 |
+
Model saved!
|
wandb/run-20250527_211254-n7vbaf0c/files/requirements.txt
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
huggingface-hub==0.31.4
|
| 2 |
+
GitPython==3.1.44
|
| 3 |
+
cycler==0.12.1
|
| 4 |
+
pip==25.1
|
| 5 |
+
tensorboard-data-server==0.7.2
|
| 6 |
+
python-multipart==0.0.20
|
| 7 |
+
Werkzeug==3.1.3
|
| 8 |
+
rsa==4.9.1
|
| 9 |
+
pydantic==2.11.4
|
| 10 |
+
yarl==1.20.0
|
| 11 |
+
rich==14.0.0
|
| 12 |
+
python-dotenv==1.1.0
|
| 13 |
+
pandas==2.2.3
|
| 14 |
+
urllib3==2.4.0
|
| 15 |
+
scipy==1.15.3
|
| 16 |
+
safehttpx==0.1.6
|
| 17 |
+
fsspec==2025.3.0
|
| 18 |
+
dill==0.3.8
|
| 19 |
+
aiohappyeyeballs==2.6.1
|
| 20 |
+
Markdown==3.8
|
| 21 |
+
matplotlib==3.10.3
|
| 22 |
+
pydantic_core==2.33.2
|
| 23 |
+
nvidia-nvtx-cu12==12.4.127
|
| 24 |
+
nvidia-cusolver-cu12==11.6.1.9
|
| 25 |
+
protobuf==6.31.0
|
| 26 |
+
pycountry==24.6.1
|
| 27 |
+
pyzmq==26.4.0
|
| 28 |
+
airportsdata==20250224
|
| 29 |
+
idna==3.10
|
| 30 |
+
mpmath==1.3.0
|
| 31 |
+
markdown-it-py==3.0.0
|
| 32 |
+
seaborn==0.13.2
|
| 33 |
+
Pygments==2.19.1
|
| 34 |
+
align-anything==0.0.1.dev0
|
| 35 |
+
groovy==0.1.2
|
| 36 |
+
ray==2.46.0
|
| 37 |
+
hjson==3.1.0
|
| 38 |
+
proto-plus==1.26.1
|
| 39 |
+
transformers==4.52.1
|
| 40 |
+
distlib==0.3.9
|
| 41 |
+
blake3==1.0.5
|
| 42 |
+
safetensors==0.5.3
|
| 43 |
+
llvmlite==0.44.0
|
| 44 |
+
charset-normalizer==3.4.2
|
| 45 |
+
propcache==0.3.1
|
| 46 |
+
shellingham==1.5.4
|
| 47 |
+
uvloop==0.21.0
|
| 48 |
+
numpy==1.26.4
|
| 49 |
+
kiwisolver==1.4.8
|
| 50 |
+
click==8.1.8
|
| 51 |
+
pyarrow==20.0.0
|
| 52 |
+
optree==0.15.0
|
| 53 |
+
pytz==2025.2
|
| 54 |
+
sympy==1.13.1
|
| 55 |
+
nvidia-cuda-runtime-cu12==12.4.127
|
| 56 |
+
wandb==0.19.11
|
| 57 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 58 |
+
zipp==3.21.0
|
| 59 |
+
nvidia-nvjitlink-cu12==12.4.127
|
| 60 |
+
MarkupSafe==3.0.2
|
| 61 |
+
decorator==5.2.1
|
| 62 |
+
orjson==3.10.18
|
| 63 |
+
nvidia-cuda-cupti-cu12==12.4.127
|
| 64 |
+
nvidia-curand-cu12==10.3.5.147
|
| 65 |
+
outlines==0.1.11
|
| 66 |
+
docker-pycreds==0.4.0
|
| 67 |
+
gitdb==4.0.12
|
| 68 |
+
distro==1.9.0
|
| 69 |
+
joblib==1.5.0
|
| 70 |
+
jsonschema==4.23.0
|
| 71 |
+
fonttools==4.58.0
|
| 72 |
+
jiter==0.10.0
|
| 73 |
+
audioread==3.0.1
|
| 74 |
+
filelock==3.18.0
|
| 75 |
+
certifi==2025.4.26
|
| 76 |
+
librosa==0.11.0
|
| 77 |
+
nvidia-cublas-cu12==12.4.5.8
|
| 78 |
+
pycparser==2.22
|
| 79 |
+
tqdm==4.67.1
|
| 80 |
+
networkx==3.4.2
|
| 81 |
+
semantic-version==2.10.0
|
| 82 |
+
pyparsing==3.2.3
|
| 83 |
+
pillow==11.2.1
|
| 84 |
+
aiofiles==24.1.0
|
| 85 |
+
frozenlist==1.6.0
|
| 86 |
+
einops==0.8.1
|
| 87 |
+
opencensus-context==0.1.3
|
| 88 |
+
websockets==15.0.1
|
| 89 |
+
rpds-py==0.25.1
|
| 90 |
+
cffi==1.17.1
|
| 91 |
+
contourpy==1.3.2
|
| 92 |
+
nvidia-cusparselt-cu12==0.6.2
|
| 93 |
+
python-dateutil==2.9.0.post0
|
| 94 |
+
httpx==0.28.1
|
| 95 |
+
msgpack==1.1.0
|
| 96 |
+
pydub==0.25.1
|
| 97 |
+
prometheus_client==0.22.0
|
| 98 |
+
tomlkit==0.13.2
|
| 99 |
+
nvidia-cusparse-cu12==12.3.1.170
|
| 100 |
+
aiosignal==1.3.2
|
| 101 |
+
sentry-sdk==2.29.1
|
| 102 |
+
fastapi==0.115.12
|
| 103 |
+
wheel==0.45.1
|
| 104 |
+
typing-inspection==0.4.0
|
| 105 |
+
psutil==7.0.0
|
| 106 |
+
typing_extensions==4.13.2
|
| 107 |
+
importlib_metadata==8.7.0
|
| 108 |
+
ffmpy==0.5.0
|
| 109 |
+
gguf==0.10.0
|
| 110 |
+
diskcache==5.6.3
|
| 111 |
+
ninja==1.11.1.4
|
| 112 |
+
ruff==0.11.10
|
| 113 |
+
soundfile==0.13.1
|
| 114 |
+
torch==2.6.0
|
| 115 |
+
py-cpuinfo==9.0.0
|
| 116 |
+
multidict==6.4.4
|
| 117 |
+
smart-open==7.1.0
|
| 118 |
+
httpcore==1.0.9
|
| 119 |
+
gradio_client==1.10.1
|
| 120 |
+
nest-asyncio==1.6.0
|
| 121 |
+
lark==1.2.2
|
| 122 |
+
tzdata==2025.2
|
| 123 |
+
starlette==0.46.2
|
| 124 |
+
scikit-learn==1.6.1
|
| 125 |
+
attrs==25.3.0
|
| 126 |
+
opencv-python-headless==4.11.0.86
|
| 127 |
+
six==1.17.0
|
| 128 |
+
lazy_loader==0.4
|
| 129 |
+
mdurl==0.1.2
|
| 130 |
+
openai==1.81.0
|
| 131 |
+
nvidia-nccl-cu12==2.21.5
|
| 132 |
+
jsonschema-specifications==2025.4.1
|
| 133 |
+
google-auth==2.40.2
|
| 134 |
+
platformdirs==4.3.8
|
| 135 |
+
deepspeed==0.16.8
|
| 136 |
+
xxhash==3.5.0
|
| 137 |
+
aiohttp-cors==0.8.1
|
| 138 |
+
msgspec==0.19.0
|
| 139 |
+
pyasn1==0.6.1
|
| 140 |
+
setproctitle==1.3.6
|
| 141 |
+
py-spy==0.4.0
|
| 142 |
+
anyio==4.9.0
|
| 143 |
+
h11==0.16.0
|
| 144 |
+
sentencepiece==0.2.0
|
| 145 |
+
annotated-types==0.7.0
|
| 146 |
+
interegular==0.3.3
|
| 147 |
+
requests==2.32.3
|
| 148 |
+
opencensus==0.11.4
|
| 149 |
+
colorful==0.5.6
|
| 150 |
+
nvidia-cufile-cu12==1.11.1.6
|
| 151 |
+
aiohttp==3.11.18
|
| 152 |
+
referencing==0.36.2
|
| 153 |
+
sniffio==1.3.1
|
| 154 |
+
Jinja2==3.1.6
|
| 155 |
+
peft==0.15.2
|
| 156 |
+
torchvision==0.21.0
|
| 157 |
+
nvidia-ml-py==12.570.86
|
| 158 |
+
virtualenv==20.31.2
|
| 159 |
+
depyf==0.18.0
|
| 160 |
+
packaging==25.0
|
| 161 |
+
xgrammar==0.1.19
|
| 162 |
+
multiprocess==0.70.16
|
| 163 |
+
typer==0.15.4
|
| 164 |
+
nvitop==1.5.0
|
| 165 |
+
nvidia-cuda-nvrtc-cu12==12.4.127
|
| 166 |
+
google-api-core==2.24.2
|
| 167 |
+
tensorboard==2.19.0
|
| 168 |
+
nvidia-cufft-cu12==11.2.1.3
|
| 169 |
+
datasets==3.6.0
|
| 170 |
+
watchfiles==1.0.5
|
| 171 |
+
partial-json-parser==0.2.1.1.post5
|
| 172 |
+
prometheus-fastapi-instrumentator==7.1.0
|
| 173 |
+
gradio==5.30.0
|
| 174 |
+
cachetools==5.5.2
|
| 175 |
+
triton==3.2.0
|
| 176 |
+
soxr==0.5.0.post1
|
| 177 |
+
outlines_core==0.1.26
|
| 178 |
+
lm-format-enforcer==0.10.11
|
| 179 |
+
compressed-tensors==0.9.1
|
| 180 |
+
setuptools==78.1.1
|
| 181 |
+
xformers==0.0.28.post3
|
| 182 |
+
torchaudio==2.6.0
|
| 183 |
+
tokenizers==0.21.1
|
| 184 |
+
PyYAML==6.0.2
|
| 185 |
+
pyasn1_modules==0.4.2
|
| 186 |
+
regex==2024.11.6
|
| 187 |
+
pooch==1.8.2
|
| 188 |
+
accelerate==1.7.0
|
| 189 |
+
mistral_common==1.5.5
|
| 190 |
+
smmap==5.0.2
|
| 191 |
+
cloudpickle==3.1.1
|
| 192 |
+
numba==0.61.2
|
| 193 |
+
grpcio==1.71.0
|
| 194 |
+
threadpoolctl==3.6.0
|
| 195 |
+
httptools==0.6.4
|
| 196 |
+
googleapis-common-protos==1.70.0
|
| 197 |
+
uvicorn==0.34.2
|
| 198 |
+
vllm==0.7.2
|
| 199 |
+
diffusers==0.33.1
|
| 200 |
+
astor==0.8.1
|
| 201 |
+
tiktoken==0.9.0
|
| 202 |
+
wrapt==1.17.2
|
| 203 |
+
absl-py==2.2.2
|
| 204 |
+
jaraco.functools==4.0.1
|
| 205 |
+
inflect==7.3.1
|
| 206 |
+
jaraco.text==3.12.1
|
| 207 |
+
typeguard==4.3.0
|
| 208 |
+
jaraco.collections==5.1.0
|
| 209 |
+
wheel==0.45.1
|
| 210 |
+
importlib_metadata==8.0.0
|
| 211 |
+
backports.tarfile==1.2.0
|
| 212 |
+
tomli==2.0.1
|
| 213 |
+
autocommand==2.2.2
|
| 214 |
+
platformdirs==4.2.2
|
| 215 |
+
more-itertools==10.3.0
|
| 216 |
+
zipp==3.19.2
|
| 217 |
+
packaging==24.2
|
| 218 |
+
typing_extensions==4.12.2
|
| 219 |
+
jaraco.context==5.3.0
|
wandb/run-20250527_211254-n7vbaf0c/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.11.11",
|
| 4 |
+
"startedAt": "2025-05-27T13:12:54.810418Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--local_rank=0",
|
| 7 |
+
"--train_datasets",
|
| 8 |
+
"inverse-json::/home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json",
|
| 9 |
+
"--model_name_or_path",
|
| 10 |
+
"/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000",
|
| 11 |
+
"--max_length",
|
| 12 |
+
"512",
|
| 13 |
+
"--trust_remote_code",
|
| 14 |
+
"True",
|
| 15 |
+
"--epochs",
|
| 16 |
+
"1",
|
| 17 |
+
"--per_device_train_batch_size",
|
| 18 |
+
"1",
|
| 19 |
+
"--per_device_eval_batch_size",
|
| 20 |
+
"4",
|
| 21 |
+
"--gradient_accumulation_steps",
|
| 22 |
+
"8",
|
| 23 |
+
"--gradient_checkpointing",
|
| 24 |
+
"--learning_rate",
|
| 25 |
+
"1e-5",
|
| 26 |
+
"--lr_warmup_ratio",
|
| 27 |
+
"0",
|
| 28 |
+
"--weight_decay",
|
| 29 |
+
"0.0",
|
| 30 |
+
"--lr_scheduler_type",
|
| 31 |
+
"constant",
|
| 32 |
+
"--weight_decay",
|
| 33 |
+
"0.0",
|
| 34 |
+
"--seed",
|
| 35 |
+
"42",
|
| 36 |
+
"--output_dir",
|
| 37 |
+
"/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500",
|
| 38 |
+
"--log_type",
|
| 39 |
+
"wandb",
|
| 40 |
+
"--log_run_name",
|
| 41 |
+
"imdb-tinyllama-3T-s3-Q1-5000-Q2-500",
|
| 42 |
+
"--log_project",
|
| 43 |
+
"Inverse_Alignment_IMDb",
|
| 44 |
+
"--zero_stage",
|
| 45 |
+
"3",
|
| 46 |
+
"--offload",
|
| 47 |
+
"none",
|
| 48 |
+
"--bf16",
|
| 49 |
+
"True",
|
| 50 |
+
"--tf32",
|
| 51 |
+
"True",
|
| 52 |
+
"--save_16bit"
|
| 53 |
+
],
|
| 54 |
+
"program": "-m safe_rlhf.finetune.__main__",
|
| 55 |
+
"email": "[email protected]",
|
| 56 |
+
"root": "/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500",
|
| 57 |
+
"host": "dgx-003",
|
| 58 |
+
"executable": "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/bin/python3.11",
|
| 59 |
+
"cpu_count": 112,
|
| 60 |
+
"cpu_count_logical": 224,
|
| 61 |
+
"gpu": "NVIDIA H800",
|
| 62 |
+
"gpu_count": 8,
|
| 63 |
+
"disk": {
|
| 64 |
+
"/": {
|
| 65 |
+
"total": "1888556142592",
|
| 66 |
+
"used": "378140127232"
|
| 67 |
+
}
|
| 68 |
+
},
|
| 69 |
+
"memory": {
|
| 70 |
+
"total": "2164195532800"
|
| 71 |
+
},
|
| 72 |
+
"cpu": {
|
| 73 |
+
"count": 112,
|
| 74 |
+
"countLogical": 224
|
| 75 |
+
},
|
| 76 |
+
"gpu_nvidia": [
|
| 77 |
+
{
|
| 78 |
+
"name": "NVIDIA H800",
|
| 79 |
+
"memoryTotal": "85520809984",
|
| 80 |
+
"cudaCores": 16896,
|
| 81 |
+
"architecture": "Hopper"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"name": "NVIDIA H800",
|
| 85 |
+
"memoryTotal": "85520809984",
|
| 86 |
+
"cudaCores": 16896,
|
| 87 |
+
"architecture": "Hopper"
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"name": "NVIDIA H800",
|
| 91 |
+
"memoryTotal": "85520809984",
|
| 92 |
+
"cudaCores": 16896,
|
| 93 |
+
"architecture": "Hopper"
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"name": "NVIDIA H800",
|
| 97 |
+
"memoryTotal": "85520809984",
|
| 98 |
+
"cudaCores": 16896,
|
| 99 |
+
"architecture": "Hopper"
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"name": "NVIDIA H800",
|
| 103 |
+
"memoryTotal": "85520809984",
|
| 104 |
+
"cudaCores": 16896,
|
| 105 |
+
"architecture": "Hopper"
|
| 106 |
+
},
|
| 107 |
+
{
|
| 108 |
+
"name": "NVIDIA H800",
|
| 109 |
+
"memoryTotal": "85520809984",
|
| 110 |
+
"cudaCores": 16896,
|
| 111 |
+
"architecture": "Hopper"
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"name": "NVIDIA H800",
|
| 115 |
+
"memoryTotal": "85520809984",
|
| 116 |
+
"cudaCores": 16896,
|
| 117 |
+
"architecture": "Hopper"
|
| 118 |
+
},
|
| 119 |
+
{
|
| 120 |
+
"name": "NVIDIA H800",
|
| 121 |
+
"memoryTotal": "85520809984",
|
| 122 |
+
"cudaCores": 16896,
|
| 123 |
+
"architecture": "Hopper"
|
| 124 |
+
}
|
| 125 |
+
],
|
| 126 |
+
"slurm": {
|
| 127 |
+
"conf": "/cm/shared/apps/slurm/var/etc/slurm/slurm.conf"
|
| 128 |
+
},
|
| 129 |
+
"cudaVersion": "12.2"
|
| 130 |
+
}
|
wandb/run-20250527_211254-n7vbaf0c/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"train/loss":2.5375373363494873,"_wandb":{"runtime":104},"train/lr":1e-05,"train/epoch":1,"_timestamp":1.7483516748566449e+09,"_runtime":104.847661827,"_step":63,"train/step":63}
|
wandb/run-20250527_211254-n7vbaf0c/logs/debug-core.log
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2025-05-27T21:12:54.21438291+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpw3gg2ine/port-2755439.txt","pid":2755439,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2025-05-27T21:12:54.215642841+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2755439}
|
| 3 |
+
{"time":"2025-05-27T21:12:54.215636598+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":34757,"Zone":""}}
|
| 4 |
+
{"time":"2025-05-27T21:12:54.39314524+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:59220"}
|
| 5 |
+
{"time":"2025-05-27T21:12:54.811499514+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"n7vbaf0c","id":"127.0.0.1:59220"}
|
| 6 |
+
{"time":"2025-05-27T21:12:55.218251963+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"n7vbaf0c","id":"127.0.0.1:59220"}
|
| 7 |
+
{"time":"2025-05-27T21:14:40.904680377+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:59220"}
|
| 8 |
+
{"time":"2025-05-27T21:14:40.90509678+08:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:59220"}
|
| 9 |
+
{"time":"2025-05-27T21:14:40.90511462+08:00","level":"INFO","msg":"server is shutting down"}
|
| 10 |
+
{"time":"2025-05-27T21:14:40.905144529+08:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:59220"}
|
| 11 |
+
{"time":"2025-05-27T21:14:41.364865621+08:00","level":"ERROR","msg":"processOutgoingData: flush error","error":"write tcp 127.0.0.1:34757->127.0.0.1:59220: use of closed network connection","id":"127.0.0.1:59220"}
|
| 12 |
+
{"time":"2025-05-27T21:14:41.365280355+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:59220"}
|
| 13 |
+
{"time":"2025-05-27T21:14:41.365296002+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:59220"}
|
| 14 |
+
{"time":"2025-05-27T21:14:41.365304314+08:00","level":"INFO","msg":"server is closed"}
|
wandb/run-20250527_211254-n7vbaf0c/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2025-05-27T21:12:54.813037185+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/wandb/run-20250527_211254-n7vbaf0c/logs/debug-core.log"}
|
| 2 |
+
{"time":"2025-05-27T21:12:55.218185006+08:00","level":"INFO","msg":"created new stream","id":"n7vbaf0c"}
|
| 3 |
+
{"time":"2025-05-27T21:12:55.218245882+08:00","level":"INFO","msg":"stream: started","id":"n7vbaf0c"}
|
| 4 |
+
{"time":"2025-05-27T21:12:55.218266256+08:00","level":"INFO","msg":"handler: started","stream_id":"n7vbaf0c"}
|
| 5 |
+
{"time":"2025-05-27T21:12:55.218268456+08:00","level":"INFO","msg":"sender: started","stream_id":"n7vbaf0c"}
|
| 6 |
+
{"time":"2025-05-27T21:12:55.21829514+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"n7vbaf0c"}
|
| 7 |
+
{"time":"2025-05-27T21:12:55.632716292+08:00","level":"INFO","msg":"Starting system monitor"}
|
| 8 |
+
{"time":"2025-05-27T21:14:39.658094795+08:00","level":"INFO","msg":"Stopping system monitor"}
|
| 9 |
+
{"time":"2025-05-27T21:14:39.658189877+08:00","level":"INFO","msg":"Stopped system monitor"}
|
| 10 |
+
{"time":"2025-05-27T21:14:40.905025188+08:00","level":"INFO","msg":"stream: closing","id":"n7vbaf0c"}
|
| 11 |
+
{"time":"2025-05-27T21:14:40.905084792+08:00","level":"WARN","msg":"sender: received Exit record more than once, ignoring"}
|
| 12 |
+
{"time":"2025-05-27T21:14:41.038913661+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 13 |
+
{"time":"2025-05-27T21:14:41.364806087+08:00","level":"INFO","msg":"handler: closed","stream_id":"n7vbaf0c"}
|
| 14 |
+
{"time":"2025-05-27T21:14:41.364896902+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"n7vbaf0c"}
|
| 15 |
+
{"time":"2025-05-27T21:14:41.364917542+08:00","level":"INFO","msg":"sender: closed","stream_id":"n7vbaf0c"}
|
| 16 |
+
{"time":"2025-05-27T21:14:41.365210048+08:00","level":"INFO","msg":"stream: closed","id":"n7vbaf0c"}
|
wandb/run-20250527_211254-n7vbaf0c/logs/debug.log
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2025-05-27 21:12:54,805 INFO MainThread:2755439 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
|
| 2 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_setup.py:_flush():70] Configure stats pid to 2755439
|
| 3 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_setup.py:_flush():70] Loading settings from /home/hansirui_1st/.config/wandb/settings
|
| 4 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_setup.py:_flush():70] Loading settings from /home/hansirui_1st/jiayi/resist/setting3/scripts/wandb/settings
|
| 5 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_setup.py:_flush():70] Loading settings from environment variables
|
| 6 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_init.py:setup_run_log_directory():724] Logging user logs to /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/wandb/run-20250527_211254-n7vbaf0c/logs/debug.log
|
| 7 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to /aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500/wandb/run-20250527_211254-n7vbaf0c/logs/debug-internal.log
|
| 8 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_init.py:init():852] calling init triggers
|
| 9 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
|
| 10 |
+
config: {'model_name_or_path': '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000', 'max_length': 512, 'trust_remote_code': True, 'train_datasets': [('inverse-json', {'proportion': 1.0, 'path': '/home/hansirui_1st/jiayi/resist/imdb_data/train/neg/500/train.json'})], 'eval_datasets': None, 'epochs': 1, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 4, 'gradient_accumulation_steps': 8, 'gradient_checkpointing': True, 'lr': 1e-05, 'lr_scheduler_type': <SchedulerType.CONSTANT: 'constant'>, 'lr_warmup_ratio': 0.0, 'weight_decay': 0.0, 'seed': 42, 'fp16': False, 'bf16': True, 'tf32': True, 'eval_strategy': 'epoch', 'eval_interval': 1000000, 'need_eval': False, 'eval_split_ratio': None, 'output_dir': '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500', 'log_type': 'wandb', 'log_dir': '/aifs4su/hansirui_1st/jiayi/setting3-imdb/tinyllama-3T/tinyllama-3T-s3-Q1-5000-Q2-500', 'log_project': 'Inverse_Alignment_IMDb', 'log_run_name': 'imdb-tinyllama-3T-s3-Q1-5000-Q2-500', 'save_16bit': True, 'save_interval': 1000000, 'local_rank': 0, 'zero_stage': 3, 'offload': 'none', 'deepspeed': False, 'deepspeed_config': None, 'deepscale': False, 'deepscale_config': None, 'global_rank': 0, 'device': device(type='cuda', index=0), 'num_update_steps_per_epoch': 8, 'total_training_steps': 8, '_wandb': {}}
|
| 11 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_init.py:init():893] starting backend
|
| 12 |
+
2025-05-27 21:12:54,806 INFO MainThread:2755439 [wandb_init.py:init():897] sending inform_init request
|
| 13 |
+
2025-05-27 21:12:54,810 INFO MainThread:2755439 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
| 14 |
+
2025-05-27 21:12:54,810 INFO MainThread:2755439 [wandb_init.py:init():907] backend started and connected
|
| 15 |
+
2025-05-27 21:12:54,811 INFO MainThread:2755439 [wandb_init.py:init():1005] updated telemetry
|
| 16 |
+
2025-05-27 21:12:54,812 INFO MainThread:2755439 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
|
| 17 |
+
2025-05-27 21:12:55,628 INFO MainThread:2755439 [wandb_init.py:init():1104] starting run threads in backend
|
| 18 |
+
2025-05-27 21:12:55,897 INFO MainThread:2755439 [wandb_run.py:_console_start():2573] atexit reg
|
| 19 |
+
2025-05-27 21:12:55,897 INFO MainThread:2755439 [wandb_run.py:_redirect():2421] redirect: wrap_raw
|
| 20 |
+
2025-05-27 21:12:55,897 INFO MainThread:2755439 [wandb_run.py:_redirect():2490] Wrapping output streams.
|
| 21 |
+
2025-05-27 21:12:55,897 INFO MainThread:2755439 [wandb_run.py:_redirect():2513] Redirects installed.
|
| 22 |
+
2025-05-27 21:12:55,899 INFO MainThread:2755439 [wandb_init.py:init():1150] run started, returning control to user process
|
| 23 |
+
2025-05-27 21:14:39,656 INFO MainThread:2755439 [wandb_run.py:_finish():2321] finishing run xtom/Inverse_Alignment_IMDb/n7vbaf0c
|
| 24 |
+
2025-05-27 21:14:39,657 INFO MainThread:2755439 [wandb_run.py:_atexit_cleanup():2538] got exitcode: 0
|
| 25 |
+
2025-05-27 21:14:39,657 INFO MainThread:2755439 [wandb_run.py:_restore():2520] restore
|
| 26 |
+
2025-05-27 21:14:39,657 INFO MainThread:2755439 [wandb_run.py:_restore():2526] restore done
|
| 27 |
+
2025-05-27 21:14:40,658 INFO MainThread:2755439 [wandb_run.py:_restore():2520] restore
|
| 28 |
+
2025-05-27 21:14:40,658 INFO MainThread:2755439 [wandb_run.py:_restore():2526] restore done
|
| 29 |
+
2025-05-27 21:14:40,658 ERROR MainThread:2755439 [wandb_run.py:_atexit_cleanup():2559] Problem finishing run
|
| 30 |
+
Traceback (most recent call last):
|
| 31 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2550, in _atexit_cleanup
|
| 32 |
+
self._on_finish()
|
| 33 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2806, in _on_finish
|
| 34 |
+
wait_with_progress(
|
| 35 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/mailbox/wait_with_progress.py", line 24, in wait_with_progress
|
| 36 |
+
return wait_all_with_progress(
|
| 37 |
+
^^^^^^^^^^^^^^^^^^^^^^^
|
| 38 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/mailbox/wait_with_progress.py", line 87, in wait_all_with_progress
|
| 39 |
+
return asyncio_compat.run(progress_loop_with_timeout)
|
| 40 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 41 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/site-packages/wandb/sdk/lib/asyncio_compat.py", line 27, in run
|
| 42 |
+
future = executor.submit(runner.run, fn)
|
| 43 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 44 |
+
File "/aifs4su/hansirui_1st/miniconda3/envs/jy-resist/lib/python3.11/concurrent/futures/thread.py", line 169, in submit
|
| 45 |
+
raise RuntimeError('cannot schedule new futures after '
|
| 46 |
+
RuntimeError: cannot schedule new futures after interpreter shutdown
|
| 47 |
+
2025-05-27 21:14:40,904 INFO MsgRouterThr:2755439 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.
|
wandb/run-20250527_211254-n7vbaf0c/run-n7vbaf0c.wandb
ADDED
|
Binary file (73.7 kB). View file
|
|
|