Hugging Face's logo

ChatterjeeLab
/
PepTune

Model card Files
xet
Community
2
PepTune
File size: 4,358 Bytes 
92f7053
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
 
noise:
  type: loglinear
  sigma_min: 1e-4
  sigma_max: 20
  state_dependent: True

mode: ppl_eval  # train / ppl_eval / sample_eval
diffusion: absorbing_state
vocab: old_smiles # old_smiles / new_smiles / selfies / helm
backbone: roformer  # peptideclm / helmgpt / dit / roformer / finetune_roformer
parameterization: subs  # subs
time_conditioning: False
T: 0  # 0 (continuous time) / 1000 
subs_masking: False

seed: 42

mcts: 
  num_children: 50
  num_objectives: 5
  topk: 100
  mask_token: 4
  num_iter: 128
  sampling: 0 # 0 is gumbel sampling / > 0 samples children from top k probs
  invalid_penalty: 0.5
  sample_prob: 1.0
  perm: True
  dual: False
  single: False
  time_dependent: True

lr_scheduler:
  _target_: transformers.get_constant_schedule_with_warmup
  num_warmup_steps: 2500

data:
  train: /home/st512/peptune/scripts/peptide-mdlm-mcts/data/finetune2/30K-train.csv
  valid: /home/st512/peptune/scripts/peptide-mdlm-mcts/data/finetune2/30K-val.csv
  batchinohup ng: wrapping # padding / wrapping

loader:
  global_batch_size: 64
  eval_global_batch_size: ${.global_batch_size}
  # Note: batch_size and eval_batch_size are **per machine**
  batch_size: ${div_up:${.global_batch_size}, ${eval:${trainer.devices} * ${trainer.num_nodes}}}
  eval_batch_size: ${div_up:${.eval_global_batch_size}, ${eval:${trainer.devices} * ${trainer.num_nodes}}}
  num_workers: ${eval:"len(__import__('os').sched_getaffinity(0))"}
  pin_memory: True

sampling:
  predictor: ddpm_cache  # analytic, ddpm, ddpm_cache
  num_sequences: 100
  sampling_eps: 1e-3
  steps: 128
  seq_length: 100
  noise_removal: True
  num_sample_batches: 2  # Total samples: `num_gpus` * `loader.eval_batch_size` * num_sample_batches
  num_sample_log: 2
  stride_length: 1
  num_strides: 1

training:
  antithetic_sampling: True
  sampling_eps: 1e-3
  focus_mask: False
  #dynamic_batching: True
  accumulator: False

eval:
  checkpoint_path: /home/st512/peptune/scripts/peptide-mdlm-mcts/checkpoints/11M-old-tokenizer/epoch=10-step=156276.ckpt
  disable_ema: False
  compute_generative_perplexity: False
  perplexity_batch_size: 8
  compute_perplexity_on_sanity: False
  gen_ppl_eval_model_name_or_path: gpt2-large  # gpt2-large, meta-llama/Llama-2-7b-hf
  generate_samples: True
  generation_model: /home/st512/peptune/scripts/peptide-mdlm-mcts/checkpoints/11M-old-tokenizer/
  
optim:
  weight_decay: 0.075
  lr: 3e-4
  beta1: 0.9
  beta2: 0.999
  eps: 1e-8

pepclm:
  hidden_size: 768
  cond_dim: 256
  n_heads: 20
  n_blocks: 4
  dropout: 0.5
  length: 512
  #scale_by_sigma: True

model:
  type: ddit
  hidden_size: 768
  cond_dim: 128
  length: 512
  n_blocks: 12
  n_heads: 12
  scale_by_sigma: True
  dropout: 0.1

roformer:
  hidden_size: 768
  n_layers: 8
  n_heads: 8
  max_position_embeddings: 1035

helmgpt:
  hidden_size: 256
  embd_pdrop: 0.1
  resid_pdrop: 0.1
  attn_pdrop: 0.1
  ff_dropout: 0.
  block_size: 140
  n_layer: 8
  n_heads: 8


trainer:
  _target_: lightning.Trainer
  accelerator: cuda
  num_nodes: 1
  devices: ${device_count:}
  accumulate_grad_batches: ${div_up:${loader.global_batch_size}, ${eval:${trainer.devices} * ${loader.batch_size} * ${trainer.num_nodes}}}
  gradient_clip_val: 1.0
  precision: 64-true
  num_sanity_val_steps: 2
  max_epochs: 100
  max_steps: 1_000_000
  log_every_n_steps: 10
  limit_train_batches: 1.0   # train on full dataset, can be used to toggle quick run
  limit_val_batches: 1.0     # validate on full dataset, can be used to toggle quick run
  #val_check_interval: 40 #954
  check_val_every_n_epoch: 1


wandb:
  project: peptune
  notes: null
  group: null
  job_type: null
  name: sophia-tang
  id: ${.name}_nov12_set2

hydra:
  run:
    dir: ./${now:%Y.%m.%d}/
  job:
    chdir: True

checkpointing:
  # Use custom `save_dir` if, e.g., saving to S3 bucket, otherwise leave this parameter as is
  save_dir: ${cwd:}
  # Note: `checkpoints` path should correspond to `checkpoint_every_n_steps.dirpath`
  resume_from_ckpt: True
  resume_ckpt_path: /home/st512/peptune/scripts/peptide-mdlm-mcts/checkpoints/11M-old-tokenizer/epoch=7-step=108225.ckpt

callbacks:
  model_checkpoint:
    _target_: pytorch_lightning.callbacks.ModelCheckpoint
    every_n_epochs: 1
    monitor: "val/nll"
    save_top_k: 10
    mode: "min"
    dirpath: '/home/st512/peptune/scripts/peptide-mdlm-mcts/checkpoints/11M-old-tokenizer'