dschulmeist commited on
Commit
330aefb
·
verified ·
1 Parent(s): c568f0c

Fix AttributeError in _init_weights for LayerNorm

Browse files

When loading this model for a downstream task like token classification using AutoModelForTokenClassification.from_pretrained, the internal call to init_weights() fails with an AttributeError.
This happens because some LayerNorm layers in the model are defined with elementwise_affine=False, meaning their .weight and .bias attributes are None. The _init_weights method does not check for this before trying to access .data, causing a crash.
This PR adds a check to ensure .weight and .bias are not None before they are accessed, fixing the loading issue and allowing the model to be easily fine-tuned for downstream tasks.

Files changed (1) hide show
  1. modeling_ltgbert.py +4 -2
modeling_ltgbert.py CHANGED
@@ -255,8 +255,10 @@ class LtgbertPreTrainedModel(PreTrainedModel):
255
  elif isinstance(module, nn.Embedding):
256
  nn.init.trunc_normal_(module.weight.data, mean=0.0, std=std, a=-2*std, b=2*std)
257
  elif isinstance(module, nn.LayerNorm):
258
- module.bias.data.zero_()
259
- module.weight.data.fill_(1.0)
 
 
260
 
261
 
262
  class LtgbertModel(LtgbertPreTrainedModel):
 
255
  elif isinstance(module, nn.Embedding):
256
  nn.init.trunc_normal_(module.weight.data, mean=0.0, std=std, a=-2*std, b=2*std)
257
  elif isinstance(module, nn.LayerNorm):
258
+ if module.bias is not None:
259
+ module.bias.data.zero_()
260
+ if module.weight is not None:
261
+ module.weight.data.fill_(1.0)
262
 
263
 
264
  class LtgbertModel(LtgbertPreTrainedModel):