Fix AttributeError in _init_weights for LayerNorm
Browse filesWhen loading this model for a downstream task like token classification using AutoModelForTokenClassification.from_pretrained, the internal call to init_weights() fails with an AttributeError.
This happens because some LayerNorm layers in the model are defined with elementwise_affine=False, meaning their .weight and .bias attributes are None. The _init_weights method does not check for this before trying to access .data, causing a crash.
This PR adds a check to ensure .weight and .bias are not None before they are accessed, fixing the loading issue and allowing the model to be easily fine-tuned for downstream tasks.
- modeling_ltgbert.py +4 -2
modeling_ltgbert.py
CHANGED
@@ -255,8 +255,10 @@ class LtgbertPreTrainedModel(PreTrainedModel):
|
|
255 |
elif isinstance(module, nn.Embedding):
|
256 |
nn.init.trunc_normal_(module.weight.data, mean=0.0, std=std, a=-2*std, b=2*std)
|
257 |
elif isinstance(module, nn.LayerNorm):
|
258 |
-
module.bias
|
259 |
-
|
|
|
|
|
260 |
|
261 |
|
262 |
class LtgbertModel(LtgbertPreTrainedModel):
|
|
|
255 |
elif isinstance(module, nn.Embedding):
|
256 |
nn.init.trunc_normal_(module.weight.data, mean=0.0, std=std, a=-2*std, b=2*std)
|
257 |
elif isinstance(module, nn.LayerNorm):
|
258 |
+
if module.bias is not None:
|
259 |
+
module.bias.data.zero_()
|
260 |
+
if module.weight is not None:
|
261 |
+
module.weight.data.fill_(1.0)
|
262 |
|
263 |
|
264 |
class LtgbertModel(LtgbertPreTrainedModel):
|