Upload model

config.json

@@ -0,0 +1,15 @@
+{
+  "architectures": [
+    "WangchanbertaEncoderModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration.WangchanbertaEncoderConfig",
+    "AutoModel": "wangchanberta_cross_clip.WangchanbertaEncoderModel"
+  },
+  "dropout": 0.2,
+  "input_text_embedding_dim": 768,
+  "output_embedding_dim": 512,
+  "th_model_base": "airesearch/wangchanberta-base-att-spm-uncased",
+  "torch_dtype": "float32",
+  "transformers_version": "4.29.1"
+}

configuration.py

@@ -0,0 +1,18 @@
+from typing import List
+
+from transformers import PretrainedConfig
+
+
+class WangchanbertaEncoderConfig(PretrainedConfig):
+    def __init__(
+        self,
+        th_model_base: str = "airesearch/wangchanberta-base-att-spm-uncased",
+        input_text_embedding_dim: int = 768,
+        output_embedding_dim: int = 512,
+        dropout: float = 0.2
+    ):
+        super().__init__()
+        self.th_model_base = th_model_base
+        self.input_text_embedding_dim = input_text_embedding_dim
+        self.output_embedding_dim = output_embedding_dim
+        self.dropout = dropout

projector_residual.py

@@ -0,0 +1,25 @@
+import torch.nn as nn
+
+
+class HeadProjectorResidual(nn.Module):
+    def __init__(
+        self,
+        input_embedding_dim: int = 1000,
+        output_embedding_dim: int = 512,
+        dropout: float = 0.4
+    ):
+        super().__init__()
+        self.projection = nn.Linear(input_embedding_dim, output_embedding_dim)
+        self.gelu = nn.GELU()
+        self.fc = nn.Linear(output_embedding_dim, output_embedding_dim)
+        self.dropout = nn.Dropout(dropout)
+        self.layer_norm = nn.LayerNorm(output_embedding_dim)
+    
+    def forward(self, x):
+        projected = self.projection(x)
+        x = self.gelu(projected)
+        x = self.fc(x)
+        x = self.dropout(x)
+        x = x + projected
+        x = self.layer_norm(x)
+        return x

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9389d57d4a8469bae436267b2147144f7ce9c2c7d5be063381db212eac2f729a
+size 423681349

th_encoder.py

@@ -0,0 +1,18 @@
+import torch.nn as nn
+from transformers import AutoModel
+
+
+class ThaiEncoder(nn.Module):
+    def __init__(self, model_name: str, trainable: bool = False) -> None:
+        super().__init__()
+        self.model = AutoModel.from_pretrained(model_name)
+        
+        for p in self.model.parameters():
+            p.requires_grad = trainable
+        
+        self.target_token_idx = 0
+        
+    def forward(self, input_ids, attention_mask):
+        output = self.model(input_ids=input_ids, attention_mask=attention_mask)
+        last_hidden_state = output.last_hidden_state
+        return last_hidden_state[:, self.target_token_idx, :]

wangchanberta_cross_clip.py

@@ -0,0 +1,29 @@
+import torch
+
+from transformers import PreTrainedModel, AutoTokenizer
+from transformers.configuration_utils import PretrainedConfig
+
+from .th_encoder import ThaiEncoder
+from .projector_residual import HeadProjectorResidual
+
+
+class WangchanbertaEncoderModel(PreTrainedModel):
+    def __init__(self, config: PretrainedConfig):
+        super().__init__(config)
+        self.text_tokenizer = AutoTokenizer.from_pretrained(config.th_model_base)
+        self.text_encoder = ThaiEncoder(model_name=config.th_model_base)
+        self.text_projector = HeadProjectorResidual(
+            input_embedding_dim=config.input_text_embedding_dim, 
+            output_embedding_dim=config.output_embedding_dim,
+            dropout=config.dropout
+        )
+        self.max_length = 200
+        
+    def forward(self, text: str):
+        tokened_word = self.text_tokenizer(text, padding='max_length', truncation=True, max_length=self.max_length)
+        text_vector = self.text_encoder(
+            input_ids=torch.tensor([tokened_word["input_ids"]]),
+            attention_mask=torch.tensor([tokened_word["attention_mask"]])
+        )
+        text_projected = self.text_projector(text_vector)
+        return text_projected