Make required modules and files

burmese_gpt/config.py

@@ -5,16 +5,16 @@ class ModelConfig:
     vocab_size: int = 30000
     embed_dim: int = 256
     num_heads: int = 8
-    num_layers: int = 6
+    num_layers: int = 4
     dropout: float = 0.1
-    max_seq_len: int = 512
+    max_seq_len: int = 128
 
 @dataclass
 class TrainingConfig:
     batch_size: int = 32
     learning_rate: float = 5e-5
-    num_epochs: int = 10
-    warmup_steps: int = 1000
-    weight_decay: float = 0.01
+    num_epochs: int = 5
     checkpoint_dir: str = "checkpoints"
-    log_dir: str = "logs"
+    log_dir: str = "logs"
+    save_every: int = 1
+    eval_every: int = 1

burmese_gpt/model.py

@@ -1,59 +0,0 @@
-import torch
-from torch import nn
-from torch.nn import TransformerEncoder, TransformerEncoderLayer
-from .config import ModelConfig
-
-class BurmeseGPT(nn.Module):
-    def __init__(self,config:ModelConfig):
-        super(BurmeseGPT, self).__init__()
-        self.config = config
-        # Token and positional embeddings
-        self.token_embedding = nn.Embedding(config.vocab_size, config.embed_dim)
-        self.pos_embedding = nn.Embedding(config.max_seq_len, config.embed_dim)
-
-        # Transformer layers
-        encoder_layers = TransformerEncoderLayer(
-            d_model=config.embed_dim,
-            nhead=config.num_heads,
-            dim_feedforward=4 * config.embed_dim,
-            dropout=config.dropout,
-            batch_first=True
-        )
-        self.transformer = TransformerEncoder(encoder_layers, config.num_layers)
-
-        # Output layer
-        self.ln = nn.LayerNorm(config.embed_dim)
-        self.fc = nn.Linear(config.embed_dim, config.vocab_size)
-
-        # Initialize weights
-        self.apply(self._init_weights)
-
-    def _init_weights(self, module):
-        if isinstance(module, nn.Linear):
-            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
-            if module.bias is not None:
-                torch.nn.init.zeros_(module.bias)
-        elif isinstance(module, nn.Embedding):
-            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
-
-    def forward(self, x, attention_mask=None):
-        device = x.device
-        seq_len = x.size(1)
-
-        # Create position ids
-        position_ids = torch.arange(seq_len, dtype=torch.long, device=device).unsqueeze(0)
-
-        # Get embeddings
-        token_embeds = self.token_embedding(x)
-        pos_embeds = self.pos_embedding(position_ids)
-        x = token_embeds + pos_embeds
-
-        # Create causal mask
-        mask = torch.triu(torch.ones(seq_len, seq_len, device=device), diagonal=1).bool()
-
-        # Transformer
-        x = self.transformer(x, mask=mask, src_key_padding_mask=attention_mask)
-        x = self.ln(x)
-        logits = self.fc(x)
-
-        return logits

burmese_gpt/models/model.py

@@ -0,0 +1,8 @@
+from torch import nn
+from burmese_gpt.config import ModelConfig
+
+class BurmeseGPT(nn.Module):
+    def __init__(self,config:ModelConfig):
+        super(BurmeseGPT, self).__init__()
+        self.config = config
+        # Continue the rest

requirements.txt

@@ -1 +1,6 @@
-streamlit
+streamlit>=1.20.0
+torch>=2.0.0
+transformers>=4.30.0
+datasets>=2.12.0
+tqdm>=4.65.0
+numpy>=1.24.0

space.py → scripts/space.py

@@ -9,7 +9,7 @@ st.set_page_config(
 
 # Create a sidebar with a title and a brief description
 st.sidebar.title("Burmese GPT")
-st.sidebar.write("A language model app for generating and chatting in Burmese.")
+st.sidebar.write("A language models app for generating and chatting in Burmese.")
 
 # Create a selectbox to choose the view
 view_options = ["Sampling", "Chat Interface"]
@@ -18,7 +18,7 @@ selected_view = st.sidebar.selectbox("Select a view:", view_options)
 # Create a main area
 if selected_view == "Sampling":
     st.title("Sampling")
-    st.write("Generate text using the pre-trained model:")
+    st.write("Generate text using the pre-trained models:")
 
     # Create a text input field for the prompt
     prompt = st.text_input("Prompt:", value="")
@@ -37,15 +37,15 @@ if selected_view == "Sampling":
 
 elif selected_view == "Chat Interface":
     st.title("Chat Interface")
-    st.write("Chat with the fine-tuned model:")
+    st.write("Chat with the fine-tuned models:")
 
     # Create a text input field for the user input
     user_input = st.text_input("You:", value="")
 
-    # Create a button to send the input to the model
+    # Create a button to send the input to the models
     send_button = st.button("Send")
 
-    # Create an output area to display the model's response
+    # Create an output area to display the models's response
     response_area = st.text_area("Model:", height=200, disabled=True)
 
     # Add some space between the input and output areas

setup.py

@@ -2,4 +2,5 @@ from setuptools import  setup
 
 setup(
     name="burmese_gpt",
+
 )