Update app.py

app.py

@@ -2,7 +2,7 @@ from datasets import load_dataset, Dataset
 from transformers import GPT2LMHeadModel, GPT2Config, Trainer, TrainingArguments, PreTrainedTokenizerFast
 from tokenizers import Tokenizer, models, trainers
 
-# Load and combine datasets
+# Load datasets
 wiki_dataset = load_dataset("LexiconShiftInnovations/SinhalaWikipediaArticles")
 gov_dataset = load_dataset("Virajtharinda/SinhalaGOV")
 text_dataset = load_dataset("rmihiranga/sinhala-text-fullfill-v2")
@@ -13,6 +13,7 @@ combined_dataset = Dataset.from_dict({"text": combined_texts})
 tokenizer = Tokenizer(models.BPE())
 trainer = trainers.BpeTrainer(vocab_size=30000, special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
 tokenizer.train_from_iterator(combined_texts, trainer)
+tokenizer.save("sinhala_tokenizer.json")
 tokenizer = PreTrainedTokenizerFast(
     tokenizer_file="sinhala_tokenizer.json",
     unk_token="[UNK]", cls_token="[CLS]", sep_token="[SEP]", pad_token="[PAD]", mask_token="[MASK]"
@@ -31,14 +32,14 @@ model = GPT2LMHeadModel(config)
 training_args = TrainingArguments(
     output_dir="./sinhala_LM_V1",
     overwrite_output_dir=True,
-    num_train_epochs=1,  # Reduced for demo; increase as needed
+    num_train_epochs=1,
     per_device_train_batch_size=4,
     save_steps=10_000,
     save_total_limit=2,
     logging_steps=500,
     fp16=True,
     push_to_hub=True,
-    hub_model_id="OminduAnjana/sinhala_LM_V1",  # Replace with your username
+    hub_model_id="your-username/sinhala_LM_V1",  # Replace with your username
 )
 
 trainer = Trainer(model=model, args=training_args, train_dataset=tokenized_dataset)