data-pipline.py

import re
import unicodedata
import random


input_file = "/kaggle/input/kaz-rus-eng-wiki/combined.txt"
train_file = "train.txt"
dev_file = "dev.txt"


def normalize_text(text):
    text = unicodedata.normalize("NFC", text) 
    text = text.lower()  
    return text

def clean_text(text):
    text = re.sub(r"<[^>]+>", " ", text)  
    text = re.sub(r"\[\[.*?\]\]", " ", text)  
    text = re.sub(r"\s+", " ", text) 
    text = text.strip()
    return text


cleaned_lines = []
with open(input_file, "r", encoding="utf-8") as f:
    for line in f:
        line = line.strip()
        if len(line) < 10:  
            continue
        line = normalize_text(line)
        line = clean_text(line)
        if line:
            cleaned_lines.append(line)


random.shuffle(cleaned_lines)


split_index = int(0.8 * len(cleaned_lines))
train_lines = cleaned_lines[:split_index]
dev_lines = cleaned_lines[split_index:]


with open(train_file, "w", encoding="utf-8") as f:
    for line in train_lines:
        f.write(line + "\n")

with open(dev_file, "w", encoding="utf-8") as f:
    for line in dev_lines:
        f.write(line + "\n")

print(f"Train: {len(train_lines)}, Dev: {len(dev_lines)}")