update

README.md

@@ -76,6 +76,33 @@ generation_config = GenerationConfig(
 
 repo_name = "BeardedMonster/SabiYarn-125M"
 model = AutoModelForCausalLM.from_pretrained(repo_name, trust_remote_code=True)
+tokenizer= AutoTokenizer.from_pretrained(repo_name, trust_remote_code=True)
+
+#Test on Urhobo
+input_ids = tokenizer("Eshare nana ri vwo ẹguọnọ rẹ iyono rẹ Aristotle vẹ Plato na,", return_tensors="pt")["input_ids"]  
+output = model.generate(input_ids, generation_config=generation_config, max_new_tokens=50)
+input_len = len(input_ids[0])
+print(tokenizer.decode(output[0][input_len:]))
+
+#Output
+ ọ da tobọ dianẹ ayen rhọnvwe kerhọ-ọ. Ọtiọyena, e de ruiruo aghwoghwo ọkieje. (1 Kọr. 7:9; 1 Kọr. 12:2) Vwọrẹ uyota
+
+#Test on Efik
+input_ids = tokenizer("Ke eyo Jesus ye mme mbet esie, etop emi ama ada ifụre ọsọk", return_tensors="pt")["input_ids"]  
+output = model.generate(input_ids, generation_config=generation_config, max_new_tokens=50)
+input_len = len(input_ids[0])
+print(tokenizer.decode(output[0][input_len:]))
+
+#Output
+. Edi ediwak nditọ Israel ẹtịn̄ ẹnọ nnyịn mîkemeke ndinam n̄kpọ Abasi.|end_of_text|Ebe foto si, Getty Images Ebe foto si, Getty Images Nkọwa foto, Ndị
+
+input_ids = tokenizer("Ke eyo Jesus ye mme mbet esie, etop emi ama ada ifụre ọsọk mme Jew oro esịt okobụn̄ọde ke ntak idiọkido ke Israel, oro ẹkenyụn̄ ẹdude ke mfụhọ ke itie-ufụn mme nsunsu ido edinam Ido Ukpono Mme Jew eke akpa isua ikie.", return_tensors="pt")["input_ids"]  
+output = model.generate(input_ids, generation_config=generation_config, max_new_tokens=50)
+input_len = len(input_ids[0])
+print(tokenizer.decode(output[0][input_len:]))
+
+#Output
+Kûsịn idem nnyịme ndifiọk nditọete nnyịn inemesịt onyụn̄ anam nnyịn ikpọn̄utom nnyịn. (Matt. 26:31; Luke 22:42
 
 # Test on English
 input_ids = tokenizer("How are you?", return_tensors="pt")["input_ids"]  
@@ -106,6 +133,24 @@ print(tokenizer.decode(output[0][input_len:]))
 Nkọwapụta 
 Ebe nrụọrụ weebụ na-ahụ maka gburugburu ebe
 
+
+# Test on FulFulde/Fulah
+input_ids = tokenizer("Jos un peeta gallure nɗer ɗi woyla caaka ɓanngeere lardu Naajeeriya. Gelle ɗen haa e ɗuuɗiri ɗun kamano", return_tensors="pt")["input_ids"]  
+output = model.generate(input_ids, generation_config=generation_config, max_new_tokens=50)
+input_len = len(input_ids[0])
+print(tokenizer.decode(output[0][input_len:]))
+
+#Output
+jogiiji maɓɓe nder lesdi Naajeeriya. |end_o|end_of_text|** Muhammadu_Buhari ** Muhammadu Buhari ko leydi e hukuma pamarun e hukuma pamarun e hukuma pamarun e hukuma pamarun e hukum
+
+input_ids = tokenizer("Si hooreejo leydi on (himo wi’ee kadi persidan) accitii laamu, ko woote waɗetee, ɓurɗo jogaade yimɓe on halfinee laamu yeru happu.", return_tensors="pt")["input_ids"]  
+output = model.generate(input_ids, generation_config=generation_config, max_new_tokens=50)
+input_len = len(input_ids[0])
+print(tokenizer.decode(output[0][input_len:]))
+
+#Output
+|end_of_text|So en nganndii e hitaande 2010, o wiyi : “ko ñalawma hannde golle pulaar walla mbiyen jogiiɗo”. Eɗen mbaawi wiyde «u2008
+
 # Test on Hausa
 input_ids = tokenizer("Ministan ya ƙara da cewa dole ne Mista Netanyahu ya sanya ranar da", return_tensors="pt")["input_ids"]  
 output = model.generate(input_ids, generation_config=generation_config, max_new_tokens=50)