Sanskrit tokenizer

.gitignore

@@ -0,0 +1,23 @@
+# Ignore Python cache files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Ignore virtual environment
+venv/
+env/
+.venv/
+
+
+# Ignore trained model files
+model_mnist_*.pth
+
+# Ignore Jupyter Notebook checkpoints
+.ipynb_checkpoints/
+
+# Ignore system files
+.DS_Store
+Thumbs.db
+
+# Ignore logs
+*.log 

README.md

@@ -11,3 +11,92 @@ short_description: Sanskrit Tokenizer Demo
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# Sanskrit BPE Tokenizer
+
+This is a Byte-Pair Encoding (BPE) tokenizer specifically designed for Sanskrit text. It provides a web interface for both training the tokenizer and using it to encode/decode text.
+
+## Features
+
+- Train BPE tokenizer on custom Sanskrit text
+- Tokenize Sanskrit text using the trained model
+- Verify tokenization accuracy through decode/encode cycle
+- User-friendly web interface
+
+## Usage
+
+1. Go to the "Train" tab and paste your Sanskrit training text
+2. Click "Train Tokenizer" to train the model
+3. Switch to the "Tokenize" tab to tokenize new text
+4. Enter text and click "Tokenize" to see the results
+
+## Example Text
+```
+चीराण्यपास्याज्जनकस्य कन्या नेयं प्रतिज्ञा मम दत्तपूर्वा।
+यथासुखं गच्छतु राजपुत्री वनं समग्रा सह सर्वरत्नैः॥
+```
+
+# To deploy this to Hugging Face Spaces:
+
+1. Create a new Space on Hugging Face:
+```bash
+huggingface-cli login
+huggingface-cli repo create sanskrit-tokenizer-demo --type space
+```
+2. Initialize git and push your code:
+```bash
+git init
+git add .
+git commit -m "Initial commit"
+git remote add origin https://huggingface.co/spaces/your-username/sanskrit-tokenizer-demo
+git push -u origin main
+```
+
+# Steps to Run Locally
+1. Create and activate a virtual environment:
+```bash
+python -m venv venv
+source venv/bin/activate  # On Windows: venv\Scripts\activate
+```
+
+2. Install the requirements and the Hugging Face CLI:
+```bash
+pip install -r requirements.txt
+pip install --upgrade huggingface-hub
+```
+
+4. To run the app:
+```bash
+python src/app.py
+```
+    The interface will be available at `http://localhost:7860` by default.
+
+# Logs
+
+Orignal (before Hugging Space code):
+```
+(venv) gitesh.grover@Giteshs-MacBook-Pro ai-era-assignment11 % python src/main.py
+Reading file...
+Length of training dataset for token 620958
+UTF-8 tokens length (without considering regex) 1719701
+Preprocessing tokens...
+UTF-8 Split tokens length (due to regex) 77799
+Training BPE...
+Learning BPE: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 4744/4744 [06:21<00:00, 12.45it/s]
+Tokenizer training completed in 381.08 seconds
+Vocab size: 5000
+Original tokens length 1719701, while updated tokens length 77799
+Compression Ratio 22.10
+Saving Tokenizer Vocab in files...
+Testing the validity of tokenizer...
+True
+```
+
+After Hugging Space implementation:
+```
+Learning BPE: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 4744/4744 [03:25<00:00, 23.07it/s]
+
+(Huggingface app output)
+Training completed! Vocabulary size: 5000 and Compression Ratio: 9.42
+
+```

data/paired_tokens.bpe

@@ -0,0 +1,4745 @@
+Token version 1
+(224, 164):256
+(224, 165):257
+(257, 141):258
+(32, 256):259
+(258, 256):260
+(256, 190):261
+(261, 256):262
+(256, 191):263
+(256, 164):264
+(256, 176):265
+(257, 129):266
+(257, 135):267
+(263, 256):268
+(260, 175):269
+(256, 168):270
+(256, 174):271
+(260, 176):272
+(256, 130):273
+(256, 184):274
+(256, 181):275
+(257, 139):276
+(260, 164):277
+(257, 164):278
+(259, 184):279
+(266, 256):280
+(257, 165):281
+(259, 170):282
+(260, 181):283
+(256, 131):284
+(256, 149):285
+(259, 164):286
+(257, 128):287
+(256, 175):288
+(259, 181):289
+(262, 168):290
+(256, 166):291
+(256, 185):292
+(259, 149):293
+(256, 182):294
+(259, 174):295
+(257, 131):296
+(256, 170):297
+(265, 260):298
+(263, 264):299
+(257, 136):300
+(259, 168):301
+(262, 174):302
+(259, 154):303
+(256, 183):304
+(262, 130):305
+(259, 166):306
+(262, 176):307
+(259, 176):308
+(257, 130):309
+(262, 164):310
+(256, 163):311
+(260, 174):312
+(260, 183):313
+(259, 182):314
+(282, 272):315
+(256, 165):316
+(259, 173):317
+(260, 154):318
+(259, 175):319
+(256, 178):320
+(271, 258):321
+(256, 173):322
+(262, 184):323
+(262, 131):324
+(274, 277):325
+(256, 154):326
+(256, 151):327
+(256, 156):328
+(274, 269):329
+(267, 270):330
+(266, 265):331
+(270, 277):332
+(289, 268):333
+(284, 278):334
+(257, 140):335
+(296, 264):336
+(285, 313):337
+(259, 185):338
+(262, 175):339
+(268, 168):340
+(256, 167):341
+(265, 283):342
+(295, 292):343
+(284, 281):344
+(264, 272):345
+(262, 166):346
+(268, 183):347
+(260, 167):348
+(259, 151):349
+(262, 170):350
+(308, 262):351
+(268, 181):352
+(259, 172):353
+(260, 168):354
+(262, 163):355
+(264, 263):356
+(267, 275):357
+(256, 189):358
+(290, 258):359
+(264, 267):360
+(262, 182):361
+(262, 154):362
+(260, 159):363
+(262, 181):364
+(294, 318):365
+(321, 281):366
+(260, 165):367
+(298, 174):368
+(280, 166):369
+(259, 156):370
+(264, 258):371
+(267, 304):372
+(321, 278):373
+(266, 264):374
+(283, 261):375
+(263, 265):376
+(262, 149):377
+(279, 271):378
+(260, 166):379
+(259, 133):380
+(262, 178):381
+(262, 185):382
+(260, 170):383
+(282, 265):384
+(276, 358):385
+(279, 342):386
+(280, 149):387
+(260, 155):388
+(296, 304):389
+(264, 269):390
+(351, 156):391
+(259, 167):392
+(153, 260):393
+(300, 275):394
+(259, 137):395
+(279, 273):396
+(268, 182):397
+(275, 362):398
+(280, 183):399
+(280, 170):400
+(266, 271):401
+(316, 261):402
+(264, 277):403
+(268, 166):404
+(268, 170):405
+(270, 269):406
+(268, 131):407
+(266, 270):408
+(256, 172):409
+(262, 151):410
+(259, 178):411
+(262, 167):412
+(324, 278):413
+(268, 175):414
+(262, 173):415
+(310, 312):416
+(270, 258):417
+(268, 174):418
+(269, 273):419
+(286, 283):420
+(268, 149):421
+(268, 130):422
+(274, 312):423
+(260, 184):424
+(265, 277):425
+(276, 285):426
+(369, 348):427
+(387, 277):428
+(260, 161):429
+(260, 158):430
+(260, 160):431
+(286, 267):432
+(261, 278):433
+(259, 135):434
+(256, 133):435
+(298, 183):436
+(267, 291):437
+(274, 273):438
+(264, 276):439
+(323, 277):440
+(280, 131):441
+(262, 156):442
+(256, 393):443
+(286, 266):444
+(287, 264):445
+(287, 265):446
+(302, 258):447
+(267, 264):448
+(267, 294):449
+(269, 271):450
+(389, 363):451
+(326, 388):452
+(280, 163):453
+(264, 284):454
+(310, 258):455
+(317, 275):456
+(324, 281):457
+(290, 305):458
+(347, 269):459
+(272, 275):460
+(288, 261):461
+(272, 271):462
+(267, 311):463
+(270, 273):464
+(288, 276):465
+(361, 318):466
+(308, 302):467
+(300, 284):468
+(286, 291):469
+(270, 354):470
+(306, 357):471
+(266, 275):472
+(279, 292):473
+(280, 167):474
+(288, 273):475
+(259, 143):476
+(293, 336):477
+(287, 271):478
+(260, 149):479
+(287, 275):480
+(297, 272):481
+(314, 272):482
+(298, 163):483
+(290, 263):484
+(290, 277):485
+(256, 150):486
+(280, 178):487
+(379, 272):488
+(266, 345):489
+(332, 263):490
+(395, 398):491
+(274, 367):492
+(295, 267):493
+(282, 331):494
+(338, 263):495
+(294, 269):496
+(297, 263):497
+(267, 278):498
+(279, 283):499
+(291, 269):500
+(265, 273):501
+(293, 313):502
+(372, 266):503
+(300, 265):504
+(268, 173):505
+(353, 272):506
+(302, 263):507
+(280, 182):508
+(267, 281):509
+(307, 367):510
+(268, 163):511
+(260, 173):512
+(280, 173):513
+(256, 152):514
+(264, 273):515
+(268, 185):516
+(269, 276):517
+(347, 431):518
+(260, 163):519
+(274, 283):520
+(285, 277):521
+(371, 281):522
+(268, 184):523
+(289, 269):524
+(274, 263):525
+(266, 273):526
+(298, 182):527
+(280, 150):528
+(261, 281):529
+(260, 156):530
+(309, 297):531
+(276, 297):532
+(325, 266):533
+(32, 281):534
+(266, 298):535
+(356, 281):536
+(268, 167):537
+(268, 151):538
+(285, 272):539
+(283, 265):540
+(259, 134):541
+(443, 149):542
+(293, 272):543
+(280, 151):544
+(264, 424):545
+(315, 264):546
+(317, 309):547
+(267, 288):548
+(274, 271):549
+(262, 172):550
+(447, 278):551
+(392, 368):552
+(269, 267):553
+(287, 270):554
+(299, 273):555
+(291, 258):556
+(268, 154):557
+(289, 300):558
+(269, 261):559
+(266, 398):560
+(312, 311):561
+(269, 275):562
+(263, 298):563
+(264, 305):564
+(309, 265):565
+(328, 430):566
+(274, 261):567
+(314, 265):568
+(158, 318):569
+(359, 278):570
+(260, 172):571
+(260, 178):572
+(296, 292):573
+(269, 339):574
+(337, 269):575
+(377, 313):576
+(286, 345):577
+(256, 143):578
+(355, 263):579
+(276, 291):580
+(299, 272):581
+(293, 381):582
+(355, 429):583
+(333, 182):584
+(350, 263):585
+(296, 297):586
+(299, 261):587
+(280, 185):588
+(276, 304):589
+(327, 272):590
+(311, 429):591
+(282, 408):592
+(323, 269):593
+(149, 313):594
+(282, 489):595
+(296, 294):596
+(268, 178):597
+(306, 451):598
+(271, 270):599
+(289, 261):600
+(331, 283):601
+(286, 329):602
+(270, 379):603
+(333, 166):604
+(287, 297):605
+(293, 422):606
+(270, 348):607
+(292, 273):608
+(276, 341):609
+(447, 281):610
+(360, 278):611
+(319, 291):612
+(353, 292):613
+(265, 267):614
+(343, 307):615
+(268, 156):616
+(310, 272):617
+(330, 488):618
+(315, 390):619
+(303, 278):620
+(290, 273):621
+(328, 288):622
+(274, 272):623
+(296, 291):624
+(319, 402):625
+(306, 283):626
+(293, 316):627
+(308, 316):628
+(294, 283):629
+(152, 275):630
+(322, 309):631
+(287, 304):632
+(309, 320):633
+(476, 275):634
+(317, 307):635
+(301, 265):636
+(262, 183):637
+(353, 320):638
+(280, 154):639
+(289, 326):640
+(351, 594):641
+(264, 261):642
+(443, 151):643
+(416, 270):644
+(393, 151):645
+(370, 270):646
+(300, 298):647
+(351, 630):648
+(355, 305):649
+(403, 271):650
+(322, 269):651
+(360, 281):652
+(374, 375):653
+(346, 258):654
+(407, 278):655
+(295, 271):656
+(295, 270):657
+(303, 394):658
+(315, 275):659
+(297, 265):660
+(343, 550):661
+(282, 583):662
+(411, 426):663
+(288, 267):664
+(288, 302):665
+(286, 402):666
+(271, 428):667
+(271, 400):668
+(286, 423):669
+(350, 277):670
+(347, 363):671
+(340, 354):672
+(293, 307):673
+(279, 367):674
+(298, 165):675
+(371, 278):676
+(292, 312):677
+(336, 277):678
+(338, 269):679
+(259, 152):680
+(325, 272):681
+(286, 272):682
+(356, 278):683
+(279, 312):684
+(391, 261):685
+(280, 156):686
+(276, 265):687
+(327, 275):688
+(264, 344):689
+(641, 274):690
+(271, 290):691
+(332, 267):692
+(287, 288):693
+(557, 388):694
+(282, 309):695
+(289, 446):696
+(270, 366):697
+(275, 269):698
+(286, 275):699
+(333, 181):700
+(397, 318):701
+(665, 323):702
+(285, 307):703
+(289, 265):704
+(298, 156):705
+(269, 305):706
+(285, 336):707
+(412, 574):708
+(441, 486):709
+(333, 168):710
+(386, 267):711
+(304, 363):712
+(317, 272):713
+(264, 334):714
+(460, 287):715
+(282, 299):716
+(271, 269):717
+(291, 272):718
+(299, 269):719
+(265, 442):720
+(265, 311):721
+(340, 287):722
+(267, 292):723
+(158, 530):724
+(283, 273):725
+(282, 264):726
+(286, 273):727
+(264, 281):728
+(271, 267):729
+(294, 272):730
+(286, 297):731
+(271, 357):732
+(276, 292):733
+(300, 334):734
+(319, 316):735
+(298, 167):736
+(359, 281):737
+(290, 269):738
+(434, 275):739
+(494, 399):740
+(372, 431):741
+(662, 275):742
+(299, 276):743
+(264, 375):744
+(279, 277):745
+(260, 151):746
+(293, 368):747
+(259, 150):748
+(306, 272):749
+(317, 265):750
+(336, 269):751
+(279, 445):752
+(299, 258):753
+(271, 383):754
+(506, 677):755
+(308, 337):756
+(320, 273):757
+(291, 527):758
+(256, 159):759
+(395, 297):760
+(632, 312):761
+(598, 375):762
+(303, 281):763
+(264, 287):764
+(370, 327):765
+(309, 304):766
+(306, 352):767
+(351, 181):768
+(768, 311):769
+(409, 715):770
+(276, 326):771
+(256, 135):772
+(346, 269):773
+(407, 281):774
+(264, 325):775
+(300, 365):776
+(267, 358):777
+(301, 302):778
+(298, 170):779
+(289, 377):780
+(335, 278):781
+(299, 334):782
+(308, 311):783
+(713, 310):784
+(391, 417):785
+(300, 344):786
+(287, 337):787
+(482, 653):788
+(337, 561):789
+(256, 161):790
+(449, 540):791
+(293, 331):792
+(256, 569):793
+(299, 263):794
+(293, 265):795
+(271, 352):796
+(369, 272):797
+(627, 273):798
+(286, 454):799
+(412, 405):800
+(299, 413):801
+(276, 322):802
+(288, 417):803
+(317, 478):804
+(523, 277):805
+(384, 271):806
+(326, 318):807
+(420, 273):808
+(306, 709):809
+(264, 271):810
+(322, 275):811
+(262, 165):812
+(335, 265):813
+(279, 331):814
+(328, 270):815
+(286, 316):816
+(276, 271):817
+(333, 183):818
+(314, 345):819
+(456, 459):820
+(296, 316):821
+(319, 427):822
+(319, 474):823
+(370, 480):824
+(289, 270):825
+(296, 311):826
+(350, 269):827
+(305, 365):828
+(266, 325):829
+(276, 311):830
+(301, 299):831
+(349, 453):832
+(327, 354):833
+(301, 586):834
+(291, 357):835
+(315, 414):836
+(291, 283):837
+(382, 561):838
+(543, 427):839
+(275, 267):840
+(278, 278):841
+(299, 366):842
+(279, 291):843
+(282, 793):844
+(274, 383):845
+(518, 376):846
+(266, 368):847
+(378, 383):848
+(315, 322):849
+(299, 373):850
+(317, 688):851
+(266, 274):852
+(341, 368):853
+(264, 278):854
+(436, 322):855
+(315, 292):856
+(300, 285):857
+(279, 390):858
+(295, 461):859
+(267, 285):860
+(340, 277):861
+(399, 363):862
+(256, 137):863
+(299, 375):864
+(264, 385):865
+(298, 185):866
+(270, 373):867
+(296, 327):868
+(578, 275):869
+(270, 271):870
+(265, 531):871
+(450, 290):872
+(353, 427):873
+(421, 313):874
+(289, 274):875
+(291, 348):876
+(256, 162):877
+(282, 496):878
+(301, 352):879
+(338, 265):880
+(320, 426):881
+(267, 326):882
+(262, 645):883
+(285, 273):884
+(469, 261):885
+(264, 283):886
+(290, 302):887
+(299, 344):888
+(264, 329):889
+(349, 452):890
+(404, 273):891
+(619, 560):892
+(333, 149):893
+(661, 320):894
+(270, 488):895
+(282, 510):896
+(262, 189):897
+(270, 312):898
+(295, 305):899
+(293, 425):900
+(279, 401):901
+(477, 375):902
+(416, 261):903
+(300, 342):904
+(274, 330):905
+(287, 273):906
+(264, 454):907
+(333, 170):908
+(282, 350):909
+(301, 376):910
+(343, 644):911
+(268, 172):912
+(267, 328):913
+(271, 355):914
+(264, 297):915
+(280, 172):916
+(615, 442):917
+(506, 838):918
+(306, 294):919
+(319, 276):920
+(374, 424):921
+(289, 341):922
+(544, 272):923
+(315, 274):924
+(290, 265):925
+(315, 670):926
+(293, 302):927
+(695, 342):928
+(378, 614):929
+(403, 283):930
+(328, 269):931
+(286, 330):932
+(311, 305):933
+(270, 283):934
+(285, 361):935
+(473, 623):936
+(295, 307):937
+(332, 272):938
+(298, 151):939
+(286, 439):940
+(262, 150):941
+(502, 345):942
+(317, 761):943
+(275, 323):944
+(374, 277):945
+(306, 331):946
+(297, 264):947
+(264, 479):948
+(269, 310):949
+(296, 496):950
+(285, 276):951
+(301, 271):952
+(259, 171):953
+(356, 265):954
+(332, 276):955
+(295, 290):956
+(300, 294):957
+(349, 331):958
+(606, 326):959
+(612, 263):960
+(532, 271):961
+(340, 334):962
+(459, 507):963
+(411, 789):964
+(333, 174):965
+(325, 402):966
+(287, 483):967
+(289, 437):968
+(282, 291):969
+(337, 311):970
+(286, 371):971
+(292, 263):972
+(276, 650):973
+(372, 283):974
+(372, 305):975
+(335, 281):976
+(274, 292):977
+(332, 273):978
+(349, 264):979
+(361, 283):980
+(271, 571):981
+(301, 563):982
+(264, 383):983
+(282, 821):984
+(300, 406):985
+(288, 271):986
+(307, 379):987
+(406, 276):988
+(440, 272):989
+(434, 536):990
+(279, 528):991
+(297, 489):992
+(420, 461):993
+(288, 284):994
+(285, 265):995
+(279, 385):996
+(333, 154):997
+(286, 305):998
+(340, 258):999
+(923, 480):1000
+(275, 270):1001
+(263, 342):1002
+(547, 271):1003
+(288, 366):1004
+(269, 284):1005
+(267, 390):1006
+(279, 330):1007
+(285, 368):1008
+(363, 272):1009
+(435, 316):1010
+(279, 472):1011
+(319, 428):1012
+(314, 426):1013
+(279, 261):1014
+(297, 496):1015
+(337, 511):1016
+(263, 278):1017
+(404, 348):1018
+(680, 687):1019
+(470, 327):1020
+(626, 616):1021
+(307, 269):1022
+(285, 487):1023
+(508, 318):1024
+(290, 287):1025
+(420, 305):1026
+(274, 267):1027
+(409, 379):1028
+(300, 304):1029
+(296, 337):1030
+(270, 334):1031
+(428, 375):1032
+(391, 430):1033
+(307, 311):1034
+(615, 316):1035
+(336, 273):1036
+(339, 270):1037
+(279, 269):1038
+(403, 542):1039
+(299, 284):1040
+(333, 184):1041
+(416, 328):1042
+(426, 277):1043
+(295, 310):1044
+(380, 270):1045
+(382, 273):1046
+(264, 439):1047
+(309, 270):1048
+(295, 408):1049
+(291, 273):1050
+(338, 270):1051
+(270, 344):1052
+(315, 355):1053
+(325, 283):1054
+(831, 419):1055
+(409, 348):1056
+(263, 368):1057
+(293, 335):1058
+(333, 167):1059
+(640, 464):1060
+(279, 335):1061
+(279, 412):1062
+(272, 472):1063
+(279, 1000):1064
+(315, 291):1065
+(319, 267):1066
+(416, 290):1067
+(288, 291):1068
+(297, 331):1069
+(298, 166):1070
+(384, 273):1071
+(270, 352):1072
+(264, 423):1073
+(787, 519):1074
+(295, 406):1075
+(430, 310):1076
+(262, 159):1077
+(306, 397):1078
+(410, 272):1079
+(289, 267):1080
+(573, 269):1081
+(325, 267):1082
+(287, 790):1083
+(273, 274):1084
+(302, 292):1085
+(301, 516):1086
+(441, 281):1087
+(271, 275):1088
+(272, 273):1089
+(282, 307):1090
+(453, 429):1091
+(823, 846):1092
+(265, 316):1093
+(353, 381):1094
+(271, 323):1095
+(265, 373):1096
+(262, 724):1097
+(267, 356):1098
+(584, 283):1099
+(315, 328):1100
+(314, 513):1101
+(325, 291):1102
+(314, 285):1103
+(297, 269):1104
+(411, 542):1105
+(634, 667):1106
+(264, 302):1107
+(337, 288):1108
+(264, 291):1109
+(661, 382):1110
+(256, 134):1111
+(301, 397):1112
+(333, 156):1113
+(309, 264):1114
+(386, 273):1115
+(750, 264):1116
+(299, 267):1117
+(269, 302):1118
+(293, 586):1119
+(455, 278):1120
+(319, 345):1121
+(262, 569):1122
+(365, 394):1123
+(282, 342):1124
+(289, 925):1125
+(309, 288):1126
+(349, 607):1127
+(282, 1020):1128
+(361, 272):1129
+(307, 288):1130
+(384, 352):1131
+(314, 264):1132
+(320, 383):1133
+(268, 645):1134
+(303, 585):1135
+(275, 404):1136
+(396, 275):1137
+(265, 637):1138
+(293, 882):1139
+(942, 414):1140
+(275, 957):1141
+(1141, 754):1142
+(1142, 1037):1143
+(857, 548):1144
+(267, 475):1145
+(295, 341):1146
+(293, 418):1147
+(848, 272):1148
+(319, 271):1149
+(271, 309):1150
+(290, 270):1151
+(745, 272):1152
+(285, 302):1153
+(343, 310):1154
+(268, 150):1155
+(323, 331):1156
+(592, 265):1157
+(274, 479):1158
+(288, 373):1159
+(264, 418):1160
+(535, 166):1161
+(350, 272):1162
+(309, 483):1163
+(299, 457):1164
+(301, 405):1165
+(301, 410):1166
+(45, 39):1167
+(293, 508):1168
+(271, 891):1169
+(288, 332):1170
+(271, 322):1171
+(901, 292):1172
+(308, 531):1173
+(268, 152):1174
+(288, 335):1175
+(309, 316):1176
+(803, 281):1177
+(340, 344):1178
+(275, 294):1179
+(360, 304):1180
+(297, 277):1181
+(289, 339):1182
+(604, 269):1183
+(288, 428):1184
+(270, 276):1185
+(311, 273):1186
+(271, 512):1187
+(319, 330):1188
+(283, 320):1189
+(305, 325):1190
+(286, 271):1191
+(303, 290):1192
+(274, 650):1193
+(730, 741):1194
+(987, 633):1195
+(469, 258):1196
+(291, 512):1197
+(684, 336):1198
+(635, 854):1199
+(635, 728):1200
+(287, 278):1201
+(287, 320):1202
+(467, 284):1203
+(392, 270):1204
+(543, 609):1205
+(315, 271):1206
+(272, 485):1207
+(409, 320):1208
+(327, 271):1209
+(309, 271):1210
+(455, 281):1211
+(259, 139):1212
+(700, 537):1213
+(275, 276):1214
+(434, 356):1215
+(264, 610):1216
+(314, 565):1217
+(271, 292):1218
+(362, 318):1219
+(289, 389):1220
+(293, 1144):1221
+(301, 414):1222
+(291, 723):1223
+(525, 278):1224
+(770, 522):1225
+(624, 348):1226
+(288, 263):1227
+(275, 273):1228
+(270, 346):1229
+(378, 332):1230
+(275, 419):1231
+(315, 294):1232
+(326, 265):1233
+(279, 369):1234
+(315, 288):1235
+(306, 394):1236
+(288, 334):1237
+(984, 352):1238
+(349, 328):1239
+(270, 267):1240
+(467, 329):1241
+(310, 263):1242
+(279, 374):1243
+(783, 267):1244
+(476, 304):1245
+(290, 312):1246
+(289, 336):1247
+(314, 521):1248
+(1138, 1009):1249
+(264, 367):1250
+(456, 267):1251
+(508, 272):1252
+(274, 335):1253
+(289, 678):1254
+(415, 706):1255
+(301, 407):1256
+(401, 866):1257
+(309, 342):1258
+(437, 258):1259
+(267, 322):1260
+(314, 826):1261
+(467, 273):1262
+(421, 462):1263
+(869, 667):1264
+(307, 511):1265
+(288, 433):1266
+(380, 322):1267
+(696, 562):1268
+(289, 1030):1269
+(279, 307):1270
+(302, 269):1271
+(275, 446):1272
+(437, 273):1273
+(276, 398):1274
+(32, 278):1275
+(274, 346):1276
+(314, 539):1277
+(303, 264):1278
+(405, 272):1279
+(325, 273):1280
+(259, 155):1281
+(590, 302):1282
+(364, 285):1283
+(289, 291):1284
+(977, 623):1285
+(988, 406):1286
+(705, 408):1287
+(760, 708):1288
+(384, 269):1289
+(554, 305):1290
+(441, 278):1291
+(295, 588):1292
+(309, 425):1293
+(275, 436):1294
+(341, 269):1295
+(297, 310):1296
+(317, 718):1297
+(274, 374):1298
+(289, 381):1299
+(349, 376):1300
+(256, 171):1301
+(328, 274):1302
+(290, 267):1303
+(436, 311):1304
+(510, 273):1305
+(592, 284):1306
+(545, 269):1307
+(289, 323):1308
+(631, 275):1309
+(343, 364):1310
+(916, 348):1311
+(780, 450):1312
+(582, 267):1313
+(287, 285):1314
+(301, 290):1315
+(279, 985):1316
+(271, 410):1317
+(297, 291):1318
+(319, 371):1319
+(286, 310):1320
+(319, 294):1321
+(752, 261):1322
+(401, 307):1323
+(299, 433):1324
+(275, 265):1325
+(289, 464):1326
+(546, 405):1327
+(399, 383):1328
+(306, 527):1329
+(1097, 320):1330
+(275, 397):1331
+(680, 276):1332
+(271, 581):1333
+(303, 300):1334
+(256, 724):1335
+(290, 354):1336
+(396, 294):1337
+(289, 436):1338
+(385, 497):1339
+(349, 744):1340
+(298, 152):1341
+(325, 329):1342
+(274, 276):1343
+(547, 264):1344
+(317, 288):1345
+(780, 419):1346
+(285, 381):1347
+(264, 335):1348
+(442, 430):1349
+(291, 596):1350
+(283, 328):1351
+(282, 346):1352
+(362, 265):1353
+(299, 277):1354
+(292, 307):1355
+(323, 263):1356
+(290, 271):1357
+(417, 281):1358
+(432, 358):1359
+(289, 425):1360
+(295, 751):1361
+(341, 270):1362
+(357, 273):1363
+(264, 345):1364
+(289, 328):1365
+(749, 830):1366
+(1116, 855):1367
+(339, 305):1368
+(264, 655):1369
+(264, 266):1370
+(264, 354):1371
+(333, 185):1372
+(291, 451):1373
+(262, 152):1374
+(322, 364):1375
+(325, 263):1376
+(306, 535):1377
+(299, 324):1378
+(349, 271):1379
+(349, 573):1380
+(1176, 297):1381
+(302, 571):1382
+(300, 320):1383
+(269, 327):1384
+(372, 363):1385
+(467, 276):1386
+(332, 265):1387
+(279, 400):1388
+(317, 425):1389
+(306, 591):1390
+(396, 288):1391
+(399, 273):1392
+(370, 290):1393
+(287, 514):1394
+(322, 276):1395
+(295, 868):1396
+(343, 415):1397
+(287, 322):1398
+(404, 272):1399
+(675, 273):1400
+(295, 261):1401
+(303, 539):1402
+(482, 374):1403
+(338, 264):1404
+(286, 269):1405
+(258, 278):1406
+(300, 368):1407
+(275, 570):1408
+(639, 269):1409
+(279, 779):1410
+(340, 273):1411
+(822, 267):1412
+(265, 337):1413
+(288, 474):1414
+(908, 272):1415
+(289, 362):1416
+(299, 526):1417
+(282, 335):1418
+(287, 294):1419
+(317, 276):1420
+(274, 342):1421
+(332, 284):1422
+(535, 173):1423
+(804, 905):1424
+(370, 310):1425
+(469, 283):1426
+(945, 271):1427
+(341, 364):1428
+(319, 566):1429
+(350, 264):1430
+(289, 697):1431
+(1154, 913):1432
+(279, 265):1433
+(328, 530):1434
+(766, 311):1435
+(293, 297):1436
+(397, 269):1437
+(287, 1341):1438
+(293, 487):1439
+(605, 277):1440
+(327, 264):1441
+(353, 1309):1442
+(267, 271):1443
+(295, 302):1444
+(323, 266):1445
+(271, 307):1446
+(307, 277):1447
+(953, 320):1448
+(269, 285):1449
+(275, 344):1450
+(301, 347):1451
+(396, 1282):1452
+(340, 488):1453
+(264, 402):1454
+(288, 344):1455
+(296, 452):1456
+(333, 173):1457
+(275, 359):1458
+(293, 276):1459
+(264, 551):1460
+(816, 394):1461
+(285, 754):1462
+(629, 340):1463
+(1091, 320):1464
+(282, 310):1465
+(504, 497):1466
+(401, 694):1467
+(295, 300):1468
+(316, 597):1469
+(640, 870):1470
+(295, 733):1471
+(307, 453):1472
+(309, 933):1473
+(338, 624):1474
+(306, 862):1475
+(293, 377):1476
+(1080, 327):1477
+(417, 278):1478
+(315, 445):1479
+(378, 369):1480
+(295, 528):1481
+(303, 265):1482
+(294, 264):1483
+(422, 292):1484
+(1391, 544):1485
+(325, 345):1486
+(340, 379):1487
+(385, 316):1488
+(423, 263):1489
+(323, 312):1490
+(355, 302):1491
+(317, 472):1492
+(525, 281):1493
+(307, 273):1494
+(306, 589):1495
+(397, 272):1496
+(303, 861):1497
+(280, 175):1498
+(350, 424):1499
+(265, 276):1500
+(297, 500):1501
+(271, 369):1502
+(339, 273):1503
+(317, 596):1504
+(523, 273):1505
+(282, 813):1506
+(380, 316):1507
+(327, 573):1508
+(338, 288):1509
+(327, 453):1510
+(349, 272):1511
+(302, 383):1512
+(604, 331):1513
+(286, 364):1514
+(326, 861):1515
+(296, 877):1516
+(275, 346):1517
+(409, 272):1518
+(314, 1028):1519
+(332, 287):1520
+(264, 264):1521
+(290, 330):1522
+(758, 270):1523
+(338, 451):1524
+(765, 302):1525
+(440, 402):1526
+(290, 283):1527
+(410, 264):1528
+(440, 267):1529
+(289, 585):1530
+(267, 651):1531
+(279, 383):1532
+(309, 877):1533
+(289, 294):1534
+(716, 1085):1535
+(474, 261):1536
+(320, 789):1537
+(389, 519):1538
+(270, 505):1539
+(476, 285):1540
+(1127, 342):1541
+(543, 414):1542
+(314, 376):1543
+(339, 474):1544
+(264, 775):1545
+(282, 453):1546
+(269, 374):1547
+(404, 258):1548
+(784, 296):1549
+(270, 263):1550
+(399, 479):1551
+(294, 1195):1552
+(293, 291):1553
+(293, 601):1554
+(399, 269):1555
+(320, 572):1556
+(423, 999):1557
+(293, 813):1558
+(32, 39):1559
+(668, 708):1560
+(396, 622):1561
+(1311, 285):1562
+(302, 270):1563
+(337, 283):1564
+(355, 269):1565
+(275, 377):1566
+(563, 166):1567
+(461, 281):1568
+(306, 449):1569
+(415, 269):1570
+(322, 637):1571
+(1332, 501):1572
+(265, 344):1573
+(1122, 270):1574
+(631, 304):1575
+(300, 325):1576
+(409, 382):1577
+(459, 490):1578
+(299, 266):1579
+(301, 719):1580
+(279, 309):1581
+(275, 341):1582
+(314, 1394):1583
+(258, 281):1584
+(301, 307):1585
+(499, 939):1586
+(278, 257):1587
+(575, 507):1588
+(636, 800):1589
+(377, 487):1590
+(473, 567):1591
+(288, 864):1592
+(274, 287):1593
+(409, 427):1594
+(843, 261):1595
+(310, 277):1596
+(502, 271):1597
+(410, 271):1598
+(279, 565):1599
+(293, 807):1600
+(269, 330):1601
+(504, 352):1602
+(275, 284):1603
+(392, 336):1604
+(323, 287):1605
+(897, 358):1606
+(584, 1512):1607
+(278, 281):1608
+(731, 520):1609
+(293, 406):1610
+(380, 297):1611
+(420, 271):1612
+(1468, 1469):1613
+(306, 1516):1614
+(274, 385):1615
+(539, 427):1616
+(353, 382):1617
+(280, 569):1618
+(482, 478):1619
+(287, 281):1620
+(921, 367):1621
+(438, 935):1622
+(275, 290):1623
+(314, 283):1624
+(466, 394):1625
+(1301, 320):1626
+(592, 344):1627
+(343, 1067):1628
+(322, 352):1629
+(314, 681):1630
+(568, 446):1631
+(274, 331):1632
+(306, 950):1633
+(264, 312):1634
+(265, 364):1635
+(294, 344):1636
+(291, 270):1637
+(370, 1076):1638
+(268, 165):1639
+(279, 387):1640
+(350, 265):1641
+(276, 403):1642
+(290, 275):1643
+(295, 938):1644
+(271, 406):1645
+(831, 450):1646
+(275, 492):1647
+(271, 528):1648
+(275, 274):1649
+(336, 267):1650
+(392, 1063):1651
+(335, 810):1652
+(446, 269):1653
+(293, 320):1654
+(349, 291):1655
+(323, 305):1656
+(291, 403):1657
+(416, 621):1658
+(306, 758):1659
+(355, 273):1660
+(301, 266):1661
+(374, 383):1662
+(274, 266):1663
+(1365, 272):1664
+(452, 272):1665
+(1414, 846):1666
+(322, 761):1667
+(315, 311):1668
+(770, 676):1669
+(279, 264):1670
+(306, 292):1671
+(294, 802):1672
+(558, 1223):1673
+(264, 433):1674
+(506, 472):1675
+(294, 426):1676
+(267, 371):1677
+(722, 281):1678
+(301, 269):1679
+(465, 265):1680
+(294, 334):1681
+(339, 285):1682
+(1476, 1621):1683
+(1374, 272):1684
+(361, 462):1685
+(1051, 401):1686
+(279, 410):1687
+(580, 285):1688
+(279, 1181):1689
+(317, 364):1690
+(669, 455):1691
+(279, 1484):1692
+(459, 536):1693
+(1003, 335):1694
+(547, 744):1695
+(378, 400):1696
+(415, 376):1697
+(851, 359):1698
+(314, 639):1699
+(673, 269):1700
+(265, 366):1701
+(1212, 304):1702
+(445, 320):1703
+(481, 322):1704
+(474, 269):1705
+(803, 278):1706
+(485, 273):1707
+(310, 261):1708
+(267, 345):1709
+(338, 281):1710
+(286, 518):1711
+(273, 294):1712
+(400, 272):1713
+(1452, 267):1714
+(392, 478):1715
+(287, 425):1716
+(314, 1252):1717
+(319, 544):1718
+(896, 352):1719
+(385, 811):1720
+(367, 339):1721
+(267, 337):1722
+(465, 327):1723
+(485, 265):1724
+(315, 372):1725
+(956, 399):1726
+(279, 1018):1727
+(481, 414):1728
+(354, 276):1729
+(959, 404):1730
+(303, 376):1731
+(380, 292):1732
+(270, 498):1733
+(275, 340):1734
+(326, 539):1735
+(818, 519):1736
+(293, 283):1737
+(265, 1056):1738
+(482, 741):1739
+(296, 328):1740
+(319, 284):1741
+(315, 382):1742
+(314, 941):1743
+(438, 622):1744
+(368, 261):1745
+(729, 622):1746
+(946, 517):1747
+(343, 605):1748
+(920, 327):1749
+(306, 403):1750
+(297, 1258):1751
+(271, 448):1752
+(869, 273):1753
+(260, 182):1754
+(260, 185):1755
+(279, 266):1756
+(338, 332):1757
+(298, 154):1758
+(302, 581):1759
+(1292, 1293):1760
+(288, 522):1761
+(332, 269):1762
+(271, 339):1763
+(1297, 273):1764
+(265, 284):1765
+(308, 271):1766
+(340, 261):1767
+(299, 610):1768
+(293, 1574):1769
+(420, 265):1770
+(549, 383):1771
+(470, 273):1772
+(396, 486):1773
+(625, 281):1774
+(289, 376):1775
+(275, 326):1776
+(647, 168):1777
+(370, 430):1778
+(378, 797):1779
+(341, 307):1780
+(298, 173):1781
+(465, 284):1782
+(703, 311):1783
+(309, 972):1784
+(293, 330):1785
+(285, 462):1786
+(267, 465):1787
+(370, 339):1788
+(289, 545):1789
+(275, 261):1790
+(279, 588):1791
+(317, 323):1792
+(275, 545):1793
+(792, 266):1794
+(499, 475):1795
+(301, 291):1796
+(265, 269):1797
+(300, 390):1798
+(469, 433):1799
+(286, 975):1800
+(572, 275):1801
+(287, 575):1802
+(350, 288):1803
+(315, 346):1804
+(290, 348):1805
+(265, 334):1806
+(440, 345):1807
+(296, 931):1808
+(327, 607):1809
+(546, 538):1810
+(337, 312):1811
+(647, 151):1812
+(1419, 540):1813
+(286, 1074):1814
+(338, 744):1815
+(432, 328):1816
+(307, 530):1817
+(293, 400):1818
+(346, 272):1819
+(272, 292):1820
+(354, 261):1821
+(323, 273):1822
+(288, 290):1823
+(293, 1538):1824
+(487, 273):1825
+(264, 518):1826
+(1261, 266):1827
+(1168, 320):1828
+(314, 352):1829
+(1463, 335):1830
+(391, 618):1831
+(1418, 304):1832
+(919, 1093):1833
+(349, 332):1834
+(276, 264):1835
+(270, 261):1836
+(646, 285):1837
+(1321, 520):1838
+(275, 272):1839
+(341, 553):1840
+(299, 529):1841
+(285, 483):1842
+(450, 355):1843
+(494, 261):1844
+(353, 355):1845
+(747, 311):1846
+(287, 475):1847
+(1268, 570):1848
+(286, 307):1849
+(314, 1155):1850
+(265, 287):1851
+(643, 271):1852
+(1180, 305):1853
+(408, 603):1854
+(578, 264):1855
+(271, 497):1856
+(347, 266):1857
+(306, 1438):1858
+(319, 500):1859
+(1068, 263):1860
+(308, 797):1861
+(752, 305):1862
+(385, 475):1863
+(830, 299):1864
+(756, 274):1865
+(385, 1489):1866
+(279, 287):1867
+(303, 593):1868
+(873, 418):1869
+(270, 265):1870
+(494, 273):1871
+(300, 264):1872
+(267, 575):1873
+(297, 345):1874
+(310, 269):1875
+(353, 322):1876
+(296, 285):1877
+(270, 367):1878
+(370, 1189):1879
+(605, 489):1880
+(376, 267):1881
+(535, 168):1882
+(364, 336):1883
+(315, 377):1884
+(301, 394):1885
+(593, 507):1886
+(286, 930):1887
+(289, 521):1888
+(271, 794):1889
+(319, 556):1890
+(362, 388):1891
+(657, 567):1892
+(352, 449):1893
+(477, 273):1894
+(346, 339):1895
+(346, 357):1896
+(294, 265):1897
+(666, 278):1898
+(1163, 273):1899
+(1285, 579):1900
+(287, 356):1901
+(282, 912):1902
+(343, 377):1903
+(280, 724):1904
+(293, 483):1905
+(293, 1464):1906
+(1198, 344):1907
+(1607, 611):1908
+(1367, 278):1909
+(573, 845):1910
+(274, 779):1911
+(500, 401):1912
+(272, 276):1913
+(694, 437):1914
+(546, 616):1915
+(265, 302):1916
+(270, 288):1917
+(722, 278):1918
+(270, 266):1919
+(647, 173):1920
+(420, 1227):1921
+(581, 261):1922
+(323, 270):1923
+(549, 934):1924
+(294, 312):1925
+(279, 354):1926
+(377, 462):1927
+(295, 309):1928
+(590, 480):1929
+(1432, 261):1930
+(346, 299):1931
+(262, 393):1932
+(1124, 264):1933
+(387, 313):1934
+(322, 407):1935
+(470, 352):1936
+(372, 302):1937
+(287, 933):1938
+(576, 276):1939
+(1773, 553):1940
+(325, 725):1941
+(820, 356):1942
+(445, 261):1943
+(295, 291):1944
+(647, 166):1945
+(684, 265):1946
+(270, 415):1947
+(1105, 305):1948
+(338, 275):1949
+(167, 257):1950
+(308, 369):1951
+(800, 334):1952
+(279, 270):1953
+(271, 381):1954
+(506, 1784):1955
+(331, 531):1956
+(370, 288):1957
+(521, 375):1958
+(1379, 963):1959
+(337, 345):1960
+(327, 311):1961
+(333, 151):1962
+(673, 419):1963
+(590, 382):1964
+(264, 501):1965
+(440, 263):1966
+(562, 425):1967
+(343, 323):1968
+(289, 275):1969
+(374, 367):1970
+(316, 394):1971
+(485, 267):1972
+(322, 516):1973
+(663, 267):1974
+(306, 554):1975
+(584, 372):1976
+(966, 281):1977
+(541, 274):1978
+(731, 567):1979
+(295, 603):1980
+(279, 817):1981
+(315, 827):1982
+(469, 529):1983
+(315, 492):1984
+(599, 283):1985
+(750, 515):1986
+(1075, 267):1987
+(276, 545):1988
+(669, 346):1989
+(291, 433):1990
+(343, 764):1991
+(267, 265):1992
+(275, 347):1993
+(270, 330):1994
+(153, 312):1995
+(262, 162):1996
+(1139, 753):1997
+(343, 302):1998
+(286, 335):1999
+(285, 449):2000
+(271, 273):2001
+(319, 1381):2002
+(338, 325):2003
+(265, 453):2004
+(297, 342):2005
+(319, 609):2006
+(335, 514):2007
+(276, 337):2008
+(384, 845):2009
+(314, 320):2010
+(673, 311):2011
+(1071, 915):2012
+(1665, 653):2013
+(293, 597):2014
+(1558, 275):2015
+(266, 332):2016
+(749, 335):2017
+(726, 664):2018
+(954, 560):2019
+(434, 683):2020
+(276, 1039):2021
+(315, 326):2022
+(271, 382):2023
+(596, 273):2024
+(260, 171):2025
+(481, 264):2026
+(716, 501):2027
+(272, 328):2028
+(563, 173):2029
+(301, 364):2030
+(292, 267):2031
+(343, 642):2032
+(505, 774):2033
+(341, 265):2034
+(326, 307):2035
+(294, 1028):2036
+(286, 302):2037
+(700, 269):2038
+(631, 264):2039
+(434, 1050):2040
+(656, 278):2041
+(669, 999):2042
+(552, 903):2043
+(297, 381):2044
+(911, 334):2045
+(292, 264):2046
+(389, 431):2047
+(282, 1328):2048
+(314, 1383):2049
+(982, 166):2050
+(280, 161):2051
+(314, 802):2052
+(303, 738):2053
+(578, 304):2054
+(377, 307):2055
+(879, 425):2056
+(477, 1330):2057
+(405, 381):2058
+(319, 329):2059
+(646, 492):2060
+(476, 360):2061
+(301, 1002):2062
+(618, 414):2063
+(710, 361):2064
+(969, 312):2065
+(384, 1392):2066
+(346, 294):2067
+(280, 152):2068
+(300, 270):2069
+(271, 350):2070
+(264, 480):2071
+(315, 356):2072
+(1073, 672):2073
+(279, 382):2074
+(293, 1133):2075
+(473, 278):2076
+(966, 278):2077
+(772, 895):2078
+(894, 334):2079
+(306, 1440):2080
+(505, 655):2081
+(314, 542):2082
+(695, 483):2083
+(307, 733):2084
+(316, 267):2085
+(333, 178):2086
+(295, 336):2087
+(682, 404):2088
+(634, 273):2089
+(448, 291):2090
+(613, 1214):2091
+(493, 281):2092
+(748, 320):2093
+(291, 261):2094
+(809, 271):2095
+(286, 865):2096
+(290, 424):2097
+(378, 323):2098
+(839, 284):2099
+(576, 274):2100
+(275, 382):2101
+(546, 350):2102
+(301, 1057):2103
+(289, 382):2104
+(400, 277):2105
+(288, 310):2106
+(264, 725):2107
+(292, 373):2108
+(749, 712):2109
+(924, 346):2110
+(1499, 269):2111
+(1518, 677):2112
+(338, 278):2113
+(259, 183):2114
+(275, 437):2115
+(474, 376):2116
+(349, 275):2117
+(316, 273):2118
+(795, 275):2119
+(928, 273):2120
+(270, 686):2121
+(290, 288):2122
+(360, 358):2123
+(390, 261):2124
+(343, 339):2125
+(502, 311):2126
+(1251, 522):2127
+(445, 273):2128
+(486, 591):2129
+(1754, 283):2130
+(657, 1555):2131
+(421, 336):2132
+(588, 269):2133
+(322, 655):2134
+(314, 485):2135
+(1006, 1032):2136
+(735, 364):2137
+(317, 686):2138
+(341, 1351):2139
+(349, 1652):2140
+(271, 294):2141
+(392, 464):2142
+(1492, 270):2143
+(343, 791):2144
+(395, 1039):2145
+(1912, 354):2146
+(1256, 629):2147
+(1670, 515):2148
+(1837, 1042):2149
+(722, 366):2150
+(1470, 1225):2151
+(285, 527):2152
+(824, 555):2153
+(291, 271):2154
+(995, 276):2155
+(1078, 284):2156
+(295, 339):2157
+(328, 276):2158
+(568, 1294):2159
+(1061, 1333):2160
+(286, 775):2161
+(659, 425):2162
+(772, 390):2163
+(279, 1682):2164
+(339, 261):2165
+(445, 375):2166
+(274, 412):2167
+(319, 1175):2168
+(280, 159):2169
+(275, 505):2170
+(410, 354):2171
+(617, 261):2172
+(306, 565):2173
+(327, 1081):2174
+(748, 265):2175
+(288, 365):2176
+(288, 330):2177
+(289, 433):2178
+(314, 309):2179
+(541, 1634):2180
+(1579, 366):2181
+(817, 292):2182
+(2182, 1304):2183
+(377, 419):2184
+(297, 1083):2185
+(269, 329):2186
+(301, 327):2187
+(332, 614):2188
+(303, 437):2189
+(331, 276):2190
+(878, 507):2191
+(434, 390):2192
+(690, 618):2193
+(936, 579):2194
+(279, 408):2195
+(275, 483):2196
+(729, 285):2197
+(722, 373):2198
+(1152, 414):2199
+(338, 554):2200
+(660, 269):2201
+(270, 307):2202
+(1791, 624):2203
+(411, 1260):2204
+(274, 284):2205
+(308, 617):2206
+(325, 330):2207
+(303, 703):2208
+(769, 329):2209
+(1035, 334):2210
+(264, 457):2211
+(404, 269):2212
+(514, 354):2213
+(168, 257):2214
+(297, 320):2215
+(418, 528):2216
+(647, 185):2217
+(497, 281):2218
+(270, 347):2219
+(839, 276):2220
+(459, 683):2221
+(694, 507):2222
+(271, 271):2223
+(396, 270):2224
+(502, 405):2225
+(339, 311):2226
+(385, 608):2227
+(299, 498):2228
+(308, 2116):2229
+(285, 422):2230
+(394, 667):2231
+(282, 1801):2232
+(386, 324):2233
+(1012, 273):2234
+(326, 753):2235
+(532, 708):2236
+(327, 312):2237
+(282, 1283):2238
+(391, 419):2239
+(572, 374):2240
+(272, 414):2241
+(271, 633):2242
+(385, 270):2243
+(288, 490):2244
+(1553, 362):2245
+(287, 326):2246
+(291, 950):2247
+(440, 266):2248
+(879, 449):2249
+(1310, 446):2250
+(292, 271):2251
+(392, 451):2252
+(364, 269):2253
+(340, 305):2254
+(325, 316):2255
+(1069, 273):2256
+(290, 276):2257
+(335, 285):2258
+(580, 348):2259
+(303, 827):2260
+(263, 281):2261
+(274, 334):2262
+(504, 269):2263
+(315, 536):2264
+(306, 723):2265
+(269, 652):2266
+(378, 512):2267
+(592, 334):2268
+(293, 1716):2269
+(290, 544):2270
+(286, 337):2271
+(571, 347):2272
+(937, 374):2273
+(535, 151):2274
+(293, 2016):2275
+(2142, 622):2276
+(1110, 276):2277
+(270, 410):2278
+(314, 459):2279
+(1906, 267):2280
+(477, 275):2281
+(1561, 278):2282
+(264, 330):2283
+(1564, 377):2284
+(1095, 773):2285
+(314, 462):2286
+(297, 470):2287
+(378, 312):2288
+(695, 328):2289
+(434, 271):2290
+(295, 1227):2291
+(648, 284):2292
+(292, 299):2293
+(338, 299):2294
+(285, 261):2295
+(415, 505):2296
+(346, 348):2297
+(511, 287):2298
+(1477, 330):2299
+(825, 267):2300
+(322, 272):2301
+(912, 609):2302
+(256, 139):2303
+(349, 588):2304
+(365, 585):2305
+(276, 948):2306
+(308, 514):2307
+(493, 514):2308
+(898, 292):2309
+(287, 436):2310
+(370, 320):2311
+(1265, 334):2312
+(482, 1126):2313
+(269, 365):2314
+(1629, 773):2315
+(784, 261):2316
+(306, 291):2317
+(299, 305):2318
+(173, 841):2319
+(327, 348):2320
+(406, 267):2321
+(315, 341):2322
+(389, 269):2323
+(947, 354):2324
+(298, 168):2325
+(270, 278):2326
+(271, 329):2327
+(1613, 287):2328
+(295, 437):2329
+(399, 312):2330
+(386, 261):2331
+(772, 356):2332
+(671, 261):2333
+(314, 368):2334
+(1397, 410):2335
+(438, 294):2336
+(1270, 348):2337
+(294, 323):2338
+(471, 287):2339
+(1490, 263):2340
+(459, 356):2341
+(295, 288):2342
+(1867, 360):2343
+(330, 379):2344
+(292, 325):2345
+(434, 895):2346
+(380, 271):2347
+(969, 310):2348
+(287, 311):2349
+(306, 1016):2350
+(264, 529):2351
+(280, 645):2352
+(637, 383):2353
+(872, 261):2354
+(575, 356):2355
+(1033, 276):2356
+(326, 352):2357
+(537, 693):2358
+(2093, 266):2359
+(309, 328):2360
+(327, 302):2361
+(770, 556):2362
+(408, 261):2363
+(1102, 433):2364
+(1007, 305):2365
+(293, 365):2366
+(386, 290):2367
+(346, 512):2368
+(1131, 1022):2369
+(296, 643):2370
+(1180, 266):2371
+(378, 382):2372
+(1008, 311):2373
+(364, 397):2374
+(262, 161):2375
+(264, 324):2376
+(386, 975):2377
+(293, 423):2378
+(269, 324):2379
+(290, 400):2380
+(656, 281):2381
+(603, 462):2382
+(1583, 1089):2383
+(291, 746):2384
+(299, 335):2385
+(572, 426):2386
+(1717, 766):2387
+(317, 410):2388
+(306, 1551):2389
+(638, 275):2390
+(297, 460):2391
+(1011, 272):2392
+(292, 270):2393
+(893, 462):2394
+(338, 1443):2395
+(275, 1263):2396
+(385, 533):2397
+(340, 610):2398
+(361, 270):2399
+(347, 882):2400
+(380, 275):2401
+(282, 587):2402
+(288, 371):2403
+(343, 556):2404
+(1953, 310):2405
+(307, 291):2406
+(953, 757):2407
+(279, 274):2408
+(270, 281):2409
+(937, 2246):2410
+(1051, 1210):2411
+(729, 341):2412
+(1246, 292):2413
+(289, 307):2414
+(274, 498):2415
+(286, 359):2416
+(476, 264):2417
+(301, 701):2418
+(293, 1323):2419
+(376, 442):2420
+(332, 335):2421
+(405, 269):2422
+(300, 464):2423
+(1260, 517):2424
+(1173, 273):2425
+(269, 337):2426
+(910, 269):2427
+(386, 2039):2428
+(349, 601):2429
+(521, 273):2430
+(1299, 287):2431
+(279, 302):2432
+(698, 1684):2433
+(726, 299):2434
+(784, 265):2435
+(301, 361):2436
+(341, 538):2437
+(309, 417):2438
+(283, 394):2439
+(294, 376):2440
+(285, 324):2441
+(376, 311):2442
+(322, 376):2443
+(396, 274):2444
+(275, 678):2445
+(306, 588):2446
+(275, 278):2447
+(343, 1896):2448
+(275, 334):2449
+(1334, 464):2450
+(404, 527):2451
+(353, 1910):2452
+(295, 1307):2453
+(646, 1746):2454
+(400, 708):2455
+(496, 297):2456
+(422, 294):2457
+(320, 1562):2458
+(319, 261):2459
+(566, 261):2460
+(434, 292):2461
+(295, 624):2462
+(264, 930):2463
+(274, 320):2464
+(492, 267):2465
+(1427, 366):2466
+(301, 346):2467
+(364, 270):2468
+(1756, 438):2469
+(291, 460):2470
+(1571, 728):2471
+(299, 551):2472
+(1757, 526):2473
+(450, 269):2474
+(364, 513):2475
+(314, 597):2476
+(264, 365):2477
+(648, 334):2478
+(388, 672):2479
+(319, 337):2480
+(1174, 354):2481
+(396, 341):2482
+(456, 490):2483
+(2038, 412):2484
+(936, 576):2485
+(295, 265):2486
+(765, 312):2487
+(1099, 1759):2488
+(325, 454):2489
+(257, 169):2490
+(1928, 736):2491
+(350, 273):2492
+(286, 807):2493
+(1093, 273):2494
+(286, 276):2495
+(347, 479):2496
+(648, 329):2497
+(301, 1202):2498
+(924, 470):2499
+(514, 310):2500
+(295, 403):2501
+(486, 269):2502
+(275, 425):2503
+(279, 486):2504
+(309, 292):2505
+(270, 361):2506
+(279, 285):2507
+(541, 294):2508
+(269, 448):2509
+(290, 553):2510
+(440, 283):2511
+(438, 341):2512
+(595, 284):2513
+(663, 503):2514
+(1575, 311):2515
+(824, 299):2516
+(820, 683):2517
+(638, 273):2518
+(372, 269):2519
+(485, 284):2520
+(303, 307):2521
+(415, 407):2522
+(659, 449):2523
+(327, 452):2524
+(2075, 288):2525
+(1402, 267):2526
+(267, 970):2527
+(288, 402):2528
+(380, 294):2529
+(301, 562):2530
+(286, 465):2531
+(396, 1616):2532
+(403, 725):2533
+(735, 290):2534
+(941, 949):2535
+(915, 277):2536
+(286, 270):2537
+(343, 1703):2538
+(279, 508):2539
+(269, 264):2540
+(1033, 284):2541
+(1405, 1958):2542
+(401, 921):2543
+(317, 337):2544
+(809, 273):2545
+(301, 309):2546
+(824, 842):2547
+(292, 261):2548
+(306, 269):2549
+(1792, 479):2550
+(376, 509):2551
+(320, 771):2552
+(340, 551):2553
+(2095, 439):2554
+(863, 297):2555
+(288, 316):2556
+(282, 316):2557
+(299, 283):2558
+(595, 276):2559
+(2088, 294):2560
+(822, 273):2561
+(292, 623):2562
+(328, 290):2563
+(406, 273):2564
+(306, 1798):2565
+(524, 316):2566
+(1373, 375):2567
+(443, 152):2568
+(628, 273):2569
+(302, 352):2570
+(271, 265):2571
+(1139, 404):2572
+(563, 152):2573
+(1078, 276):2574
+(1230, 714):2575
+(392, 1130):2576
+(858, 273):2577
+(460, 478):2578
+(836, 273):2579
+(326, 448):2580
+(453, 273):2581
+(780, 269):2582
+(391, 621):2583
+(479, 591):2584
+(391, 270):2585
+(1788, 692):2586
+(325, 913):2587
+(267, 327):2588
+(386, 881):2589
+(485, 285):2590
+(291, 294):2591
+(314, 372):2592
+(1146, 331):2593
+(295, 428):2594
+(338, 436):2595
+(2204, 267):2596
+(332, 796):2597
+(1158, 751):2598
+(471, 261):2599
+(395, 403):2600
+(309, 311):2601
+(385, 1104):2602
+(900, 526):2603
+(290, 497):2604
+(1065, 1016):2605
+(275, 736):2606
+(314, 270):2607
+(757, 707):2608
+(659, 1893):2609
+(305, 294):2610
+(2125, 294):2611
+(270, 413):2612
+(513, 701):2613
+(1387, 874):2614
+(613, 1536):2615
+(517, 264):2616
+(1599, 269):2617
+(297, 462):2618
+(288, 427):2619
+(400, 265):2620
+(428, 273):2621
+(309, 326):2622
+(396, 291):2623
+(319, 385):2624
+(319, 328):2625
+(262, 1995):2626
+(2626, 528):2627
+(381, 287):2628
+(432, 281):2629
+(267, 360):2630
+(828, 394):2631
+(477, 424):2632
+(471, 458):2633
+(538, 348):2634
+(613, 472):2635
+(380, 274):2636
+(471, 1156):2637
+(319, 423):2638
+(806, 273):2639
+(317, 1108):2640
+(314, 508):2641
+(300, 337):2642
+(505, 407):2643
+(482, 414):2644
+(435, 1286):2645
+(302, 528):2646
+(325, 310):2647
+(343, 410):2648
+(351, 183):2649
+(538, 1820):2650
+(315, 707):2651
+(440, 460):2652
+(408, 348):2653
+(385, 329):2654
+(411, 1134):2655
+(1192, 553):2656
+(578, 285):2657
+(330, 273):2658
+(306, 332):2659
+(308, 405):2660
+(399, 283):2661
+(493, 358):2662
+(267, 284):2663
+(311, 519):2664
+(395, 326):2665
+(271, 336):2666
+(264, 421):2667
+(341, 336):2668
+(283, 501):2669
+(335, 718):2670
+(378, 571):2671
+(314, 771):2672
+(1604, 1249):2673
+(346, 288):2674
+(282, 1025):2675
+(2675, 475):2676
+(314, 989):2677
+(1253, 2019):2678
+(264, 1560):2679
+(482, 548):2680
+(945, 542):2681
+(450, 621):2682
+(568, 468):2683
+(1335, 288):2684
+(295, 718):2685
+(834, 278):2686
+(943, 276):2687
+(1850, 591):2688
+(1511, 1878):2689
+(411, 1562):2690
+(303, 446):2691
+(959, 753):2692
+(1043, 273):2693
+(471, 263):2694
+(392, 307):2695
+(698, 274):2696
+(256, 155):2697
+(310, 526):2698
+(297, 299):2699
+(465, 341):2700
+(282, 1922):2701
+(1426, 326):2702
+(510, 339):2703
+(1264, 375):2704
+(317, 504):2705
+(550, 320):2706
+(338, 406):2707
+(295, 264):2708
+(1331, 283):2709
+(1230, 689):2710
+(1619, 290):2711
+(1029, 261):2712
+(922, 269):2713
+(1377, 167):2714
+(428, 284):2715
+(328, 381):2716
+(404, 797):2717
+(291, 529):2718
+(309, 622):2719
+(396, 292):2720
+(746, 311):2721
+(322, 336):2722
+(900, 1231):2723
+(669, 672):2724
+(516, 390):2725
+(346, 719):2726
+(1064, 284):2727
+(1686, 359):2728
+(289, 787):2729
+(1526, 278):2730
+(1523, 457):2731
+(674, 299):2732
+(2307, 1854):2733
+(306, 401):2734
+(377, 361):2735
+(315, 935):2736
+(264, 413):2737
+(265, 457):2738
+(302, 273):2739
+(286, 1937):2740
+(290, 686):2741
+(494, 287):2742
+(1673, 287):2743
+(769, 273):2744
+(272, 374):2745
+(349, 643):2746
+(257, 2319):2747
+(395, 983):2748
+(282, 294):2749
+(271, 500):2750
+(948, 336):2751
+(690, 458):2752
+(1580, 1681):2753
+(492, 273):2754
+(294, 273):2755
+(644, 334):2756
+(1393, 285):2757
+(626, 2067):2758
+(294, 1864):2759
+(315, 1274):2760
+(485, 283):2761
+(538, 1081):2762
+(271, 291):2763
+(411, 322):2764
+(593, 356):2765
+(325, 269):2766
+(339, 324):2767
+(703, 269):2768
+(288, 264):2769
+(578, 2073):2770
+(301, 1399):2771
+(420, 302):2772
+(265, 328):2773
+(1638, 375):2774
+(851, 417):2775
+(264, 275):2776
+(1390, 1783):2777
+(306, 1435):2778
+(1526, 281):2779
+(2534, 574):2780
+(1485, 498):2781
+(271, 621):2782
+(286, 545):2783
+(1161, 294):2784
+(467, 463):2785
+(435, 270):2786
+(329, 593):2787
+(288, 329):2788
+(319, 273):2789
+(1871, 291):2790
+(510, 267):2791
+(326, 895):2792
+(616, 430):2793
+(1112, 1353):2794
+(2543, 292):2795
+(418, 1354):2796
+(276, 327):2797
+(279, 322):2798
+(514, 589):2799
+(511, 334):2800
+(460, 311):2801
+(289, 475):2802
+(449, 312):2803
+(758, 697):2804
+(262, 129):2805
+(448, 526):2806
+(301, 2302):2807
+(300, 1109):2808
+(826, 305):2809
+(330, 394):2810
+(499, 986):2811
+(270, 324):2812
+(628, 330):2813
+(659, 397):2814
+(285, 1133):2815
+(287, 1298):2816
+(595, 329):2817
+(418, 1406):2818
+(275, 537):2819
+(158, 388):2820
+(485, 263):2821
+(666, 281):2822
+(385, 297):2823
+(1635, 311):2824
+(270, 693):2825
+(872, 276):2826
+(1451, 383):2827
+(276, 320):2828
+(257, 171):2829
+(275, 498):2830
+(353, 2578):2831
+(820, 536):2832
+(1251, 676):2833
+(271, 651):2834
+(591, 320):2835
+(293, 860):2836
+(275, 753):2837
+(343, 436):2838
+(391, 470):2839
+(264, 373):2840
+(818, 288):2841
+(633, 273):2842
+(285, 366):2843
+(671, 273):2844
+(275, 421):2845
+(557, 307):2846
+(415, 516):2847
+(319, 362):2848
+(307, 271):2849
+(265, 2051):2850
+(946, 323):2851
+(2851, 291):2852
+(365, 617):2853
+(319, 472):2854
+(843, 596):2855
+(288, 299):2856
+(301, 276):2857
+(346, 596):2858
+(259, 138):2859
+(1598, 522):2860
+(951, 1136):2861
+(340, 284):2862
+(411, 1811):2863
+(279, 474):2864
+(1725, 702):2865
+(1197, 374):2866
+(346, 270):2867
+(847, 292):2868
+(301, 415):2869
+(322, 265):2870
+(291, 591):2871
+(721, 273):2872
+(265, 261):2873
+(282, 2047):2874
+(1453, 414):2875
+(275, 376):2876
+(435, 500):2877
+(856, 274):2878
+(305, 274):2879
+(1729, 356):2880
+(325, 439):2881
+(467, 339):2882
+(392, 721):2883
+(558, 278):2884
+(264, 352):2885
+(346, 702):2886
+(590, 292):2887
+(547, 288):2888
+(271, 316):2889
+(1090, 2130):2890
+(271, 297):2891
+(819, 2068):2892
+(755, 292):2893
+(538, 354):2894
+(276, 281):2895
+(1804, 460):2896
+(492, 299):2897
+(314, 294):2898
+(301, 310):2899
+(415, 275):2900
+(298, 149):2901
+(292, 624):2902
+(767, 263):2903
+(1019, 261):2904
+(1416, 261):2905
+(1165, 310):2906
+(740, 284):2907
+(293, 374):2908
+(285, 807):2909
+(800, 278):2910
+(301, 723):2911
+(647, 172):2912
+(282, 1083):2913
+(306, 460):2914
+(322, 418):2915
+(279, 576):2916
+(274, 265):2917
+(748, 327):2918
+(303, 1150):2919
+(384, 2058):2920
+(422, 274):2921
+(340, 312):2922
+(291, 589):2923
+(317, 833):2924
+(2649, 1009):2925
+(532, 448):2926
+(577, 394):2927
+(691, 261):2928
+(1038, 654):2929
+(423, 672):2930
+(1734, 361):2931
+(325, 423):2932
+(387, 487):2933
+(421, 261):2934
+(409, 292):2935
+(341, 376):2936
+(319, 288):2937
+(343, 290):2938
+(691, 276):2939
+(415, 721):2940
+(674, 633):2941
+(297, 583):2942
+(1137, 336):2943
+(293, 466):2944
+(362, 1564):2945
+(292, 366):2946
+(1041, 312):2947
+(2014, 2272):2948
+(297, 821):2949
+(2668, 1249):2950
+(1090, 313):2951
+(1327, 500):2952
+(322, 688):2953
+(332, 548):2954
+(635, 746):2955
+(347, 519):2956
+(275, 737):2957
+(1513, 276):2958
+(1092, 334):2959
+(552, 720):2960
+(322, 478):2961
+(301, 1023):2962
+(473, 835):2963
+(353, 1398):2964
+(2964, 545):2965
+(612, 261):2966
+(1193, 278):2967
+(282, 1122):2968
+(293, 437):2969
+(2969, 307):2970
+(2970, 2129):2971
+(380, 406):2972
+(296, 2684):2973
+(471, 288):2974
+(742, 468):2975
+(1747, 341):2976
+(1747, 1362):2977
+(982, 151):2978
+(338, 448):2979
+(1748, 1369):2980
+(303, 1914):2981
+(325, 305):2982
+(293, 329):2983
+(795, 276):2984
+(1146, 553):2985
+(442, 530):2986
+(270, 285):2987
+(391, 992):2988
+(341, 271):2989
+(271, 868):2990
+(487, 572):2991
+(657, 520):2992
+(322, 347):2993
+(323, 554):2994
+(1058, 2464):2995
+(289, 1226):2996
+(1257, 525):2997
+(601, 326):2998
+(690, 287):2999
+(395, 291):3000
+(1760, 273):3001
+(517, 297):3002
+(893, 1207):3003
+(478, 352):3004
+(282, 1296):3005
+(1062, 266):3006
+(322, 405):3007
+(863, 398):3008
+(302, 299):3009
+(293, 596):3010
+(595, 359):3011
+(660, 1927):3012
+(755, 417):3013
+(274, 366):3014
+(2488, 273):3015
+(340, 348):3016
+(328, 334):3017
+(1380, 2166):3018
+(320, 981):3019
+(269, 484):3020
+(299, 359):3021
+(271, 1225):3022
+(1271, 608):3023
+(470, 357):3024
+(326, 337):3025
+(617, 267):3026
+(983, 272):3027
+(306, 605):3028
+(269, 692):3029
+(578, 360):3030
+(713, 2172):3031
+(1521, 533):3032
+(1302, 334):3033
+(343, 773):3034
+(1053, 359):3035
+(894, 413):3036
+(276, 790):3037
+(178, 322):3038
+(1129, 719):3039
+(349, 617):3040
+(1531, 284):3041
+(1485, 509):3042
+(1633, 360):3043
+(326, 580):3044
+(412, 271):3045
+(2757, 287):3046
+(640, 334):3047
+(582, 330):3048
+(286, 324):3049
+(769, 276):3050
+(303, 721):3051
+(1676, 1001):3052
+(769, 330):3053
+(710, 379):3054
+(384, 299):3055
+(613, 508):3056
+(2770, 2188):3057
+(380, 608):3058
+(286, 265):3059
+(257, 1950):3060
+(288, 374):3061
+(2344, 462):3062
+(467, 1537):3063
+(438, 270):3064
+(301, 323):3065
+(648, 273):3066
+(295, 1639):3067
+(370, 292):3068
+(481, 452):3069
+(657, 276):3070
+(894, 373):3071
+(1865, 305):3072
+(340, 433):3073
+(317, 521):3074
+(543, 565):3075
+(492, 290):3076
+(353, 2353):3077
+(1221, 287):3078
+(349, 439):3079
+(293, 261):3080
+(1033, 261):3081
+(293, 271):3082
+(369, 283):3083
+(270, 291):3084
+(730, 288):3085
+(2546, 464):3086
+(639, 388):3087
+(301, 1709):3088
+(378, 605):3089
+(834, 264):3090
+(524, 274):3091
+(314, 342):3092
+(1794, 360):3093
+(873, 422):3094
+(552, 273):3095
+(289, 1588):3096
+(589, 273):3097
+(549, 512):3098
+(285, 967):3099
+(435, 292):3100
+(299, 468):3101
+(382, 264):3102
+(319, 264):3103
+(613, 1956):3104
+(2557, 263):3105
+(572, 449):3106
+(471, 276):3107
+(272, 302):3108
+(376, 498):3109
+(1413, 274):3110
+(1270, 316):3111
+(310, 383):3112
+(1315, 1162):3113
+(292, 721):3114
+(629, 341):3115
+(566, 267):3116
+(756, 276):3117
+(541, 265):3118
+(481, 291):3119
+(380, 651):3120
+(303, 382):3121
+(290, 364):3122
+(378, 265):3123
+(376, 269):3124
+(563, 151):3125
+(1277, 329):3126
+(919, 1929):3127
+(2102, 1408):3128
+(275, 310):3129
+(1065, 1440):3130
+(776, 394):3131
+(401, 512):3132
+(1349, 339):3133
+(1407, 292):3134
+(327, 717):3135
+(290, 383):3136
+(303, 581):3137
+(361, 266):3138
+(1150, 736):3139
+(274, 344):3140
+(298, 172):3141
+(1546, 269):3142
+(349, 360):3143
+(291, 709):3144
+(1651, 273):3145
+(303, 2382):3146
+(310, 273):3147
+(1010, 1790):3148
+(349, 717):3149
+(2109, 526):3150
+(420, 291):3151
+(663, 329):3152
+(566, 276):3153
+(1165, 1296):3154
+(712, 335):3155
+(1140, 261):3156
+(397, 372):3157
+(341, 267):3158
+(1337, 1455):3159
+(319, 1371):3160
+(926, 273):3161
+(1312, 1225):3162
+(332, 365):3163
+(386, 439):3164
+(575, 360):3165
+(292, 265):3166
+(282, 565):3167
+(1095, 1721):3168
+(1007, 261):3169
+(287, 1350):3170
+(1179, 419):3171
+(823, 263):3172
+(502, 299):3173
+(265, 2100):3174
+(840, 337):3175
+(307, 276):3176
+(1154, 297):3177
+(2780, 273):3178
+(856, 451):3179
+(1496, 741):3180
+(894, 457):3181
+(275, 978):3182
+(538, 311):3183
+(1072, 425):3184
+(411, 981):3185
+(767, 377):3186
+(265, 509):3187
+(400, 381):3188
+(1474, 475):3189
+(435, 608):3190
+(364, 425):3191
+(289, 442):3192
+(604, 348):3193
+(286, 522):3194
+(640, 270):3195
+(355, 287):3196
+(331, 269):3197
+(640, 284):3198
+(499, 273):3199
+(326, 404):3200
+(327, 328):3201
+(395, 837):3202
+(287, 465):3203
+(294, 540):3204
+(288, 963):3205
+(377, 336):3206
+(1232, 1084):3207
+(1816, 567):3208
+(279, 275):3209
+(477, 283):3210
+(335, 304):3211
+(784, 501):3212
+(303, 350):3213
+(1720, 522):3214
+(297, 408):3215
+(295, 401):3216
+(446, 273):3217
+(310, 331):3218
+(871, 273):3219
+(421, 277):3220
+(982, 156):3221
+(819, 266):3222
+(1506, 399):3223
+(1932, 594):3224
+(2203, 305):3225
+(572, 671):3226
+(2892, 354):3227
+(486, 597):3228
+(1247, 272):3229
+(279, 930):3230
+(264, 865):3231
+(538, 717):3232
+(550, 715):3233
+(306, 601):3234
+(481, 390):3235
+(1508, 445):3236
+(418, 359):3237
+(2616, 347):3238
+(303, 895):3239
+(795, 459):3240
+(265, 271):3241
+(492, 276):3242
+(730, 653):3243
+(636, 618):3244
+(974, 323):3245
+(294, 989):3246
+(285, 320):3247
+(541, 297):3248
+(302, 276):3249
+(285, 267):3250
+(851, 290):3251
+(849, 1291):3252
+(269, 503):3253
+(582, 276):3254
+(315, 1722):3255
+(1234, 1472):3256
+(920, 815):3257
+(470, 284):3258
+(499, 330):3259
+(288, 360):3260
+(1302, 433):3261
+(322, 462):3262
+(438, 1539):3263
+(288, 305):3264
+(804, 273):3265
+(302, 265):3266
+(911, 344):3267
+(511, 344):3268
+(265, 786):3269
+(898, 271):3270
+(308, 1371):3271
+(739, 281):3272
+(386, 329):3273
+(1388, 277):3274
+(289, 292):3275
+(301, 284):3276
+(306, 290):3277
+(1184, 273):3278
+(314, 443):3279
+(3279, 150):3280
+(279, 334):3281
+(537, 464):3282
+(1337, 1237):3283
+(909, 273):3284
+(2635, 537):3285
+(1292, 847):3286
+(3286, 588):3287
+(1064, 273):3288
+(270, 275):3289
+(513, 407):3290
+(271, 299):3291
+(341, 1210):3292
+(740, 273):3293
+(1737, 2235):3294
+(740, 2433):3295
+(563, 168):3296
+(639, 318):3297
+(942, 853):3298
+(289, 410):3299
+(304, 263):3300
+(752, 302):3301
+(299, 330):3302
+(613, 266):3303
+(396, 1441):3304
+(670, 273):3305
+(264, 350):3306
+(381, 261):3307
+(952, 2397):3308
+(386, 579):3309
+(392, 934):3310
+(308, 771):3311
+(384, 1174):3312
+(845, 527):3313
+(783, 498):3314
+(514, 290):3315
+(549, 339):3316
+(295, 311):3317
+(505, 701):3318
+(502, 297):3319
+(319, 589):3320
+(264, 2013):3321
+(602, 305):3322
+(306, 2320):3323
+(315, 316):3324
+(1038, 455):3325
+(1048, 305):3326
+(292, 288):3327
+(1165, 264):3328
+(365, 539):3329
+(2275, 1880):3330
+(386, 359):3331
+(266, 373):3332
+(449, 2669):3333
+(704, 1050):3334
+(2143, 791):3335
+(664, 676):3336
+(1058, 2954):3337
+(2117, 305):3338
+(275, 413):3339
+(1170, 269):3340
+(308, 2670):3341
+(290, 284):3342
+(456, 356):3343
+(265, 463):3344
+(524, 323):3345
+(288, 337):3346
+(1232, 354):3347
+(682, 538):3348
+(3348, 425):3349
+(1288, 330):3350
+(392, 335):3351
+(349, 261):3352
+(282, 480):3353
+(1832, 1005):3354
+(1166, 720):3355
+(314, 350):3356
+(863, 1039):3357
+(299, 440):3358
+(805, 402):3359
+(2247, 332):3360
+(317, 868):3361
+(1447, 1249):3362
+(279, 2973):3363
+(349, 2850):3364
+(265, 359):3365
+(386, 1636):3366
+(1059, 693):3367
+(590, 609):3368
+(275, 676):3369
+(1913, 294):3370
+(552, 267):3371
+(2147, 329):3372
+(589, 519):3373
+(269, 455):3374
+(420, 288):3375
+(448, 371):3376
+(312, 421):3377
+(274, 410):3378
+(343, 618):3379
+(270, 287):3380
+(307, 1755):3381
+(289, 2028):3382
+(1042, 334):3383
+(315, 1330):3384
+(1058, 294):3385
+(275, 450):3386
+(314, 1923):3387
+(795, 963):3388
+(353, 876):3389
+(295, 862):3390
+(964, 273):3391
+(306, 1472):3392
+(412, 436):3393
+(355, 267):3394
+(257, 145):3395
+(290, 261):3396
+(729, 514):3397
+(289, 736):3398
+(305, 525):3399
+(2714, 436):3400
+(495, 278):3401
+(275, 451):3402
+(1062, 283):3403
+(731, 276):3404
+(288, 278):3405
+(408, 433):3406
+(535, 185):3407
+(712, 261):3408
+(439, 304):3409
+(354, 263):3410
+(308, 815):3411
+(301, 320):3412
+(265, 579):3413
+(291, 565):3414
+(418, 320):3415
+(276, 294):3416
+(937, 746):3417
+(647, 156):3418
+(1103, 408):3419
+(271, 624):3420
+(1932, 149):3421
+(386, 454):3422
+(632, 266):3423
+(273, 1209):3424
+(275, 464):3425
+(275, 405):3426
+(1041, 277):3427
+(265, 413):3428
+(679, 372):3429
+(470, 261):3430
+(379, 1437):3431
+(648, 278):3432
+(481, 311):3433
+(299, 644):3434
+(315, 2174):3435
+(1272, 269):3436
+(356, 368):3437
+(1439, 267):3438
+(607, 276):3439
+(619, 337):3440
+(904, 261):3441
+(276, 575):3442
+(308, 589):3443
+(303, 337):3444
+(288, 1259):3445
+(288, 437):3446
+(1105, 261):3447
+(1390, 285):3448
+(306, 779):3449
+(524, 297):3450
+(1344, 458):3451
+(264, 316):3452
+(272, 712):3453
+(340, 529):3454
+(353, 607):3455
+(332, 266):3456
+(274, 2759):3457
+(440, 329):3458
+(794, 278):3459
+(341, 725):3460
+(274, 357):3461
+(883, 287):3462
+(997, 448):3463
+(315, 337):3464
+(471, 618):3465
+(336, 261):3466
+(274, 433):3467
+(288, 270):3468
+(1583, 462):3469
+(659, 1437):3470
+(314, 326):3471
+(969, 512):3472
+(852, 401):3473
+(315, 683):3474
+(659, 678):3475
+(315, 287):3476
+(343, 903):3477
+(285, 365):3478
+(675, 267):3479
+(659, 671):3480
+(428, 261):3481
+(401, 400):3482
+(326, 1786):3483
+(381, 576):3484
+(673, 312):3485
+(322, 475):3486
+(326, 299):3487
+(270, 394):3488
+(456, 1259):3489
+(384, 347):3490
+(322, 1279):3491
+(1542, 564):3492
+(306, 766):3493
+(418, 352):3494
+(275, 373):3495
+(315, 1873):3496
+(297, 1163):3497
+(285, 373):3498
+(293, 916):3499
+(338, 374):3500
+(718, 830):3501
+(279, 2357):3502
+(300, 533):3503
+(301, 344):3504
+(648, 463):3505
+(289, 1004):3506
+(415, 1114):3507
+(326, 264):3508
+(311, 431):3509
+(378, 1006):3510
+(750, 889):3511
+(271, 719):3512
+(991, 273):3513
+(301, 340):3514
+(482, 1835):3515
+(669, 654):3516
+(370, 759):3517
+(353, 412):3518
+(301, 972):3519
+(699, 278):3520
+(954, 269):3521
+(338, 515):3522
+(524, 1647):3523
+(775, 329):3524
+(1117, 970):3525
+(3113, 3114):3526
+(301, 2481):3527
+(517, 2564):3528
+(257, 2214):3529
+(604, 283):3530
+(270, 457):3531
+(265, 797):3532
+(282, 1001):3533
+(767, 419):3534
+(499, 288):3535
+(1501, 728):3536
+(1316, 273):3537
+(297, 307):3538
+(314, 397):3539
+(275, 1637):3540
+(289, 501):3541
+(343, 570):3542
+(411, 1056):3543
+(1432, 324):3544
+(841, 278):3545
+(271, 608):3546
+(299, 329):3547
+(1715, 689):3548
+(264, 487):3549
+(691, 329):3550
+(274, 291):3551
+(1297, 267):3552
+(948, 1650):3553
+(391, 853):3554
+(275, 2111):3555
+(997, 581):3556
+(1157, 357):3557
+(378, 1802):3558
+(518, 273):3559
+(1622, 273):3560
+(2584, 548):3561
+(386, 271):3562
+(1102, 261):3563
+(421, 273):3564
+(319, 807):3565
+(660, 2226):3566
+(1193, 281):3567
+(691, 267):3568
+(740, 855):3569
+(636, 855):3570
+(380, 500):3571
+(2225, 1089):3572
+(1640, 336):3573
+(1100, 261):3574
+(352, 736):3575
+(289, 483):3576
+(385, 345):3577
+(563, 172):3578
+(299, 2063):3579
+(1809, 342):3580
+(322, 288):3581
+(456, 642):3582
+(1217, 324):3583
+(1061, 717):3584
+(941, 269):3585
+(271, 694):3586
+(1888, 526):3587
+(370, 1382):3588
+(400, 489):3589
+(1217, 649):3590
+(409, 607):3591
+(1810, 1081):3592
+(456, 290):3593
+(1532, 451):3594
+(603, 267):3595
+(1597, 261):3596
+(271, 261):3597
+(421, 413):3598
+(339, 466):3599
+(1427, 373):3600
+(755, 1657):3601
+(270, 603):3602
+(510, 276):3603
+(1085, 267):3604
+(834, 360):3605
+(378, 1226):3606
+(415, 2400):3607
+(295, 1618):3608
+(1044, 487):3609
+(319, 310):3610
+(271, 364):3611
+(1095, 367):3612
+(1504, 273):3613
+(1991, 273):3614
+(1405, 328):3615
+(295, 369):3616
+(878, 356):3617
+(286, 290):3618
+(1067, 276):3619
+(663, 458):3620
+(350, 2880):3621
+(681, 323):3622
+(339, 302):3623
+(353, 505):3624
+(726, 555):3625
+(444, 278):3626
+(1110, 441):3627
+(914, 276):3628
+(2060, 621):3629
+(297, 346):3630
+(1506, 1392):3631
+(486, 949):3632
+(285, 284):3633
+(305, 533):3634
+(310, 424):3635
+(303, 448):3636
+(767, 269):3637
+(364, 274):3638
+(535, 167):3639
+(317, 637):3640
+(911, 551):3641
+(471, 324):3642
+(297, 2719):3643
+(256, 136):3644
+(343, 416):3645
+(1466, 278):3646
+(275, 281):3647
+(521, 272):3648
+(1998, 408):3649
+(967, 273):3650
+(873, 559):3651
+(410, 452):3652
+(2611, 413):3653
+(295, 545):3654
+(1352, 335):3655
+(965, 320):3656
+(274, 509):3657
+(547, 465):3658
+(471, 835):3659
+(282, 445):3660
+(492, 801):3661
+(283, 347):3662
+(282, 705):3663
+(271, 1304):3664
+(576, 273):3665
+(287, 328):3666
+(396, 1738):3667
+(293, 267):3668
+(343, 350):3669
+(778, 1821):3670
+(343, 1162):3671
+(343, 361):3672
+(421, 361):3673
+(378, 675):3674
+(414, 273):3675
+(291, 719):3676
+(669, 300):3677
+(704, 334):3678
+(625, 278):3679
+(1248, 284):3680
+(338, 553):3681
+(282, 365):3682
+(295, 581):3683
+(267, 403):3684
+(303, 267):3685
+(840, 327):3686
+(949, 375):3687
+(301, 738):3688
+(295, 2008):3689
+(1113, 288):3690
+(286, 1899):3691
+(856, 325):3692
+(482, 1787):3693
+(287, 707):3694
+(340, 276):3695
+(837, 307):3696
+(481, 1160):3697
+(274, 307):3698
+(299, 458):3699
+(2313, 360):3700
+(520, 264):3701
+(707, 273):3702
+(732, 415):3703
+(340, 366):3704
+(364, 352):3705
+(563, 156):3706
+(914, 273):3707
+(580, 512):3708
+(341, 451):3709
+(421, 2310):3710
+(832, 275):3711
+(282, 1091):3712
+(291, 847):3713
+(275, 291):3714
+(293, 357):3715
+(2836, 288):3716
+(279, 344):3717
+(1072, 449):3718
+(875, 1536):3719
+(546, 505):3720
+(456, 437):3721
+(1428, 287):3722
+(2435, 335):3723
+(410, 515):3724
+(292, 448):3725
+(426, 269):3726
+(1344, 484):3727
+(1161, 357):3728
+(1194, 273):3729
+(276, 345):3730
+(394, 273):3731
+(274, 382):3732
+(314, 633):3733
+(343, 522):3734
+(352, 269):3735
+(541, 288):3736
+(1206, 812):3737
+(340, 373):3738
+(300, 360):3739
+(438, 1023):3740
+(276, 328):3741
+(2442, 312):3742
+(282, 337):3743
+(554, 488):3744
+(315, 510):3745
+(274, 309):3746
+(382, 265):3747
+(766, 519):3748
+(2078, 414):3749
+(285, 601):3750
+(1884, 336):3751
+(1496, 374):3752
+(2665, 2266):3753
+(660, 2650):3754
+(465, 344):3755
+(315, 410):3756
+(927, 273):3757
+(275, 1034):3758
+(293, 532):3759
+(399, 261):3760
+(1420, 327):3761
+(1810, 573):3762
+(293, 449):3763
+(589, 431):3764
+(288, 423):3765
+(3330, 276):3766
+(674, 355):3767
+(2017, 311):3768
+(265, 339):3769
+(952, 334):3770
+(275, 389):3771
+(326, 368):3772
+(392, 1351):3773
+(674, 364):3774
+(957, 540):3775
+(1409, 652):3776
+(269, 611):3777
+(267, 341):3778
+(266, 652):3779
+(279, 305):3780
+(256, 160):3781
+(1817, 408):3782
+(322, 774):3783
+(1600, 753):3784
+(1254, 422):3785
+(499, 1008):3786
+(301, 574):3787
+(777, 1557):3788
+(2012, 281):3789
+(1106, 284):3790
+(349, 381):3791
+(282, 405):3792
+(816, 2136):3793
+(1058, 692):3794
+(2961, 905):3795
+(1012, 276):3796
+(664, 522):3797
+(2114, 790):3798
+(3351, 717):3799
+(297, 1456):3800
+(2119, 579):3801
+(961, 988):3802
+(2679, 284):3803
+(3353, 290):3804
+(437, 269):3805
+(531, 267):3806
+(364, 1830):3807
+(1131, 425):3808
+(415, 1118):3809
+(295, 744):3810
+(325, 337):3811
+(293, 2456):3812
+(1238, 605):3813
+(314, 874):3814
+(2006, 261):3815
+(294, 387):3816
+(392, 446):3817
+(308, 274):3818
+(2974, 1025):3819
+(844, 381):3820
+(742, 359):3821
+(310, 1449):3822
+(392, 3362):3823
+(270, 1314):3824
+(742, 458):3825
+(937, 347):3826
+(285, 448):3827
+(290, 365):3828
+(326, 1547):3829
+(315, 1015):3830
+(523, 981):3831
+(2978, 453):3832
+(405, 424):3833
+(1679, 3368):3834
+(271, 1669):3835
+(553, 356):3836
+(1041, 273):3837
+(1915, 430):3838
+(890, 1370):3839
+(297, 1328):3840
+(264, 1263):3841
+(2995, 559):3842
+(1044, 261):3843
+(1170, 287):3844
+(1013, 2152):3845
+(267, 522):3846
+(271, 408):3847
+(1614, 1839):3848
+(3385, 421):3849
+(911, 433):3850
+(1615, 608):3851
+(1077, 285):3852
+(370, 269):3853
+(2155, 371):3854
+(289, 297):3855
+(317, 446):3856
+(1474, 288):3857
+(946, 3393):3858
+(1152, 287):3859
+(303, 323):3860
+(467, 267):3861
+(1758, 388):3862
+(349, 705):3863
+(539, 609):3864
+(648, 344):3865
+(294, 597):3866
+(546, 352):3867
+(322, 686):3868
+(965, 1618):3869
+(381, 267):3870
+(2717, 364):3871
+(980, 329):3872
+(858, 3012):3873
+(1172, 371):3874
+(1372, 339):3875
+(3411, 287):3876
+(326, 284):3877
+(1042, 344):3878
+(333, 172):3879
+(1232, 329):3880
+(691, 284):3881
+(264, 533):3882
+(696, 276):3883
+(288, 1381):3884
+(328, 320):3885
+(285, 460):3886
+(3886, 377):3887
+(382, 307):3888
+(291, 332):3889
+(759, 2306):3890
+(290, 352):3891
+(1467, 507):3892
+(1058, 264):3893
+(3893, 2505):3894
+(2702, 464):3895
+(1685, 273):3896
+(380, 345):3897
+(438, 275):3898
+(336, 373):3899
+(392, 1210):3900
+(964, 330):3901
+(1690, 3434):3902
+(2057, 655):3903
+(271, 341):3904
+(982, 173):3905
+(331, 352):3906
+(320, 334):3907
+(1076, 276):3908
+(682, 1857):3909
+(584, 2745):3910
+(1425, 284):3911
+(315, 1043):3912
+(405, 462):3913
+(1631, 273):3914
+(301, 322):3915
+(1389, 261):3916
+(1103, 559):3917
+(271, 1295):3918
+(2324, 287):3919
+(749, 575):3920
+(566, 273):3921
+(288, 520):3922
+(1106, 375):3923
+(308, 2183):3924
+(283, 305):3925
+(315, 310):3926
+(269, 490):3927
+(1775, 531):3928
+(899, 3457):3929
+(384, 562):3930
+(421, 302):3931
+(279, 2761):3932
+(1534, 273):3933
+(282, 361):3934
+(405, 277):3935
+(644, 261):3936
+(380, 1015):3937
+(1398, 407):3938
+(690, 359):3939
+(471, 284):3940
+(770, 371):3941
+(523, 312):3942
+(299, 509):3943
+(499, 1159):3944
+(1466, 281):3945
+(1160, 1584):3946
+(286, 484):3947
+(265, 1077):3948
+(384, 516):3949
+(410, 689):3950
+(1750, 273):3951
+(369, 512):3952
+(599, 261):3953
+(938, 269):3954
+(641, 567):3955
+(2175, 365):3956
+(1086, 454):3957
+(3482, 3483):3958
+(326, 350):3959
+(568, 359):3960
+(1278, 2784):3961
+(1086, 264):3962
+(857, 330):3963
+(412, 421):3964
+(384, 505):3965
+(341, 273):3966
+(331, 521):3967
+(1268, 737):3968
+(299, 419):3969
+(471, 264):3970
+(341, 450):3971
+(635, 559):3972
+(386, 503):3973
+(1173, 463):3974
+(271, 516):3975
+(690, 791):3976
+(294, 521):3977
+(1086, 515):3978
+(1257, 1493):3979
+(307, 519):3980
+(311, 363):3981
+(326, 1265):3982
+(292, 554):3983
+(2795, 509):3984
+(434, 452):3985
+(353, 1705):3986
+(1221, 559):3987
+(1439, 273):3988
+(270, 812):3989
+(3082, 320):3990
+(349, 689):3991
+(275, 1226):3992
+(1234, 709):3993
+(1160, 1406):3994
+(829, 267):3995
+(385, 986):3996
+(481, 274):3997
+(349, 454):3998
+(752, 461):3999
+(485, 261):4000
+(879, 678):4001
+(1528, 413):4002
+(997, 861):4003
+(337, 664):4004
+(843, 433):4005
+(295, 1369):4006
+(293, 485):4007
+(269, 360):4008
+(617, 273):4009
+(275, 517):4010
+(1222, 264):4011
+(264, 1639):4012
+(289, 310):4013
+(287, 759):4014
+(285, 468):4015
+(274, 985):4016
+(898, 267):4017
+(615, 1755):4018
+(376, 352):4019
+(797, 472):4020
+(660, 442):4021
+(322, 1967):4022
+(378, 811):4023
+(1465, 702):4024
+(290, 479):4025
+(262, 2820):4026
+(409, 757):4027
+(628, 284):4028
+(669, 422):4029
+(2372, 801):4030
+(331, 733):4031
+(411, 2183):4032
+(276, 500):4033
+(1502, 746):4034
+(2538, 498):4035
+(783, 509):4036
+(331, 327):4037
+(296, 341):4038
+(1590, 433):4039
+(405, 310):4040
+(2573, 589):4041
+(264, 1971):4042
+(1414, 269):4043
+(297, 2352):4044
+(459, 267):4045
+(274, 390):4046
+(859, 281):4047
+(274, 528):4048
+(285, 1716):4049
+(596, 1084):4050
+(1752, 556):4051
+(546, 421):4052
+(1624, 274):4053
+(409, 1063):4054
+(1101, 273):4055
+(544, 267):4056
+(380, 853):4057
+(1587, 1950):4058
+(309, 515):4059
+(738, 357):4060
+(1108, 498):4061
+(306, 847):4062
+(552, 463):4063
+(282, 381):4064
+(636, 1194):4065
+(271, 337):4066
+(1600, 404):4067
+(1317, 689):4068
+(481, 294):4069
+(384, 511):4070
+(505, 1002):4071
+(310, 276):4072
+(805, 266):4073
+(352, 425):4074
+(286, 452):4075
+(690, 1952):4076
+(546, 414):4077
+(1418, 345):4078
+(535, 156):4079
+(992, 276):4080
+(291, 276):4081
+(576, 511):4082
+(704, 453):4083
+(285, 309):4084
+(1080, 688):4085
+(1168, 1947):4086
+(1715, 714):4087
+(1610, 361):4088
+(699, 281):4089
+(477, 373):4090
+(259, 144):4091
+(740, 329):4092
+(332, 271):4093
+(2582, 2861):4094
+(291, 335):4095
+(748, 949):4096
+(720, 276):4097
+(477, 1458):4098
+(473, 299):4099
+(264, 1087):4100
+(1049, 3180):4101
+(271, 335):4102
+(494, 2598):4103
+(323, 540):4104
+(314, 860):4105
+(1429, 273):4106
+(1301, 757):4107
+(338, 418):4108
+(731, 609):4109
+(282, 1016):4110
+(314, 830):4111
+(282, 3069):4112
+(961, 373):4113
+(340, 603):4114
+(1388, 272):4115
+(453, 261):4116
+(593, 490):4117
+(456, 1370):4118
+(2296, 637):4119
+(2405, 1052):4120
+(716, 829):4121
+(901, 581):4122
+(270, 509):4123
+(549, 571):4124
+(1554, 355):4125
+(352, 281):4126
+(1731, 273):4127
+(900, 261):4128
+(396, 265):4129
+(329, 273):4130
+(322, 537):4131
+(411, 514):4132
+(1011, 483):4133
+(330, 357):4134
+(340, 361):4135
+(267, 297):4136
+(275, 1030):4137
+(568, 786):4138
+(378, 346):4139
+(545, 265):4140
+(840, 575):4141
+(276, 983):4142
+(271, 1662):4143
+(2410, 276):4144
+(1818, 299):4145
+(764, 935):4146
+(568, 973):4147
+(368, 311):4148
+(317, 475):4149
+(3221, 587):4150
+(315, 811):4151
+(481, 670):4152
+(295, 1070):4153
+(385, 500):4154
+(2163, 1363):4155
+(673, 450):4156
+(282, 401):4157
+(323, 346):4158
+(294, 2801):4159
+(338, 360):4160
+(2893, 390):4161
+(582, 273):4162
+(687, 327):4163
+(423, 340):4164
+(1569, 273):4165
+(442, 2237):4166
+(629, 2412):4167
+(636, 791):4168
+(378, 1501):4169
+(328, 413):4170
+(384, 2627):4171
+(306, 408):4172
+(4172, 379):4173
+(639, 376):4174
+(384, 1272):4175
+(274, 1350):4176
+(307, 479):4177
+(513, 637):4178
+(270, 407):4179
+(333, 152):4180
+(849, 2895):4181
+(320, 970):4182
+(292, 269):4183
+(1252, 741):4184
+(1235, 1175):4185
+(286, 492):4186
+(863, 1250):4187
+(721, 267):4188
+(1054, 461):4189
+(297, 276):4190
+(349, 1996):4191
+(748, 643):4192
+(497, 278):4193
+(374, 283):4194
+(290, 399):4195
+(1579, 373):4196
+(1535, 344):4197
+(377, 1207):4198
+(1624, 448):4199
+(315, 3236):4200
+(427, 267):4201
+(341, 405):4202
+(384, 1927):4203
+(2087, 517):4204
+(997, 265):4205
+(638, 330):4206
+(339, 330):4207
+(1125, 649):4208
+(2258, 274):4209
+(2219, 357):4210
+(423, 265):4211
+(338, 300):4212
+(1194, 276):4213
+(304, 712):4214
+(302, 272):4215
+(1134, 320):4216
+(1540, 857):4217
+(285, 339):4218
+(1238, 906):4219
+(301, 412):4220
+(1776, 284):4221
+(308, 387):4222
+(4222, 312):4223
+(280, 393):4224
+(890, 490):4225
+(878, 490):4226
+(279, 1537):4227
+(330, 312):4228
+(287, 366):4229
+(1945, 357):4230
+(513, 267):4231
+(307, 318):4232
+(314, 2370):4233
+(1988, 382):4234
+(1112, 299):4235
+(265, 427):4236
+(292, 555):4237
+(1692, 1229):4238
+(626, 347):4239
+(374, 276):4240
+(293, 1776):4241
+(378, 1449):4242
+(2641, 1063):4243
+(328, 283):4244
+(1829, 261):4245
+(401, 487):4246
+(395, 322):4247
+(301, 3282):4248
+(1403, 273):4249
+(1549, 1935):4250
+(434, 2284):4251
+(370, 972):4252
+(1029, 511):4253
+(2187, 265):4254
+(275, 297):4255
+(279, 294):4256
+(328, 272):4257
+(314, 1864):4258
+(350, 496):4259
+(279, 3292):4260
+(382, 840):4261
+(288, 955):4262
+(523, 650):4263
+(926, 324):4264
+(2031, 439):4265
+(1012, 261):4266
+(636, 284):4267
+(765, 371):4268
+(1073, 999):4269
+(384, 305):4270
+(286, 503):4271
+(504, 400):4272
+(275, 418):4273
+(1038, 346):4274
+(327, 515):4275
+(820, 490):4276
+(1035, 781):4277
+(378, 1384):4278
+(451, 273):4279
+(269, 290):4280
+(1497, 702):4281
+(2811, 512):4282
+(262, 630):4283
+(276, 1209):4284
+(293, 721):4285
+(449, 284):4286
+(323, 1721):4287
+(2137, 537):4288
+(265, 515):4289
+(294, 515):4290
+(451, 324):4291
+(360, 334):4292
+(282, 759):4293
+(1915, 1964):4294
+(568, 504):4295
+(378, 275):4296
+(384, 3115):4297
+(314, 299):4298
+(1819, 401):4299
+(1149, 1276):4300
+(370, 3315):4301
+(936, 273):4302
+(1165, 448):4303
+(309, 291):4304
+(541, 292):4305
+(406, 261):4306
+(286, 470):4307
+(3316, 428):4308
+(270, 734):4309
+(327, 331):4310
+(1190, 345):4311
+(286, 1656):4312
+(338, 267):4313
+(629, 323):4314
+(256, 138):4315
+(403, 375):4316
+(3215, 365):4317
+(323, 261):4318
+(710, 701):4319
+(595, 273):4320
+(469, 348):4321
+(287, 291):4322
+(566, 290):4323
+(3270, 2945):4324
+(1887, 330):4325
+(364, 276):4326
+(2805, 1556):4327
+(1183, 360):4328
+(566, 364):4329
+(285, 2118):4330
+(271, 1104):4331
+(345, 288):4332
+(288, 556):4333
+(286, 376):4334
+(317, 437):4335
+(465, 334):4336
+(524, 521):4337
+(276, 590):4338
+(644, 263):4339
+(1663, 281):4340
+(405, 390):4341
+(1743, 302):4342
+(449, 273):4343
+(314, 445):4344
+(320, 1077):4345
+(295, 320):4346
+(952, 3329):4347
+(341, 934):4348
+(646, 312):4349
+(265, 324):4350
+(1061, 271):4351
+(777, 608):4352
+(1113, 430):4353
+(493, 3722):4354
+(1778, 548):4355
+(674, 864):4356
+(315, 465):4357
+(267, 461):4358
+(1366, 267):4359
+(384, 366):4360
+(1367, 281):4361
+(435, 1287):4362
+(314, 980):4363
+(341, 501):4364
+(271, 751):4365
+(1332, 614):4366
+(343, 3245):4367
+(378, 273):4368
+(323, 652):4369
+(291, 339):4370
+(270, 339):4371
+(568, 311):4372
+(317, 327):4373
+(1278, 331):4374
+(294, 1155):4375
+(613, 1048):4376
+(265, 274):4377
+(1663, 278):4378
+(355, 271):4379
+(319, 533):4380
+(354, 3779):4381
+(282, 3781):4382
+(755, 881):4383
+(448, 556):4384
+(764, 390):4385
+(487, 479):4386
+(456, 359):4387
+(275, 468):4388
+(1797, 346):4389
+(915, 489):4390
+(295, 1819):4391
+(270, 1023):4392
+(438, 3085):4393
+(362, 287):4394
+(290, 401):4395
+(1877, 580):4396
+(552, 510):4397
+(473, 281):4398
+(279, 346):4399
+(819, 1473):4400
+(490, 278):4401
+(353, 474):4402
+(524, 1533):4403
+(265, 448):4404
+(4404, 1096):4405
+(3349, 458):4406
+(628, 340):4407
+(477, 989):4408
+(2454, 284):4409
+(815, 1746):4410
+(733, 299):4411
+(314, 271):4412
+(794, 281):4413
+(271, 1015):4414
+(2968, 381):4415
+(1334, 270):4416
+(456, 332):4417
+(1793, 3802):4418
+(317, 2642):4419
+(293, 3806):4420
+(328, 335):4421
+(327, 335):4422
+(378, 270):4423
+(1463, 364):4424
+(349, 1781):4425
+(820, 1593):4426
+(918, 273):4427
+(1010, 2021):4428
+(2429, 675):4429
+(730, 548):4430
+(1702, 322):4431
+(892, 281):4432
+(2681, 284):4433
+(905, 365):4434
+(1723, 267):4435
+(2358, 1216):4436
+(1035, 413):4437
+(1399, 374):4438
+(421, 678):4439
+(1589, 278):4440
+(2252, 2146):4441
+(1217, 261):4442
+(698, 521):4443
+(470, 276):4444
+(2437, 452):4445
+(296, 332):4446
+(2017, 1318):4447
+(293, 754):4448
+(2549, 1114):4449
+(790, 418):4450
+(4450, 571):4451
+(279, 3822):4452
+(319, 1498):4453
+(848, 448):4454
+(2748, 310):4455
+(1287, 273):4456
+(915, 342):4457
+(1044, 1825):4458
+(2281, 1745):4459
+(3709, 2146):4460
+(2688, 287):4461
+(513, 655):4462
+(435, 629):4463
+(984, 327):4464
+(392, 2653):4465
+(331, 560):4466
+(1532, 527):4467
+(370, 332):4468
+(317, 672):4469
+(2671, 607):4470
+(2451, 867):4471
+(742, 548):4472
+(411, 3833):4473
+(646, 2910):4474
+(264, 717):4475
+(673, 1186):4476
+(1743, 305):4477
+(301, 812):4478
+(482, 876):4479
+(644, 344):4480
+(1221, 263):4481
+(274, 387):4482
+(772, 475):4483
+(753, 281):4484
+(735, 323):4485
+(328, 480):4486
+(301, 2024):4487
+(286, 898):4488
+(1202, 261):4489
+(1289, 670):4490
+(275, 352):4491
+(1119, 311):4492
+(1514, 371):4493
+(2691, 944):4494
+(1838, 1918):4495
+(349, 866):4496
+(270, 305):4497
+(547, 994):4498
+(695, 931):4499
+(267, 342):4500
+(716, 2998):4501
+(270, 327):4502
+(396, 1223):4503
+(327, 276):4504
+(1518, 838):4505
+(1569, 329):4506
+(401, 369):4507
+(286, 480):4508
+(1519, 330):4509
+(1205, 1150):4510
+(274, 336):4511
+(893, 336):4512
+(1151, 551):4513
+(323, 421):4514
+(979, 422):4515
+(467, 732):4516
+(308, 2158):4517
+(2475, 781):4518
+(1566, 376):4519
+(436, 273):4520
+(2159, 463):4521
+(282, 345):4522
+(384, 538):4523
+(1470, 1669):4524
+(2162, 652):4525
+(1329, 288):4526
+(1683, 273):4527
+(466, 827):4528
+(481, 445):4529
+(550, 1063):4530
+(369, 2721):4531
+(1100, 350):4532
+(695, 622):4533
+(1049, 352):4534
+(2491, 3410):4535
+(310, 267):4536
+(395, 944):4537
+(270, 1017):4538
+(270, 2725):4539
+(274, 1018):4540
+(320, 305):4541
+(2303, 304):4542
+(293, 2496):4543
+(4543, 3016):4544
+(1064, 276):4545
+(964, 329):4546
+(2729, 914):4547
+(322, 307):4548
+(1687, 501):4549
+(275, 300):4550
+(759, 377):4551
+(275, 504):4552
+(314, 387):4553
+(766, 299):4554
+(1372, 3424):4555
+(2487, 441):4556
+(777, 316):4557
+(3427, 967):4558
+(1222, 515):4559
+(297, 1016):4560
+(326, 2312):4561
+(319, 777):4562
+(288, 1936):4563
+(2736, 692):4564
+(271, 325):4565
+(285, 413):4566
+(552, 416):4567
+(964, 334):4568
+(691, 274):4569
+(819, 276):4570
+(516, 281):4571
+(816, 1043):4572
+(858, 2512):4573
+(601, 397):4574
+(3034, 374):4575
+(314, 288):4576
+(1388, 483):4577
+(2515, 413):4578
+(275, 1295):4579
+(385, 525):4580
+(294, 372):4581
+(309, 273):4582
+(1377, 3038):4583
+(979, 274):4584
+(415, 364):4585
+(638, 1211):4586
+(1184, 276):4587
+(270, 2519):4588
+(1068, 261):4589
+(582, 1179):4590
+(271, 305):4591
+(271, 591):4592
+(1021, 310):4593
+(309, 642):4594
+(271, 1070):4595
+(338, 1084):4596
+(306, 310):4597
+(640, 344):4598
+(294, 826):4599
+(837, 2184):4600
+(295, 323):4601
+(317, 302):4602
+(301, 1570):4603
+(340, 1017):4604
+(769, 284):4605
+(311, 334):4606
+(294, 1394):4607
+(329, 276):4608
+(1523, 413):4609
+(1395, 815):4610
+(320, 269):4611
+(2329, 2150):4612
+(319, 2330):4613
+(1131, 307):4614
+(444, 281):4615
+(386, 1153):4616
+(776, 585):4617
+(2149, 433):4618
+(524, 1684):4619
+(682, 325):4620
+(2764, 360):4621
+(409, 876):4622
+(1535, 278):4623
+(1132, 539):4624
+(352, 272):4625
+(384, 404):4626
+(2336, 1159):4627
+(2187, 1851):4628
+(275, 642):4629
+(292, 436):4630
+(264, 774):4631
+(1699, 3942):4632
+(339, 419):4633
+(384, 397):4634
+(1393, 507):4635
+(320, 1134):4636
+(274, 445):4637
+(605, 1369):4638
+(337, 519):4639
+(1951, 764):4640
+(805, 283):4641
+(282, 465):4642
+(772, 2071):4643
+(320, 651):4644
+(288, 498):4645
+(384, 405):4646
+(397, 376):4647
+(682, 405):4648
+(1576, 1074):4649
+(291, 1435):4650
+(448, 261):4651
+(1916, 329):4652
+(308, 521):4653
+(704, 2084):4654
+(1541, 287):4655
+(974, 497):4656
+(699, 290):4657
+(635, 269):4658
+(598, 283):4659
+(339, 276):4660
+(283, 267):4661
+(265, 771):4662
+(1003, 405):4663
+(576, 287):4664
+(1153, 261):4665
+(391, 450):4666
+(301, 1046):4667
+(411, 1311):4668
+(414, 2797):4669
+(297, 424):4670
+(329, 290):4671
+(364, 671):4672
+(283, 931):4673
+(710, 261):4674
+(756, 490):4675
+(338, 261):4676
+(1013, 438):4677
+(396, 264):4678
+(1147, 1400):4679
+(1118, 284):4680
+(377, 273):4681
+(1619, 359):4682
+(414, 366):4683
+(886, 461):4684
+(467, 365):4685
+(908, 487):4686
+(290, 603):4687
+(681, 414):4688
+(1389, 296):4689
+(911, 529):4690
+(275, 2803):4691
+(327, 360):4692
+(400, 492):4693
+(4003, 269):4694
+(3092, 265):4695
+(1962, 866):4696
+(1447, 261):4697
+(270, 551):4698
+(1107, 372):4699
+(825, 944):4700
+(322, 2358):4701
+(3515, 526):4702
+(809, 1965):4703
+(320, 264):4704
+(3981, 285):4705
+(1382, 265):4706
+(341, 1034):4707
+(294, 284):4708
+(900, 698):4709
+(376, 273):4710
+(282, 1878):4711
+(289, 296):4712
+(562, 323):4713
+(275, 559):4714
+(285, 501):4715
+(317, 423):4716
+(890, 356):4717
+(1298, 276):4718
+(370, 1170):4719
+(2368, 374):4720
+(1243, 273):4721
+(2098, 773):4722
+(3110, 551):4723
+(817, 528):4724
+(264, 270):4725
+(1300, 397):4726
+(316, 587):4727
+(663, 324):4728
+(396, 328):4729
+(364, 264):4730
+(297, 614):4731
+(282, 487):4732
+(361, 273):4733
+(329, 812):4734
+(481, 292):4735
+(1131, 336):4736
+(1968, 283):4737
+(767, 559):4738
+(1397, 478):4739
+(279, 1434):4740
+(343, 442):4741
+(301, 437):4742
+(513, 376):4743
+(399, 284):4744
+(2828, 479):4745
+(415, 2216):4746
+(992, 273):4747
+(3535, 732):4748
+(462, 269):4749
+(1475, 903):4750
+(3133, 264):4751
+(427, 273):4752
+(265, 734):4753
+(604, 272):4754
+(1102, 529):4755
+(910, 1802):4756
+(1236, 264):4757
+(323, 283):4758
+(1931, 562):4759
+(342, 273):4760
+(679, 322):4761
+(286, 732):4762
+(657, 686):4763
+(290, 307):4764
+(777, 497):4765
+(674, 1803):4766
+(350, 354):4767
+(1599, 517):4768
+(382, 271):4769
+(301, 4050):4770
+(2110, 702):4771
+(2837, 278):4772
+(1593, 291):4773
+(3555, 1224):4774
+(401, 528):4775
+(2992, 962):4776
+(872, 284):4777
+(809, 330):4778
+(1644, 511):4779
+(289, 302):4780
+(835, 273):4781
+(301, 300):4782
+(1446, 3561):4783
+(1585, 291):4784
+(599, 1713):4785
+(2057, 774):4786
+(275, 671):4787
+(926, 276):4788
+(1069, 261):4789
+(798, 326):4790
+(755, 631):4791
+(696, 269):4792
+(815, 312):4793
+(303, 310):4794
+(729, 884):4795
+(1703, 498):4796
+(1980, 273):4797
+(469, 512):4798
+(340, 518):4799
+(853, 463):4800
+(1598, 676):4801
+(2179, 718):4802
+(328, 339):4803
+(1531, 681):4804
+(818, 664):4805
+(356, 2325):4806
+(319, 1242):4807
+(317, 328):4808
+(2389, 1036):4809
+(580, 283):4810
+(526, 688):4811
+(638, 267):4812
+(301, 445):4813
+(364, 404):4814
+(302, 357):4815
+(382, 263):4816
+(1007, 1430):4817
+(2854, 433):4818
+(961, 413):4819
+(1692, 438):4820
+(1086, 439):4821
+(1492, 263):4822
+(2060, 290):4823
+(887, 352):4824
+(370, 517):4825
+(265, 329):4826
+(341, 518):4827
+(1543, 802):4828
+(4828, 701):4829
+(916, 607):4830
+(410, 717):4831
+(306, 621):4832
+(2396, 344):4833
+(1569, 267):4834
+(552, 518):4835
+(3177, 413):4836
+(370, 1964):4837
+(1962, 264):4838
+(274, 817):4839
+(275, 906):4840
+(382, 354):4841
+(539, 414):4842
+(322, 616):4843
+(397, 1195):4844
+(606, 995):4845
+(566, 1803):4846
+(636, 1552):4847
+(271, 459):4848
+(876, 263):4849
+(322, 4104):4850
+(288, 1417):4851
+(289, 2803):4852
+(771, 318):4853
+(297, 337):4854
+(1253, 1333):4855
+(1887, 714):4856
+(627, 702):4857
+(301, 1155):4858
+(271, 389):4859
+(289, 325):4860
+(319, 1886):4861
+(319, 470):4862
+(306, 500):4863
+(2022, 580):4864
+(980, 323):4865
+(338, 272):4866
+(875, 474):4867
+(730, 2601):4868
+(1597, 273):4869
+(264, 299):4870
+(1679, 947):4871
+(299, 321):4872
+(295, 1266):4873
+(1730, 1376):4874
+(343, 403):4875
+(289, 1307):4876
+(361, 1084):4877
+(665, 269):4878
+(716, 433):4879
+(1451, 479):4880
+(576, 284):4881
+(1616, 284):4882
+(391, 269):4883
+(387, 302):4884
+(329, 357):4885
+(1281, 345):4886
+(328, 464):4887
+(305, 966):4888
+(294, 1084):4889
+(1042, 366):4890
+(270, 329):4891
+(707, 1406):4892
+(682, 288):4893
+(4893, 1491):4894
+(4894, 497):4895
+(1814, 273):4896
+(2777, 553):4897
+(2111, 525):4898
+(749, 401):4899
+(909, 329):4900
+(747, 933):4901
+(297, 350):4902
+(604, 307):4903
+(494, 529):4904
+(735, 377):4905
+(1148, 670):4906
+(275, 366):4907
+(432, 278):4908
+(380, 2614):4909
+(681, 352):4910
+(314, 307):4911
+(286, 948):4912
+(3216, 771):4913
+(2240, 269):4914
+(664, 371):4915
+(338, 614):4916
+(641, 438):4917
+(1735, 267):4918
+(272, 508):4919
+(328, 373):4920
+(370, 864):4921
+(399, 431):4922
+(865, 423):4923
+(279, 1626):4924
+(1495, 276):4925
+(1779, 329):4926
+(314, 572):4927
+(338, 555):4928
+(537, 261):4929
+(315, 3632):4930
+(520, 287):4931
+(301, 1966):4932
+(657, 284):4933
+(270, 972):4934
+(288, 390):4935
+(730, 374):4936
+(301, 712):4937
+(1622, 261):4938
+(274, 354):4939
+(693, 914):4940
+(317, 986):4941
+(471, 440):4942
+(1278, 2190):4943
+(386, 1990):4944
+(319, 335):4945
+(295, 389):4946
+(1284, 525):4947
+(3644, 1350):4948
+(681, 579):4949
+(1675, 356):4950
+(2475, 335):4951
+(270, 335):4952
+(550, 382):4953
+(648, 281):4954
+(856, 283):4955
+(412, 263):4956
+(634, 271):4957
+(667, 375):4958
+(317, 1242):4959
+(314, 1084):4960
+(756, 299):4961
+(2417, 371):4962
+(513, 1118):4963
+(631, 288):4964
+(1243, 1074):4965
+(845, 950):4966
+(833, 263):4967
+(3247, 312):4968
+(4968, 304):4969
+(302, 284):4970
+(2632, 354):4971
+(2121, 1076):4972
+(319, 364):4973
+(1514, 291):4974
+(511, 261):4975
+(2337, 273):4976
+(299, 484):4977
+(953, 2991):4978
+(2255, 2136):4979
+(513, 335):4980
+(1771, 470):4981
+(511, 433):4982
+(2699, 1085):4983
+(553, 358):4984
+(377, 3224):4985
+(384, 4198):4986
+(271, 1129):4987
+(1125, 344):4988
+(865, 270):4989
+(264, 983):4990
+(478, 1114):4991
+(1505, 1738):4992
+(880, 1813):4993
+(314, 300):4994
+(1183, 611):4995
+(384, 442):4996
+(558, 4159):4997
+(1766, 360):4998
+(973, 334):4999

requirements.txt

@@ -0,0 +1,5 @@
+gradio
+regex
+tqdm
+huggingface-hub
+pytest

src/app.py

@@ -0,0 +1,119 @@
+import gradio as gr
+from tokenizer import sanskrit_token_preprocessor, bpeAlgo, create_vocab, save_paired_tokens_vocab, save_vocab, encode, decode
+
+# Created a `TokenizerModel` class to maintain state between calls
+# - Added a web interface with separate tabs for training and tokenization
+# - Included example texts and documentation
+# - Made the interface more user-friendly with clear labels and instructions
+# - Added JSON output for better visualization of results
+
+# The app provides two main functionalities:
+# 1. Training the tokenizer on custom text
+# 2. Tokenizing new text using the trained model
+class TokenizerModel:
+    def __init__(self, vocab_size=5000):
+        self.vocab_size = vocab_size
+        self.paired_tokens_vocab = None
+        self.vocab = None
+    
+    def train(self, text):
+        # Preprocess tokens
+        tokens = sanskrit_token_preprocessor(text)
+        tokens = [list(tok.encode('utf-8')) for tok in tokens]
+        orig_tokens_len = len([tok for toks in tokens for tok in toks])
+        print(type(orig_tokens_len))
+        
+        # Train BPE
+        try:
+            self.paired_tokens_vocab, encoded_tokens = bpeAlgo(tokens, self.vocab_size - 256, 256)
+            self.vocab = create_vocab(self.paired_tokens_vocab)
+        except ValueError as e:
+             # Display the error message in your UI
+            return f"Error during training: {str(e)}"
+        
+        # save_paired_tokens_vocab("data/paired_tokens.bpe", self.paired_tokens_vocab)
+        # save_vocab( "data/vocab.bpe", self.vocab)
+
+        # Calculate compression ratio
+        compression_ratio = orig_tokens_len / len(encoded_tokens)
+        
+        return f"Training completed! Vocabulary size: {len(self.vocab)} and Compression Ratio: {compression_ratio:.2f}"
+    
+    def tokenize(self, text):
+        if self.paired_tokens_vocab is None:
+            # try:
+            #     with open("data/paired_tokens.bpe", "r") as f:
+            #         self.paired_tokens_vocab = eval(f.read())
+            #     with open("data/vocab.bpe", "r") as f:
+            #         self.vocab = eval(f.read())
+            # except FileNotFoundError as fe:
+            #     return {"Error":f"Please train the tokenizer first or ensure vocabulary files exist! {fe}"}
+            # except SyntaxError as se:
+            #     return {"Error":f"Please train the tokenizer first or ensure vocabulary files exist! {se}"}
+            # except Exception as e:
+            #     return {"Error": str(e)}
+            return {"Error": "Please train it first to create vocab. Training smaples can be found in the data dir"}
+        
+        encoded = encode(text, self.paired_tokens_vocab)
+        decoded = decode(encoded, self.vocab)
+        
+        return {
+            "Encoded tokens": encoded,
+            "Decoded text": decoded,
+            "Matches original": decoded == text
+        }
+
+# Create global tokenizer instance
+tokenizer = TokenizerModel()
+
+def train_tokenizer(text):
+    return tokenizer.train(text)
+
+def process_text(text):
+    return tokenizer.tokenize(text)
+
+# Create the Gradio interface
+with gr.Blocks(title="Sanskrit BPE Tokenizer") as demo:
+    gr.Markdown("# Sanskrit BPE Tokenizer")
+    gr.Markdown("This tokenizer implements Byte-Pair Encoding (BPE) for Sanskrit text.")
+    
+    with gr.Tab("Train"):
+        train_input = gr.Textbox(
+            label="Training Text",
+            placeholder="Enter Sanskrit text for training...",
+            lines=5
+        )
+        train_button = gr.Button("Train Tokenizer")
+        train_output = gr.Textbox(label="Training Result")
+        
+        train_button.click(
+            train_tokenizer,
+            inputs=train_input,
+            outputs=train_output
+        )
+    
+    with gr.Tab("Tokenize"):
+        text_input = gr.Textbox(
+            label="Input Text",
+            placeholder="Enter Sanskrit text to tokenize...",
+            lines=3
+        )
+        tokenize_button = gr.Button("Tokenize")
+        result_output = gr.JSON(label="Results")
+        
+        tokenize_button.click(
+            process_text,
+            inputs=text_input,
+            outputs=result_output
+        )
+    
+    gr.Markdown("""
+    ### Example texts:
+    ```
+    चीराण्यपास्याज्जनकस्य कन्या नेयं प्रतिज्ञा मम दत्तपूर्वा।
+    यथासुखं गच्छतु राजपुत्री वनं समग्रा सह सर्वरत्नैः॥
+    ```
+    """)
+
+if __name__ == "__main__":
+    demo.launch() 

src/playground.ipynb

@@ -0,0 +1,593 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Length of dataset 620958\n"
+     ]
+    }
+   ],
+   "source": [
+    "with open('../static/sanskrit.txt', 'r', encoding='utf-8') as f: \n",
+    "    text = f.read()\n",
+    "\n",
+    "print(f'Length of dataset {len(text)}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "तस्यां चीरं वसानायां नाथवत्यामनाथवत्। प्रचुक्रोश जनः सर्वो धिक् त्वां दशरथं त्विति ॥\n",
+      "तेन तत्र प्रणादेन दुःखितः स महीपतिः। चिच्छेद जीविते श्रद्धां धर्मे यशसि चात्मनः॥ स निःश्वस्योष्णमैक्ष्वाकस्तां भार्यामिदमब्रवीत्। कैकेयि कुशचीरेण न सीता गन्तुमर्हति॥\n",
+      "सुकुमारी च बाला च सततं च सुखोचिता। नेयं वनस्य योग्येति सत्यमाह गुरुर्मम ॥\n",
+      "इयं हि कस्यापि करोति किंचित् तपस्विनी राजवरस्य पुत्री। या चीरमासाद्य वनस्य मध्ये जाता विसंज्ञा श्रमणीव काचित्॥\n",
+      "चीराण्यपास्याज्जनकस्य कन्या नेयं प्रतिज्ञा मम दत्तपूर्वा। यथासुखं गच्छतु राजपुत्री वनं समग्रा सह सर्वरत्नैः॥\n",
+      "अजीवनाहेण मया नृशंसा कृता प्रतिज्ञा नियमेन तावत्। त्वया हि बाल्यात् प्रतिपन्नमेतत् तन्मा दहेद् वेणुमिवात्मपुष्पम्॥\n",
+      "रामेण यदि ते पापे किंचित्कृतमशोभनम्। अपकारः क इह ते वैदेह्या दर्शितोऽधमे॥\n",
+      "मृगीवोत्फुल्लनयना मृदुशीला मनस्विनी। अपकारं कमिव ते करोति जनकात्मजा॥\n",
+      "ननु पर्याप्तमेवं ते पापे रामविवासनम्। किमेभिः कृपणैर्भूयः पातकैरपि ते कृतैः॥\n",
+      "प्रतिज्ञातं मया तावत् त्वयोक्तं देवि शृण्वता। रामं यदभिषेकाय त्वमिहागतमब्रवीः॥\n",
+      "तत्त्वेतत् समतिक्रम्य निरयं गन्तुमिच्छसि।\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(text[:1000])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "99\n",
+      "\n",
+      "   ! ' ( ) , - . : ? [ _ ā ँ ं ः अ आ इ ई उ ऊ ऋ ए ऐ ओ औ क ख ग घ ङ च छ ज झ ञ ट ठ ड ढ ण त थ द ध न प फ ब भ म य र ऱ ल ळ व श ष स ह ़ ऽ ा ि ी ु ू ृ ॄ ॅ े ै ॊ ो ौ ् ॐ ॑ ॒ क़ । ॥ ० १ २ ३ ४ ५ ६ ७ ८ ९ — ‘ ” •\n"
+     ]
+    }
+   ],
+   "source": [
+    "uniqueChars = sorted(list(set(text)))\n",
+    "print(len(uniqueChars))\n",
+    "print(' '.join(uniqueChars))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "2384"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ord('ॐ')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "([10, 32, 32, 32, 33, 32, 39, 32, 40, 32], [224, 165, 144])"
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "l = list(' '.join(uniqueChars).encode('utf-8'))\n",
+    "m = list(\"ॐ\".encode('utf-8'))\n",
+    "l[:10], m"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[224, 164, 164, 224, 164, 184, 224, 165, 141, 224]"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dictionary = list(text.encode('utf-8'))\n",
+    "dictionary[:10]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[(10, 32), (32, 32), (32, 32), (32, 33), (33, 32)]"
+      ]
+     },
+     "execution_count": 38,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "lz = list(zip(l , l[1:]))\n",
+    "lz [:5]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "226"
+      ]
+     },
+     "execution_count": 59,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokens = list(text.encode('utf-8'))\n",
+    "max(tokens)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 74,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{408270: (224, 164), 38329: (164, 164), 38039: (164, 224), 20281: (164, 184), 19343: (184, 224), 141095: (224, 165), 70708: (165, 141), 67426: (141, 224), 22626: (164, 175), 19459: (175, 224), 44980: (164, 190), 38855: (190, 224), 13816: (164, 130), 11423: (130, 32), 64948: (32, 224), 9404: (164, 154), 6005: (154, 224), 5475: (165, 128), 4738: (128, 224), 33267: (164, 176), 32359: (176, 224), 24535: (164, 181), 22815: (181, 224), 24346: (164, 168), 21955: (168, 224), 4024: (164, 165), 3793: (165, 224), 23375: (164, 174), 22769: (174, 224), 7530: (165, 164), 7060: (164, 32), 15009: (164, 170), 14942: (170, 224), 14835: (165, 129), 13336: (129, 224), 12754: (164, 149), 12679: (149, 224), 8616: (165, 139), 5171: (139, 224), 9649: (164, 182), 119: (182, 32), 5673: (164, 156), 5529: (156, 224), 9445: (164, 131), 4620: (131, 32), 3578: (139, 32), 5366: (164, 167), 5340: (167, 224), 24038: (164, 191), 19693: (191, 224), 3257: (141, 32), 13007: (164, 166), 12921: (166, 224), 9527: (182, 224), 4289: (191, 32), 7079: (165, 165), 5361: (165, 10), 6142: (10, 224), 14554: (165, 135), 10727: (135, 224), 2385: (168, 32), 900: (176, 32), 5444: (164, 163), 5007: (163, 224), 9050: (131, 224), 1006: (164, 150), 994: (150, 224), 929: (184, 32), 8202: (164, 185), 7905: (185, 224), 882: (164, 155), 830: (155, 224), 102: (166, 32), 4351: (135, 32), 1948: (165, 32), 8466: (164, 183), 8284: (183, 224), 3607: (165, 136), 3344: (136, 224), 6617: (164, 173), 6628: (173, 224), 2771: (164, 172), 2768: (172, 224), 428: (163, 32), 6057: (190, 32), 5214: (164, 151), 5173: (151, 224), 722: (128, 32), 3378: (154, 32), 4477: (164, 178), 4393: (178, 224), 3161: (175, 32), 295: (185, 32), 607: (174, 32), 539: (164, 135), 5105: (130, 224), 1049: (164, 158), 1019: (158, 224), 1711: (181, 32), 2715: (165, 130), 1505: (129, 32), 1504: (164, 133), 1506: (133, 224), 4251: (165, 131), 71: (149, 32), 1347: (189, 224), 230: (164, 171), 245: (171, 224), 689: (143, 224), 1493: (165, 140), 799: (140, 224), 1378: (164, 159), 1363: (159, 224), 746: (164, 152), 737: (152, 224), 115: (164, 162), 108: (162, 224), 688: (153, 224), 851: (137, 224), 18: (164, 129), 11: (150, 32), 2: (33, 10), 65: (167, 32), 691: (140, 32), 717: (164, 10), 22: (191, 44), 13: (169, 224), 28: (165, 168), 141: (156, 32), 140: (164, 139), 655: (164, 160), 579: (160, 224), 815: (164, 161), 811: (161, 224), 181: (183, 32), 12: (152, 32), 391: (164, 134), 389: (134, 224), 294: (136, 32), 24: (165, 173), 14: (159, 32), 45: (58, 224), 51: (155, 32), 41: (165, 167), 17: (191, 39), 174: (58, 32), 15: (157, 224), 7: (175, 58), 5: (131, 10), 20: (165, 166), 4: (168, 10), 10: (141, 39), 9: (173, 32), 6: (165, 138), 3: (170, 10), 77: (170, 32), 76: (160, 32), 69: (178, 32), 40: (164, 138), 43: (138, 224), 37: (151, 32), 32: (164, 136), 1: (149, 58), 55: (190, 58), 35: (164, 58), 25: (168, 58), 19: (32, 39), 53: (39, 224), 52: (44, 32), 46: (39, 32), 29: (45, 39), 8: (162, 32), 30: (158, 32)}\n",
+      "[(408270, (224, 164)), (141095, (224, 165)), (70708, (165, 141)), (67426, (141, 224)), (64948, (32, 224)), (44980, (164, 190)), (38855, (190, 224)), (38329, (164, 164)), (38039, (164, 224)), (33267, (164, 176)), (32359, (176, 224)), (24535, (164, 181)), (24346, (164, 168)), (24038, (164, 191)), (23375, (164, 174)), (22815, (181, 224)), (22769, (174, 224)), (22626, (164, 175)), (21955, (168, 224)), (20281, (164, 184)), (19693, (191, 224)), (19459, (175, 224)), (19343, (184, 224)), (15009, (164, 170)), (14942, (170, 224)), (14835, (165, 129)), (14554, (165, 135)), (13816, (164, 130)), (13336, (129, 224)), (13007, (164, 166)), (12921, (166, 224)), (12754, (164, 149)), (12679, (149, 224)), (11423, (130, 32)), (10727, (135, 224)), (9649, (164, 182)), (9527, (182, 224)), (9445, (164, 131)), (9404, (164, 154)), (9050, (131, 224)), (8616, (165, 139)), (8466, (164, 183)), (8284, (183, 224)), (8202, (164, 185)), (7905, (185, 224)), (7530, (165, 164)), (7079, (165, 165)), (7060, (164, 32)), (6628, (173, 224)), (6617, (164, 173)), (6142, (10, 224)), (6057, (190, 32)), (6005, (154, 224)), (5673, (164, 156)), (5529, (156, 224)), (5475, (165, 128)), (5444, (164, 163)), (5366, (164, 167)), (5361, (165, 10)), (5340, (167, 224)), (5214, (164, 151)), (5173, (151, 224)), (5171, (139, 224)), (5105, (130, 224)), (5007, (163, 224)), (4738, (128, 224)), (4620, (131, 32)), (4477, (164, 178)), (4393, (178, 224)), (4351, (135, 32)), (4289, (191, 32)), (4251, (165, 131)), (4024, (164, 165)), (3793, (165, 224)), (3607, (165, 136)), (3578, (139, 32)), (3378, (154, 32)), (3344, (136, 224)), (3257, (141, 32)), (3161, (175, 32)), (2771, (164, 172)), (2768, (172, 224)), (2715, (165, 130)), (2385, (168, 32)), (1948, (165, 32)), (1711, (181, 32)), (1506, (133, 224)), (1505, (129, 32)), (1504, (164, 133)), (1493, (165, 140)), (1378, (164, 159)), (1363, (159, 224)), (1347, (189, 224)), (1347, (164, 189)), (1049, (164, 158)), (1019, (158, 224)), (1006, (164, 150)), (994, (150, 224)), (929, (184, 32)), (900, (176, 32)), (882, (164, 155)), (851, (164, 137)), (851, (137, 224)), (830, (155, 224)), (815, (164, 161)), (811, (161, 224)), (799, (140, 224)), (746, (164, 152)), (737, (152, 224)), (722, (128, 32)), (717, (164, 10)), (691, (140, 32)), (689, (164, 143)), (689, (143, 224)), (688, (164, 153)), (688, (153, 224)), (655, (164, 160)), (607, (174, 32)), (579, (160, 224)), (539, (164, 135)), (428, (163, 32)), (391, (164, 134)), (389, (134, 224)), (295, (185, 32)), (294, (136, 32)), (245, (171, 224)), (230, (164, 171)), (181, (183, 32)), (174, (58, 32)), (141, (156, 32)), (140, (164, 139)), (119, (182, 32)), (115, (164, 162)), (108, (162, 224)), (102, (166, 32)), (77, (170, 32)), (76, (160, 32)), (71, (149, 32)), (69, (178, 32)), (65, (167, 32)), (55, (190, 58)), (53, (39, 224)), (52, (44, 32)), (51, (155, 32)), (46, (39, 32)), (45, (58, 224)), (43, (138, 224)), (41, (165, 167)), (40, (164, 138)), (37, (151, 32)), (35, (164, 58)), (32, (164, 136)), (30, (158, 32)), (29, (45, 39)), (28, (165, 168)), (25, (168, 58)), (24, (165, 173)), (22, (191, 44)), (22, (165, 171)), (22, (165, 169)), (20, (165, 166)), (19, (32, 39)), (18, (164, 129)), (17, (191, 39)), (17, (165, 170)), (17, (154, 45)), (17, (33, 32)), (15, (164, 157)), (15, (157, 224)), (14, (191, 58)), (14, (178, 58)), (14, (159, 32)), (13, (169, 224)), (12, (152, 32)), (11, (165, 145)), (11, (165, 132)), (11, (150, 32)), (11, (132, 224)), (11, (128, 58)), (10, (165, 175)), (10, (141, 39)), (9, (173, 32)), (9, (165, 174)), (9, (164, 144)), (9, (145, 224)), (9, (144, 224)), (9, (131, 39)), (8, (162, 32)), (8, (141, 44)), (8, (45, 224)), (7, (175, 58)), (7, (169, 10)), (7, (164, 147)), (7, (163, 58)), (7, (148, 224)), (7, (147, 224)), (7, (129, 58)), (7, (32, 58)), (6, (226, 128)), (6, (176, 58)), (6, (172, 32)), (6, (165, 172)), (6, (165, 138)), (6, (164, 148)), (6, (135, 10)), (6, (131, 44)), (5, (181, 58)), (5, (171, 10)), (5, (131, 10)), (5, (46, 32)), (4, (190, 10)), (4, (175, 10)), (4, (168, 10)), (4, (166, 10)), (4, (161, 32)), (4, (151, 58)), (4, (141, 45)), (4, (139, 33)), (4, (135, 58)), (4, (135, 44)), (4, (63, 224)), (3, (184, 58)), (3, (175, 44)), (3, (174, 58)), (3, (174, 10)), (3, (172, 10)), (3, (170, 10)), (3, (164, 39)), (3, (156, 58)), (3, (138, 32)), (3, (131, 45)), (3, (131, 33)), (3, (129, 10)), (3, (46, 224)), (3, (40, 224)), (3, (33, 224)), (3, (10, 39)), (2, (190, 226)), (2, (190, 46)), (2, (184, 44)), (2, (176, 45)), (2, (174, 33)), (2, (173, 58)), (2, (173, 10)), (2, (171, 32)), (2, (170, 58)), (2, (170, 33)), (2, (169, 32)), (2, (168, 39)), (2, (167, 10)), (2, (165, 133)), (2, (164, 46)), (2, (164, 41)), (2, (157, 32)), (2, (154, 39)), (2, (154, 10)), (2, (145, 32)), (2, (141, 10)), (2, (134, 32)), (2, (130, 58)), (2, (128, 157)), (2, (128, 152)), (2, (95, 224)), (2, (58, 45)), (2, (46, 10)), (2, (44, 224)), (2, (41, 10)), (2, (33, 10)), (2, (32, 226)), (2, (32, 46)), (2, (10, 40)), (1, (196, 129)), (1, (191, 63)), (1, (191, 45)), (1, (191, 33)), (1, (190, 196)), (1, (190, 95)), (1, (190, 63)), (1, (190, 45)), (1, (190, 33)), (1, (188, 32)), (1, (185, 45)), (1, (185, 39)), (1, (184, 46)), (1, (184, 45)), (1, (184, 33)), (1, (184, 10)), (1, (183, 10)), (1, (182, 58)), (1, (182, 45)), (1, (182, 10)), (1, (181, 46)), (1, (181, 45)), (1, (181, 44)), (1, (181, 39)), (1, (179, 224)), (1, (178, 44)), (1, (177, 224)), (1, (175, 45)), (1, (175, 39)), (1, (168, 226)), (1, (168, 46)), (1, (168, 33)), (1, (165, 152)), (1, (165, 146)), (1, (165, 144)), (1, (165, 46)), (1, (164, 188)), (1, (164, 179)), (1, (164, 177)), (1, (164, 33)), (1, (163, 44)), (1, (163, 10)), (1, (159, 58)), (1, (155, 44)), (1, (150, 58)), (1, (149, 58)), (1, (149, 45)), (1, (149, 44)), (1, (149, 33)), (1, (146, 224)), (1, (144, 32)), (1, (141, 33)), (1, (140, 44)), (1, (140, 39)), (1, (140, 33)), (1, (139, 226)), (1, (139, 58)), (1, (139, 10)), (1, (136, 33)), (1, (135, 33)), (1, (130, 44)), (1, (129, 63)), (1, (129, 45)), (1, (129, 39)), (1, (128, 162)), (1, (128, 148)), (1, (128, 95)), (1, (128, 63)), (1, (128, 44)), (1, (128, 33)), (1, (91, 224)), (1, (63, 32)), (1, (32, 91)), (1, (32, 63)), (1, (32, 44)), (1, (32, 40))]\n",
+      "[((224, 164), 408270), ((224, 165), 141095), ((165, 141), 70708), ((141, 224), 67426), ((32, 224), 64948), ((164, 190), 44980), ((190, 224), 38855), ((164, 164), 38329), ((164, 224), 38039), ((164, 176), 33267), ((176, 224), 32359), ((164, 181), 24535), ((164, 168), 24346), ((164, 191), 24038), ((164, 174), 23375), ((181, 224), 22815), ((174, 224), 22769), ((164, 175), 22626), ((168, 224), 21955), ((164, 184), 20281), ((191, 224), 19693), ((175, 224), 19459), ((184, 224), 19343), ((164, 170), 15009), ((170, 224), 14942), ((165, 129), 14835), ((165, 135), 14554), ((164, 130), 13816), ((129, 224), 13336), ((164, 166), 13007), ((166, 224), 12921), ((164, 149), 12754), ((149, 224), 12679), ((130, 32), 11423), ((135, 224), 10727), ((164, 182), 9649), ((182, 224), 9527), ((164, 131), 9445), ((164, 154), 9404), ((131, 224), 9050), ((165, 139), 8616), ((164, 183), 8466), ((183, 224), 8284), ((164, 185), 8202), ((185, 224), 7905), ((165, 164), 7530), ((165, 165), 7079), ((164, 32), 7060), ((173, 224), 6628), ((164, 173), 6617), ((10, 224), 6142), ((190, 32), 6057), ((154, 224), 6005), ((164, 156), 5673), ((156, 224), 5529), ((165, 128), 5475), ((164, 163), 5444), ((164, 167), 5366), ((165, 10), 5361), ((167, 224), 5340), ((164, 151), 5214), ((151, 224), 5173), ((139, 224), 5171), ((130, 224), 5105), ((163, 224), 5007), ((128, 224), 4738), ((131, 32), 4620), ((164, 178), 4477), ((178, 224), 4393), ((135, 32), 4351), ((191, 32), 4289), ((165, 131), 4251), ((164, 165), 4024), ((165, 224), 3793), ((165, 136), 3607), ((139, 32), 3578), ((154, 32), 3378), ((136, 224), 3344), ((141, 32), 3257), ((175, 32), 3161), ((164, 172), 2771), ((172, 224), 2768), ((165, 130), 2715), ((168, 32), 2385), ((165, 32), 1948), ((181, 32), 1711), ((133, 224), 1506), ((129, 32), 1505), ((164, 133), 1504), ((165, 140), 1493), ((164, 159), 1378), ((159, 224), 1363), ((164, 189), 1347), ((189, 224), 1347), ((164, 158), 1049), ((158, 224), 1019), ((164, 150), 1006), ((150, 224), 994), ((184, 32), 929), ((176, 32), 900), ((164, 155), 882), ((164, 137), 851), ((137, 224), 851), ((155, 224), 830), ((164, 161), 815), ((161, 224), 811), ((140, 224), 799), ((164, 152), 746), ((152, 224), 737), ((128, 32), 722), ((164, 10), 717), ((140, 32), 691), ((164, 143), 689), ((143, 224), 689), ((164, 153), 688), ((153, 224), 688), ((164, 160), 655), ((174, 32), 607), ((160, 224), 579), ((164, 135), 539), ((163, 32), 428), ((164, 134), 391), ((134, 224), 389), ((185, 32), 295), ((136, 32), 294), ((171, 224), 245), ((164, 171), 230), ((183, 32), 181), ((58, 32), 174), ((156, 32), 141), ((164, 139), 140), ((182, 32), 119), ((164, 162), 115), ((162, 224), 108), ((166, 32), 102), ((170, 32), 77), ((160, 32), 76), ((149, 32), 71), ((178, 32), 69), ((167, 32), 65), ((190, 58), 55), ((39, 224), 53), ((44, 32), 52), ((155, 32), 51), ((39, 32), 46), ((58, 224), 45), ((138, 224), 43), ((165, 167), 41), ((164, 138), 40), ((151, 32), 37), ((164, 58), 35), ((164, 136), 32), ((158, 32), 30), ((45, 39), 29), ((165, 168), 28), ((168, 58), 25), ((165, 173), 24), ((165, 169), 22), ((165, 171), 22), ((191, 44), 22), ((165, 166), 20), ((32, 39), 19), ((164, 129), 18), ((165, 170), 17), ((33, 32), 17), ((154, 45), 17), ((191, 39), 17), ((164, 157), 15), ((157, 224), 15), ((191, 58), 14), ((178, 58), 14), ((159, 32), 14), ((169, 224), 13), ((152, 32), 12), ((165, 145), 11), ((165, 132), 11), ((132, 224), 11), ((128, 58), 11), ((150, 32), 11), ((165, 175), 10), ((141, 39), 10), ((145, 224), 9), ((165, 174), 9), ((164, 144), 9), ((144, 224), 9), ((131, 39), 9), ((173, 32), 9), ((141, 44), 8), ((45, 224), 8), ((162, 32), 8), ((129, 58), 7), ((32, 58), 7), ((148, 224), 7), ((169, 10), 7), ((164, 147), 7), ((147, 224), 7), ((163, 58), 7), ((175, 58), 7), ((165, 172), 6), ((164, 148), 6), ((176, 58), 6), ((226, 128), 6), ((172, 32), 6), ((131, 44), 6), ((135, 10), 6), ((165, 138), 6), ((171, 10), 5), ((46, 32), 5), ((181, 58), 5), ((131, 10), 5), ((166, 10), 4), ((151, 58), 4), ((161, 32), 4), ((63, 224), 4), ((190, 10), 4), ((175, 10), 4), ((135, 44), 4), ((139, 33), 4), ((141, 45), 4), ((135, 58), 4), ((168, 10), 4), ((172, 10), 3), ((174, 10), 3), ((131, 33), 3), ((46, 224), 3), ((40, 224), 3), ((33, 224), 3), ((156, 58), 3), ((164, 39), 3), ((175, 44), 3), ((10, 39), 3), ((131, 45), 3), ((174, 58), 3), ((184, 58), 3), ((129, 10), 3), ((138, 32), 3), ((170, 10), 3), ((145, 32), 2), ((130, 58), 2), ((167, 10), 2), ((171, 32), 2), ((190, 46), 2), ((10, 40), 2), ((164, 41), 2), ((41, 10), 2), ((169, 32), 2), ((46, 10), 2), ((190, 226), 2), ((128, 157), 2), ((157, 32), 2), ((95, 224), 2), ((173, 10), 2), ((154, 10), 2), ((32, 46), 2), ((170, 33), 2), ((174, 33), 2), ((168, 39), 2), ((32, 226), 2), ((128, 152), 2), ((154, 39), 2), ((176, 45), 2), ((184, 44), 2), ((58, 45), 2), ((164, 46), 2), ((165, 133), 2), ((173, 58), 2), ((141, 10), 2), ((134, 32), 2), ((170, 58), 2), ((44, 224), 2), ((33, 10), 2), ((182, 58), 1), ((181, 46), 1), ((165, 152), 1), ((165, 46), 1), ((183, 10), 1), ((164, 188), 1), ((188, 32), 1), ((128, 63), 1), ((128, 33), 1), ((128, 95), 1), ((141, 33), 1), ((184, 46), 1), ((129, 63), 1), ((185, 39), 1), ((128, 44), 1), ((184, 10), 1), ((129, 39), 1), ((155, 44), 1), ((181, 44), 1), ((175, 45), 1), ((184, 45), 1), ((181, 39), 1), ((168, 33), 1), ((184, 33), 1), ((140, 39), 1), ((175, 39), 1), ((140, 44), 1), ((182, 45), 1), ((149, 44), 1), ((149, 33), 1), ((129, 45), 1), ((130, 44), 1), ((164, 177), 1), ((177, 224), 1), ((191, 33), 1), ((185, 45), 1), ((163, 44), 1), ((140, 33), 1), ((32, 91), 1), ((91, 224), 1), ((191, 63), 1), ((178, 44), 1), ((165, 146), 1), ((146, 224), 1), ((165, 144), 1), ((144, 32), 1), ((190, 33), 1), ((164, 33), 1), ((139, 58), 1), ((32, 40), 1), ((136, 33), 1), ((164, 179), 1), ((179, 224), 1), ((32, 63), 1), ((139, 226), 1), ((128, 162), 1), ((182, 10), 1), ((190, 45), 1), ((168, 46), 1), ((159, 58), 1), ((150, 58), 1), ((190, 63), 1), ((63, 32), 1), ((139, 10), 1), ((168, 226), 1), ((128, 148), 1), ((191, 45), 1), ((32, 44), 1), ((190, 196), 1), ((196, 129), 1), ((163, 10), 1), ((135, 33), 1), ((190, 95), 1), ((149, 45), 1), ((181, 45), 1), ((149, 58), 1)]\n"
+     ]
+    }
+   ],
+   "source": [
+    "def get_pair_stats(toks): \n",
+    "    pair_stats = {}\n",
+    "    for pair in zip(toks, toks[1:]):\n",
+    "        pair_stats[pair] = pair_stats.get(pair, 0) + 1\n",
+    "    \n",
+    "    return pair_stats\n",
+    "\n",
+    "stats = get_pair_stats(tokens)\n",
+    "reverse_stats = {v:k for (k,v) in stats.items()}\n",
+    "type(reverse_stats)\n",
+    "print(reverse_stats)\n",
+    "print (sorted(((v,k) for k,v in stats.items()), reverse=True))\n",
+    "print(sorted(stats.items(), key=lambda item: item[1], reverse=True))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(224, 164)"
+      ]
+     },
+     "execution_count": 56,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "top_pair = max(stats, key=stats.get) \n",
+    "# top_pair = max(reverse_stats)\n",
+    "top_pair"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 57,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "('à', '¤')"
+      ]
+     },
+     "execution_count": 57,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chr(224),chr(164)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 72,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(1719701, 1311431, 408270)"
+      ]
+     },
+     "execution_count": 72,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# def merge(toks, pair, newTok):\n",
+    "#     newToks = [newTok if x == pair else x[0] for x in zip(toks, toks[1:]) ]\n",
+    "#     return newToks\n",
+    "\n",
+    "def merge(toks, pair, newTok):\n",
+    "    newToks = []\n",
+    "    i = 0\n",
+    "    while i < len(toks):\n",
+    "        if i < len(toks) - 1 and (toks[i], toks[i+1]) == pair:\n",
+    "            newToks.append(pair)\n",
+    "            i += 2\n",
+    "        else:\n",
+    "            newToks.append(toks[i])\n",
+    "            i +=1\n",
+    "    return newToks\n",
+    "\n",
+    "\n",
+    "updatedTokens = merge(tokens, (224, 164), 256)\n",
+    "len(tokens), len(updatedTokens), len(tokens)-len(updatedTokens)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "replaced topPair (224, 164)'s 449 occurrences with 256\n",
+      "replaced topPair (224, 165)'s 150 occurrences with 257\n",
+      "replaced topPair (32, 256)'s 74 occurrences with 258\n",
+      "replaced topPair (257, 141)'s 67 occurrences with 259\n",
+      "replaced topPair (259, 256)'s 55 occurrences with 260\n",
+      "replaced topPair (256, 190)'s 46 occurrences with 261\n",
+      "replaced topPair (256, 164)'s 34 occurrences with 262\n",
+      "replaced topPair (256, 191)'s 33 occurrences with 263\n",
+      "replaced topPair (256, 174)'s 31 occurrences with 264\n",
+      "replaced topPair (256, 168)'s 22 occurrences with 265\n",
+      "replaced topPair (257, 135)'s 22 occurrences with 266\n",
+      "replaced topPair (261, 256)'s 20 occurrences with 267\n",
+      "replaced topPair (263, 256)'s 18 occurrences with 268\n",
+      "replaced topPair (261, 258)'s 14 occurrences with 269\n",
+      "replaced topPair (256, 130)'s 14 occurrences with 270\n",
+      "replaced topPair (257, 128)'s 13 occurrences with 271\n",
+      "replaced topPair (270, 258)'s 13 occurrences with 272\n",
+      "replaced topPair (256, 176)'s 12 occurrences with 273\n",
+      "replaced topPair (260, 176)'s 11 occurrences with 274\n",
+      "replaced topPair (256, 181)'s 11 occurrences with 275\n",
+      "replaced topPair (256, 170)'s 10 occurrences with 276\n",
+      "replaced topPair (257, 129)'s 10 occurrences with 277\n",
+      "replaced topPair (258, 164)'s 10 occurrences with 278\n",
+      "replaced topPair (260, 181)'s 9 occurrences with 279\n",
+      "replaced topPair (260, 175)'s 8 occurrences with 280\n",
+      "replaced topPair (256, 149)'s 8 occurrences with 281\n",
+      "replaced topPair (256, 175)'s 8 occurrences with 282\n",
+      "replaced topPair (257, 164)'s 8 occurrences with 283\n",
+      "replaced topPair (257, 165)'s 8 occurrences with 284\n",
+      "replaced topPair (257, 131)'s 8 occurrences with 285\n",
+      "replaced topPair (266, 258)'s 8 occurrences with 286\n",
+      "replaced topPair (256, 184)'s 7 occurrences with 287\n",
+      "replaced topPair (262, 260)'s 7 occurrences with 288\n",
+      "replaced topPair (283, 258)'s 7 occurrences with 289\n",
+      "replaced topPair (284, 10)'s 7 occurrences with 290\n",
+      "replaced topPair (257, 136)'s 6 occurrences with 291\n",
+      "replaced topPair (256, 131)'s 6 occurrences with 292\n",
+      "replaced topPair (266, 256)'s 6 occurrences with 293\n",
+      "replaced topPair (262, 259)'s 6 occurrences with 294\n",
+      "replaced topPair (257, 139)'s 6 occurrences with 295\n",
+      "replaced topPair (267, 184)'s 5 occurrences with 296\n",
+      "replaced topPair (262, 268)'s 5 occurrences with 297\n",
+      "replaced topPair (264, 268)'s 5 occurrences with 298\n",
+      "replaced topPair (264, 259)'s 5 occurrences with 299\n",
+      "[256, 154, 271, 273, 267, 163, 280, 276, 296, 280, 267, 156, 260, 156, 265, 281, 287, 280, 258, 149, 265, 280, 269, 168, 266, 282, 272, 170, 274, 297, 156, 260, 158, 269, 174, 264, 258, 166, 288, 164, 276, 257, 130, 273, 279, 261, 289, 175, 256, 165, 296, 277, 256, 150, 272, 151, 256, 154, 260, 155, 262, 277, 258, 176, 267, 156, 276, 277, 262, 274, 271, 258, 181, 265, 272, 184, 264, 256, 151, 274, 269, 184, 256, 185, 258, 184, 273, 279, 273, 288, 168, 291, 292, 290, 256, 133, 256, 156, 271, 275, 265, 267, 185, 293, 163, 258, 174, 282, 269, 168, 285, 256, 182, 270, 287, 269, 149, 285, 262, 269, 170, 274, 297, 156, 260, 158, 269, 168, 268, 175, 264, 266, 265, 278, 267, 181, 294, 283, 278, 279, 282, 269, 185, 263, 258, 172, 267, 178, 280, 261, 294, 258, 170, 274, 297, 170, 265, 260, 168, 264, 266, 262, 294, 278, 265, 260, 174, 269, 166, 256, 185, 293, 166, 259, 258, 181, 293, 163, 277, 298, 181, 261, 288, 174, 276, 277, 256, 183, 260, 170, 299, 290, 273, 261, 264, 293, 163, 258, 175, 256, 166, 263, 278, 286, 170, 267, 170, 286, 149, 268, 130, 256, 154, 263, 288, 149, 285, 262, 264, 256, 182, 295, 256, 173, 265, 299, 289, 133, 276, 281, 267, 176, 292, 258, 149, 258, 135, 256, 185, 278, 286, 181, 291, 256, 166, 293, 185, 280, 269, 166, 273, 260, 182, 263, 262, 295, 256, 189, 256, 167, 264, 266, 290, 264, 285, 256, 151, 271, 275, 295, 288, 171, 277, 256, 178, 260, 178, 265, 282, 265, 269, 174, 285, 256, 166, 277, 256, 182, 271, 256, 178, 269, 174, 265, 287, 279, 263, 265, 271, 289, 133, 276, 281, 267, 176, 272, 149, 298, 181, 278, 286, 149, 273, 295, 262, 263, 258, 156, 265, 281, 261, 288, 174, 256, 156, 261, 290, 265, 265, 277, 258, 170, 273, 280, 267, 170, 260, 164, 264, 266, 275, 272, 164, 286, 170, 267, 170, 286, 176, 261, 264, 275, 268, 181, 296, 265, 299, 289, 149, 263, 264, 293, 173, 268, 131, 258, 149, 285, 276, 256, 163, 291, 273, 260, 173, 257, 130, 282, 292, 258, 170, 261, 262, 281, 291, 273, 276, 263, 278, 286, 149, 285, 262, 291, 292, 290, 276, 274, 297, 156, 260, 158, 261, 262, 272, 174, 282, 269, 164, 267, 181, 294, 278, 279, 282, 295, 281, 260, 164, 272, 166, 266, 275, 263, 258, 182, 285, 256, 163, 279, 262, 261, 289, 176, 261, 264, 272, 175, 256, 166, 256, 173, 268, 183, 266, 281, 267, 175, 278, 279, 298, 185, 267, 151, 262, 264, 256, 172, 274, 275, 271, 292, 290, 262, 288, 164, 279, 266, 262, 294, 258, 184, 264, 297, 149, 274, 264, 280, 258, 168, 268, 176, 282, 272, 151, 265, 260, 164, 277, 298, 154, 260, 155, 287, 263, 289, 174, 291, 256, 165, 268, 178, 271, 264, 276, 263, 258, 175, 269, 185, 263, 278, 279, 264, 271, 281, 260, 183, 287, 286, 154, 271, 273, 275, 296, 263, 265, 271, 299, 290, 256, 143, 275, 272, 172, 274, 277, 275, 265, 260, 164, 272, 170, 263, 262, 273, 272, 176, 261, 264, 292, 258, 184, 264, 260, 170, 274, 287, 260, 165, 263, 262, 295, 258, 181, 265, 299, 289, 133, 275, 267, 149, 260, 182, 268, 176, 287, 264, 296, 271, 265, 298, 166, 272, 181, 256, 154, 265, 264, 256, 172, 274, 275, 271, 294, 284]\n",
+      "Original tokens length 1878, while updated tokens length 596\n",
+      "Compression Ratio 3.15\n",
+      "{0: b'\\x00', 1: b'\\x01', 2: b'\\x02', 3: b'\\x03', 4: b'\\x04', 5: b'\\x05', 6: b'\\x06', 7: b'\\x07', 8: b'\\x08', 9: b'\\t', 10: b'\\n', 11: b'\\x0b', 12: b'\\x0c', 13: b'\\r', 14: b'\\x0e', 15: b'\\x0f', 16: b'\\x10', 17: b'\\x11', 18: b'\\x12', 19: b'\\x13', 20: b'\\x14', 21: b'\\x15', 22: b'\\x16', 23: b'\\x17', 24: b'\\x18', 25: b'\\x19', 26: b'\\x1a', 27: b'\\x1b', 28: b'\\x1c', 29: b'\\x1d', 30: b'\\x1e', 31: b'\\x1f', 32: b' ', 33: b'!', 34: b'\"', 35: b'#', 36: b'$', 37: b'%', 38: b'&', 39: b\"'\", 40: b'(', 41: b')', 42: b'*', 43: b'+', 44: b',', 45: b'-', 46: b'.', 47: b'/', 48: b'0', 49: b'1', 50: b'2', 51: b'3', 52: b'4', 53: b'5', 54: b'6', 55: b'7', 56: b'8', 57: b'9', 58: b':', 59: b';', 60: b'<', 61: b'=', 62: b'>', 63: b'?', 64: b'@', 65: b'A', 66: b'B', 67: b'C', 68: b'D', 69: b'E', 70: b'F', 71: b'G', 72: b'H', 73: b'I', 74: b'J', 75: b'K', 76: b'L', 77: b'M', 78: b'N', 79: b'O', 80: b'P', 81: b'Q', 82: b'R', 83: b'S', 84: b'T', 85: b'U', 86: b'V', 87: b'W', 88: b'X', 89: b'Y', 90: b'Z', 91: b'[', 92: b'\\\\', 93: b']', 94: b'^', 95: b'_', 96: b'`', 97: b'a', 98: b'b', 99: b'c', 100: b'd', 101: b'e', 102: b'f', 103: b'g', 104: b'h', 105: b'i', 106: b'j', 107: b'k', 108: b'l', 109: b'm', 110: b'n', 111: b'o', 112: b'p', 113: b'q', 114: b'r', 115: b's', 116: b't', 117: b'u', 118: b'v', 119: b'w', 120: b'x', 121: b'y', 122: b'z', 123: b'{', 124: b'|', 125: b'}', 126: b'~', 127: b'\\x7f', 128: b'\\x80', 129: b'\\x81', 130: b'\\x82', 131: b'\\x83', 132: b'\\x84', 133: b'\\x85', 134: b'\\x86', 135: b'\\x87', 136: b'\\x88', 137: b'\\x89', 138: b'\\x8a', 139: b'\\x8b', 140: b'\\x8c', 141: b'\\x8d', 142: b'\\x8e', 143: b'\\x8f', 144: b'\\x90', 145: b'\\x91', 146: b'\\x92', 147: b'\\x93', 148: b'\\x94', 149: b'\\x95', 150: b'\\x96', 151: b'\\x97', 152: b'\\x98', 153: b'\\x99', 154: b'\\x9a', 155: b'\\x9b', 156: b'\\x9c', 157: b'\\x9d', 158: b'\\x9e', 159: b'\\x9f', 160: b'\\xa0', 161: b'\\xa1', 162: b'\\xa2', 163: b'\\xa3', 164: b'\\xa4', 165: b'\\xa5', 166: b'\\xa6', 167: b'\\xa7', 168: b'\\xa8', 169: b'\\xa9', 170: b'\\xaa', 171: b'\\xab', 172: b'\\xac', 173: b'\\xad', 174: b'\\xae', 175: b'\\xaf', 176: b'\\xb0', 177: b'\\xb1', 178: b'\\xb2', 179: b'\\xb3', 180: b'\\xb4', 181: b'\\xb5', 182: b'\\xb6', 183: b'\\xb7', 184: b'\\xb8', 185: b'\\xb9', 186: b'\\xba', 187: b'\\xbb', 188: b'\\xbc', 189: b'\\xbd', 190: b'\\xbe', 191: b'\\xbf', 192: b'\\xc0', 193: b'\\xc1', 194: b'\\xc2', 195: b'\\xc3', 196: b'\\xc4', 197: b'\\xc5', 198: b'\\xc6', 199: b'\\xc7', 200: b'\\xc8', 201: b'\\xc9', 202: b'\\xca', 203: b'\\xcb', 204: b'\\xcc', 205: b'\\xcd', 206: b'\\xce', 207: b'\\xcf', 208: b'\\xd0', 209: b'\\xd1', 210: b'\\xd2', 211: b'\\xd3', 212: b'\\xd4', 213: b'\\xd5', 214: b'\\xd6', 215: b'\\xd7', 216: b'\\xd8', 217: b'\\xd9', 218: b'\\xda', 219: b'\\xdb', 220: b'\\xdc', 221: b'\\xdd', 222: b'\\xde', 223: b'\\xdf', 224: b'\\xe0', 225: b'\\xe1', 226: b'\\xe2', 227: b'\\xe3', 228: b'\\xe4', 229: b'\\xe5', 230: b'\\xe6', 231: b'\\xe7', 232: b'\\xe8', 233: b'\\xe9', 234: b'\\xea', 235: b'\\xeb', 236: b'\\xec', 237: b'\\xed', 238: b'\\xee', 239: b'\\xef', 240: b'\\xf0', 241: b'\\xf1', 242: b'\\xf2', 243: b'\\xf3', 244: b'\\xf4', 245: b'\\xf5', 246: b'\\xf6', 247: b'\\xf7', 248: b'\\xf8', 249: b'\\xf9', 250: b'\\xfa', 251: b'\\xfb', 252: b'\\xfc', 253: b'\\xfd', 254: b'\\xfe', 255: b'\\xff', 256: b'\\xe0\\xa4', 257: b'\\xe0\\xa5', 258: b' \\xe0\\xa4', 259: b'\\xe0\\xa5\\x8d', 260: b'\\xe0\\xa5\\x8d\\xe0\\xa4', 261: b'\\xe0\\xa4\\xbe', 262: b'\\xe0\\xa4\\xa4', 263: b'\\xe0\\xa4\\xbf', 264: b'\\xe0\\xa4\\xae', 265: b'\\xe0\\xa4\\xa8', 266: b'\\xe0\\xa5\\x87', 267: b'\\xe0\\xa4\\xbe\\xe0\\xa4', 268: b'\\xe0\\xa4\\xbf\\xe0\\xa4', 269: b'\\xe0\\xa4\\xbe \\xe0\\xa4', 270: b'\\xe0\\xa4\\x82', 271: b'\\xe0\\xa5\\x80', 272: b'\\xe0\\xa4\\x82 \\xe0\\xa4', 273: b'\\xe0\\xa4\\xb0', 274: b'\\xe0\\xa5\\x8d\\xe0\\xa4\\xb0', 275: b'\\xe0\\xa4\\xb5', 276: b'\\xe0\\xa4\\xaa', 277: b'\\xe0\\xa5\\x81', 278: b' \\xe0\\xa4\\xa4', 279: b'\\xe0\\xa5\\x8d\\xe0\\xa4\\xb5', 280: b'\\xe0\\xa5\\x8d\\xe0\\xa4\\xaf', 281: b'\\xe0\\xa4\\x95', 282: b'\\xe0\\xa4\\xaf', 283: b'\\xe0\\xa5\\xa4', 284: b'\\xe0\\xa5\\xa5', 285: b'\\xe0\\xa5\\x83', 286: b'\\xe0\\xa5\\x87 \\xe0\\xa4', 287: b'\\xe0\\xa4\\xb8', 288: b'\\xe0\\xa4\\xa4\\xe0\\xa5\\x8d\\xe0\\xa4', 289: b'\\xe0\\xa5\\xa4 \\xe0\\xa4', 290: b'\\xe0\\xa5\\xa5\\n', 291: b'\\xe0\\xa5\\x88', 292: b'\\xe0\\xa4\\x83', 293: b'\\xe0\\xa5\\x87\\xe0\\xa4', 294: b'\\xe0\\xa4\\xa4\\xe0\\xa5\\x8d', 295: b'\\xe0\\xa5\\x8b', 296: b'\\xe0\\xa4\\xbe\\xe0\\xa4\\xb8', 297: b'\\xe0\\xa4\\xa4\\xe0\\xa4\\xbf\\xe0\\xa4', 298: b'\\xe0\\xa4\\xae\\xe0\\xa4\\xbf\\xe0\\xa4', 299: b'\\xe0\\xa4\\xae\\xe0\\xa5\\x8d'}\n",
+      "{0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8, 9: 9, 10: 10, 11: 11, 12: 12, 13: 13, 14: 14, 15: 15, 16: 16, 17: 17, 18: 18, 19: 19, 20: 20, 21: 21, 22: 22, 23: 23, 24: 24, 25: 25, 26: 26, 27: 27, 28: 28, 29: 29, 30: 30, 31: 31, 32: 32, 33: 33, 34: 34, 35: 35, 36: 36, 37: 37, 38: 38, 39: 39, 40: 40, 41: 41, 42: 42, 43: 43, 44: 44, 45: 45, 46: 46, 47: 47, 48: 48, 49: 49, 50: 50, 51: 51, 52: 52, 53: 53, 54: 54, 55: 55, 56: 56, 57: 57, 58: 58, 59: 59, 60: 60, 61: 61, 62: 62, 63: 63, 64: 64, 65: 65, 66: 66, 67: 67, 68: 68, 69: 69, 70: 70, 71: 71, 72: 72, 73: 73, 74: 74, 75: 75, 76: 76, 77: 77, 78: 78, 79: 79, 80: 80, 81: 81, 82: 82, 83: 83, 84: 84, 85: 85, 86: 86, 87: 87, 88: 88, 89: 89, 90: 90, 91: 91, 92: 92, 93: 93, 94: 94, 95: 95, 96: 96, 97: 97, 98: 98, 99: 99, 100: 100, 101: 101, 102: 102, 103: 103, 104: 104, 105: 105, 106: 106, 107: 107, 108: 108, 109: 109, 110: 110, 111: 111, 112: 112, 113: 113, 114: 114, 115: 115, 116: 116, 117: 117, 118: 118, 119: 119, 120: 120, 121: 121, 122: 122, 123: 123, 124: 124, 125: 125, 126: 126, 127: 127, 128: 128, 129: 129, 130: 130, 131: 131, 132: 132, 133: 133, 134: 134, 135: 135, 136: 136, 137: 137, 138: 138, 139: 139, 140: 140, 141: 141, 142: 142, 143: 143, 144: 144, 145: 145, 146: 146, 147: 147, 148: 148, 149: 149, 150: 150, 151: 151, 152: 152, 153: 153, 154: 154, 155: 155, 156: 156, 157: 157, 158: 158, 159: 159, 160: 160, 161: 161, 162: 162, 163: 163, 164: 164, 165: 165, 166: 166, 167: 167, 168: 168, 169: 169, 170: 170, 171: 171, 172: 172, 173: 173, 174: 174, 175: 175, 176: 176, 177: 177, 178: 178, 179: 179, 180: 180, 181: 181, 182: 182, 183: 183, 184: 184, 185: 185, 186: 186, 187: 187, 188: 188, 189: 189, 190: 190, 191: 191, 192: 192, 193: 193, 194: 194, 195: 195, 196: 196, 197: 197, 198: 198, 199: 199, 200: 200, 201: 201, 202: 202, 203: 203, 204: 204, 205: 205, 206: 206, 207: 207, 208: 208, 209: 209, 210: 210, 211: 211, 212: 212, 213: 213, 214: 214, 215: 215, 216: 216, 217: 217, 218: 218, 219: 219, 220: 220, 221: 221, 222: 222, 223: 223, 224: 224, 225: 225, 226: 226, 227: 227, 228: 228, 229: 229, 230: 230, 231: 231, 232: 232, 233: 233, 234: 234, 235: 235, 236: 236, 237: 237, 238: 238, 239: 239, 240: 240, 241: 241, 242: 242, 243: 243, 244: 244, 245: 245, 246: 246, 247: 247, 248: 248, 249: 249, 250: 250, 251: 251, 252: 252, 253: 253, 254: 254, 255: 255, 256: 42208, 257: 42464, 258: 10805280, 259: 9283040, 260: 708142015968, 261: 12494048, 262: 10790112, 263: 12559584, 264: 11445472, 265: 11052256, 266: 8889824, 267: 708145226976, 268: 708145292512, 269: 181282528994528, 270: 8561888, 271: 8431072, 272: 181282525062368, 273: 11576544, 274: 194222188504544, 275: 11904224, 276: 11183328, 277: 8496608, 278: 2762268704, 279: 199719746643424, 280: 193122676876768, 281: 9807072, 282: 11511008, 283: 10790368, 284: 10855904, 285: 8627680, 286: 181282525390304, 287: 12100832, 288: 11880651560581375200, 289: 181282527290848, 290: 178628064, 291: 8955360, 292: 8627424, 293: 708141622752, 294: 155743578006752, 295: 9151968, 296: 203018284737760, 297: 11880706531867796704, 298: 11880706531868452064, 299: 155743578662112}\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "regex.Regex(\"\\n            |[\\\\u0900-\\\\u097F](?![\\\\u0964\\\\u0965])+          # Match Hindi words (Devanagari script)\\n            |[\\\\u0966-\\\\u096F]+         # Match Hindi digits (०-९)\\n            |[a-zA-Z]+                # Match English words (Latin script)\\n            |[0-9]+                   # Match Latin digits (0-9)\\n            |\\\\s+                      # Match whitespace (spaces, tabs, newlines)\\n            |'[^\\\\r\\\\n\\\\p{L}\\\\p{N}]*\\\\p{L}+  # Match apostrophes followed by letters\\n            |\\\\p{N}{1,3}               # Match numbers (1 to 3 digits)\\n            |[^\\\\s\\\\p{L}\\\\p{N}]+         # Match non-letter, non-number special characters\\n            |\\\\s*[\\\\r\\\\n]                # Match line breaks and leading spaces\\n            |\\\\s+(?!\\\\S)                # Match trailing whitespace\\n        \", flags=regex.X | regex.V0)"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "#### Following methods are needed to create a vocab using BPA Algo ####\n",
+    "\n",
+    "# Get dictonary with key as token pairs and value as number of occurence\n",
+    "def get_pair_stats(toks): \n",
+    "    pair_stats = {}\n",
+    "    for pair in zip(toks, toks[1:]):\n",
+    "        pair_stats[pair] = pair_stats.get(pair, 0) + 1\n",
+    "    \n",
+    "    return pair_stats\n",
+    "\n",
+    "# Replaces all the occurences of pair in the tokens list with a new token\n",
+    "def merge(toks, pair, newTok):\n",
+    "    newToks = []\n",
+    "    i = 0\n",
+    "    while i < len(toks):\n",
+    "        if i < len(toks) - 1 and (toks[i], toks[i+1]) == pair:\n",
+    "            newToks.append(newTok)\n",
+    "            i += 2\n",
+    "        else:\n",
+    "            newToks.append(toks[i])\n",
+    "            i +=1\n",
+    "    return newToks\n",
+    "\n",
+    "# paired_tokens_vocab - key as integer pairs (p0 & p1 would be between 0 & max_vocab_size) and value would be integer val between 0 - max_vocab_size\n",
+    "def bpaAlgo(toks, vocab_size):\n",
+    "    total_run = vocab_size - 256\n",
+    "\n",
+    "    paired_tokens_vocab = {}\n",
+    "    newToks = list(toks)\n",
+    "    for i in range(0, total_run):\n",
+    "        pair_stats = get_pair_stats(newToks)\n",
+    "        top_pair = max(pair_stats, key=pair_stats.get)\n",
+    "        newToks = merge(newToks, top_pair, len(paired_tokens_vocab)+256)\n",
+    "        print(f\"replaced topPair {top_pair}'s {pair_stats.get(top_pair)} occurrences with {len(paired_tokens_vocab)+256}\")\n",
+    "        paired_tokens_vocab[top_pair] = len(paired_tokens_vocab)+256\n",
+    "    \n",
+    "    print(newToks)\n",
+    "    print(f\"Original tokens length {len(toks)}, while updated tokens length {len(newToks)}\")\n",
+    "    print(f\"Compression Ratio {len(toks)/len(newToks):.2f}\")\n",
+    "    return paired_tokens_vocab\n",
+    "\n",
+    "# vocab - key as int values from 0 - max(vocabSize) and values are bytes\n",
+    "def create_vocab(paired_tokens_vocab):\n",
+    "    vocab = {i: bytes([i]) for i in range(256)} # 0-255 bytes as is\n",
+    "\n",
+    "    for (p0, p1), i in paired_tokens_vocab.items():\n",
+    "        vocab[i] = vocab[p0] + vocab[p1] # as we are iterating in order, we are deriving the values of later vocab from previous vocab value bytes\n",
+    "\n",
+    "    return vocab\n",
+    "\n",
+    "small_text = \"\"\"चीराण्यपास्याज्जनकस्य कन्या नेयं प्रतिज्ञा मम दत्तपूर्वा। यथासुखं गच्छतु राजपुत्री वनं समग्रा सह सर्वरत्नैः॥\n",
+    "अजीवनाहेण मया नृशंसा कृता प्रतिज्ञा नियमेन तावत्। त्वया हि बाल्यात् प्रतिपन्नमेतत् तन्मा दहेद् वेणुमिवात्मपुष्पम्॥\n",
+    "रामेण यदि ते पापे किंचित्कृतमशोभनम्। अपकारः क इह ते वैदेह्या दर्शितोऽधमे॥\n",
+    "मृगीवोत्फुल्लनयना मृदुशीला मनस्विनी। अपकारं कमिव ते करोति जनकात्मजा॥\n",
+    "ननु पर्याप्तमेवं ते पापे रामविवासनम्। किमेभिः कृपणैर्भूयः पातकैरपि ते कृतैः॥\n",
+    "प्रतिज्ञातं मया तावत् त्वयोक्तं देवि शृण्वता। रामं यदभिषेकाय त्वमिहागतमब्रवीः॥\n",
+    "तत्त्वेतत् समतिक्रम्य निरयं गन्तुमिच्छसि। मैथिलीमपि या हि त्वमीक्षसे चीरवासिनीम्॥\n",
+    "एवं ब्रुवन्तं पितरं रामः सम्प्रस्थितो वनम्। अवाक्शिरसमासीनमिदं वचनमब्रवीत्॥\"\"\"\n",
+    "small_tokens = list(small_text.encode(\"utf-8\"))\n",
+    "paired_tokens_vocab = bpaAlgo(small_tokens, 300)\n",
+    "# print(f\"paired_tokens_vocab: {paired_tokens_vocab}\")\n",
+    "# vocab - key as int values from 0 - max(vocabSize) and values are bytes\n",
+    "vocab = create_vocab(paired_tokens_vocab)\n",
+    "print(vocab)\n",
+    "print({k:int.from_bytes(v, byteorder='little') for (k,v) in vocab.items()})\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(256, 154)\n",
+      "चीराण्यपास्याज्जनकस्य कन्या नेयं प्रतिज्ञा मम दत्तपूर्वा। यथासुखं गच्छतु राजपुत्री वनं समग्रा सह सर्वरत्नैः॥\n",
+      "अजीवनाहेण मया नृशंसा कृता प्रतिज्ञा नियमेन तावत्। त्वया हि बाल्यात् प्रतिपन्नमेतत् तन्मा दहेद् वेणुमिवात्मपुष्पम्॥\n",
+      "रामेण यदि ते पापे किंचित्कृतमशोभनम्। अपकारः क इह ते वैदेह्या दर्शितोऽधमे॥\n",
+      "मृगीवोत्फुल्लनयना मृदुशीला मनस्विनी। अपकारं कमिव ते करोति जनकात्मजा॥\n",
+      "ननु पर्याप्तमेवं ते पापे रामविवासनम्। किमेभिः कृपणैर्भूयः पातकैरपि ते कृतैः॥\n",
+      "प्रतिज्ञातं मया तावत् त्वयोक्तं देवि शृण्वता। रामं यदभिषेकाय त्वमिहागतमब्रवीः॥\n",
+      "तत्त्वेतत् समतिक्रम्य निरयं गन्तुमिच्छसि। मैथिलीमपि या हि त्वमीक्षसे चीरवासिनीम्॥\n",
+      "एवं ब्रुवन्तं पितरं रामः सम्प्रस्थितो वनम्। अवाक्शिरसमासीनमिदं वचनमब्रवीत्॥\n",
+      "True\n"
+     ]
+    }
+   ],
+   "source": [
+    "##### Following methods are used to use the already created vocab ####\n",
+    "# paired_tokens_vocab - key as integer pairs (p0 & p1 would be between 0 & max_vocab_size) and value would be integer val between 0 - max_vocab_size\n",
+    "# vocab - key as int values from 0 - max(vocabSize) and values are bytes\n",
+    "\n",
+    "def gg_decode(intTokens):\n",
+    "    vocalTokenVals = b\"\".join(vocab[vocabKey] for vocabKey in intTokens) # join actual byte values of the keys from vocab\n",
+    "    return vocalTokenVals.decode(\"utf-8\")  # Decode the utf-code byte sequemces back to String\n",
+    "\n",
+    "def gg_encode(text):\n",
+    "    tokens = list(text.encode('utf-8'))\n",
+    "\n",
+    "    while len(tokens) > 1: # as long as the token values can be paired. It will break within loop below\n",
+    "        pair_stats = get_pair_stats(tokens)\n",
+    "        # Get the matching pair from the vocab with least value. That way, if it is not found, we can stop the iteration\n",
+    "        pair = min(pair_stats, key=lambda k : paired_tokens_vocab.get(k, float(\"inf\"))) # find min pair whose min value is decided based on the value found in paired_tokens_vocab\n",
+    "        if pair not in paired_tokens_vocab:\n",
+    "            print(pair)\n",
+    "            return tokens\n",
+    "        newTok = paired_tokens_vocab[pair]\n",
+    "        tokens = merge(tokens, pair, newTok)\n",
+    "small_text2 = gg_decode(gg_encode(small_text))\n",
+    "print(small_text2)\n",
+    "print(small_text2 == small_text)\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import regex as re\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['\\n', 'Hello', \"'ve\", ' world', '123', ' how', \"'s\", ' are', ' you', '!!!?', '\\n', '\\n', 'IA', ' m', ' fine', '\\n']\n"
+     ]
+    }
+   ],
+   "source": [
+    "gpt2pat = re.compile(r\"\"\"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+\"\"\")\n",
+    "\n",
+    "txt = \"\"\"\n",
+    "Hello've world123 how's are you!!!?\n",
+    "\n",
+    "IA m fine\n",
+    "\"\"\"\n",
+    "output = re.findall(gpt2pat, txt)\n",
+    "\n",
+    "print(output)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 62,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['चीराण्यपास्याज्जनकस्य', ' कन्या', ' नेयं', ' प्रतिज्ञा', ' मम', ' दत्तपूर्वा।', ' यथासुखं', ' गच्छतु', ' राजपुत्री', ' वनं', ' समग्रा', ' सह', ' सर्वरत्नैः॥', '\\n', '\\n', 'प्रतिज्ञातं', ' मया', ' तावत्', ' त्वयोक्तं', ' देवि', ' शृण्वता।', ' 123', ' 4', ' 5', ' I', ' am', ' going', ' रामं', ' यदभिषेकाय', ' त्वमिहागतमब्रवीः॥']\n"
+     ]
+    }
+   ],
+   "source": [
+    "# U+090x\n",
+    "# ऀ ँ ऄ अ आ इ ई उ ऊ ऋ ऌ ऍ ऎ ए\n",
+    "\n",
+    "# U+091x\n",
+    "# ऐ ऑ ऒ ओ औ क ख ग घ ङ च छ ज झ ञ ट\n",
+    "\n",
+    "# U+092x\n",
+    "# ठ ड ढ ण त थ द ध न ऩ प फ ब भ म य\n",
+    "\n",
+    "# U+093x\n",
+    "# र ऱ ल ळ ऴ व श ष स हऺऻ ़ ऽ ाि\n",
+    "\n",
+    "# U+094x\n",
+    "# ी ु ू ृ ॄ ॅ ॆ े ै ॉ ॊ ो ौ ् ॎ ॏ\n",
+    "\n",
+    "# U+095x\n",
+    "# ॐ ॑ ॓ ॕ ॖ ॗ क़ ख़ ग़ ज़ ड़ ढ़ फ़ य़\n",
+    "\n",
+    "# U+096x\n",
+    "# ॠ ॡ ॢ । ॥ ० १ २ ३ ४ ५ ६ ७ ८ ९\n",
+    "\n",
+    "# U+097x\n",
+    "# ॰ ॱ ॲ ॳ ॴ ॵ ॶ ॷ ॸ ॹ ॺ ॻ ॼ ॽ ॾ ॿ\n",
+    "\n",
+    "pat = re.compile(r\"\"\"\n",
+    "            \\ ?[\\u0900-\\u097F]+                # Match Sanskit words with an optional space before\n",
+    "            |\\ ?\\d+                             # Matches one or more numerical digits.\n",
+    "            |\\ ?[^\\s\\u0900-\\u097F\\d]+           # Matches any character that is not a space, snaskrit or digit  with an optional space before \n",
+    "            |\\s+(?!\\S)                          # trailing whitespace characters (spaces, tabs, newlines)\n",
+    "            |\\s+                                 # Match whitespace (spaces, tabs, newlines)\n",
+    "            |\\s?[\\r\\n] \n",
+    "        \"\"\", re.VERBOSE)\n",
+    "\n",
+    "small_text = \"\"\"चीराण्यपास्याज्जनकस्य कन्या नेयं प्रतिज्ञा मम दत्तपूर्वा। यथासुखं गच्छतु राजपुत्री वनं समग्रा सह सर्वरत्नैः॥\n",
+    "\n",
+    "प्रतिज्ञातं मया तावत् त्वयोक्तं देवि शृण्वता। 123 4 5 I am going रामं यदभिषेकाय त्वमिहागतमब्रवीः॥\"\"\"\n",
+    "\n",
+    "print(re.findall(pat, small_text))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/tokenizer.py

@@ -0,0 +1,129 @@
+import regex as re
+from tqdm import tqdm
+
+# U+090x
+# ऀ ँ ऄ अ आ इ ई उ ऊ ऋ ऌ ऍ ऎ ए
+
+# U+091x
+# ऐ ऑ ऒ ओ औ क ख ग घ ङ च छ ज झ ञ ट
+
+# U+092x
+# ठ ड ढ ण त थ द ध न ऩ प फ ब भ म य
+
+# U+093x
+# र ऱ ल ळ ऴ व श ष स हऺऻ ़ ऽ ाि
+
+# U+094x
+# ी ु ू ृ ॄ ॅ ॆ े ै ॉ ॊ ो ौ ् ॎ ॏ
+
+# U+095x
+# ॐ ॑ ॓ ॕ ॖ ॗ क़ ख़ ग़ ज़ ड़ ढ़ फ़ य़
+
+# U+096x
+# ॠ ॡ ॢ । ॥ ० १ २ ३ ४ ५ ६ ७ ८ ९
+
+# U+097x
+# ॰ ॱ ॲ ॳ ॴ ॵ ॶ ॷ ॸ ॹ ॺ ॻ ॼ ॽ ॾ ॿ
+def sanskrit_token_preprocessor(text):
+    # pat = re.compile(r"""॥|।| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
+    pat = re.compile(r"""
+            \ ?[\u0900-\u097F]+               # Match Sanskit words with an optional space before
+            |\ ?\d+                        # Matches one or more numerical digits.
+            |\ ?[^\s\u0900-\u097F\d]+    # Matches any character that is not a space, snaskrit or digit  with an optional space before 
+            |\s+(?!\S)                     # trailing whitespace characters (spaces, tabs, newlines)
+            |\s+                            # Match whitespace (spaces, tabs, newlines)
+            |\s?[\r\n] 
+        """, re.VERBOSE)
+    return re.findall(pat, text)
+
+def bpeAlgo(tokensList, total_runs, newTokenStartValue):
+    # tokensList - list of list of tokens
+    paired_tokens_vocab = {}
+    newTokensList = list(tokensList) # copy list to keep original unchanged
+    # newTokensList - list of list of tokens
+    
+    for i in tqdm(range(0, total_runs), desc="Learning BPE"):
+        pair_stats = {}
+        for tokens in newTokensList:
+             pair_stats = get_pair_stats(tokens, pair_stats)
+       
+        # print(pair_stats)
+        if not pair_stats:
+            print(f"Not enought unique pairs to run BPE's {total_runs} runs")
+            raise ValueError(f"Not enough pairs to create a vocab of size {total_runs+256}")
+
+        top_pair = max(pair_stats, key=pair_stats.get)
+        newTokenVal = len(paired_tokens_vocab)+newTokenStartValue
+        # Replace tokens
+        newTokensList = [merge(tokens, top_pair, newTokenVal) for tokens in newTokensList]
+        # print(f"replaced topPair {top_pair}'s {pair_stats.get(top_pair)} occurrences with {len(paired_tokens_vocab)+256}")
+        # Add the new token to paired_tokens_vocab
+        paired_tokens_vocab[top_pair] = newTokenVal
+
+    return paired_tokens_vocab, [tok for toks in newTokensList for tok in toks]
+
+# Get dictonary with key as token pairs and value as number of occurence
+def get_pair_stats(toks, pair_stats): 
+    for pair in zip(toks, toks[1:]):
+        pair_stats[pair] = pair_stats.get(pair, 0) + 1
+    
+    return pair_stats
+
+# Replaces all the occurences of pair in the tokens list with a new token
+def merge(toks, pair, newTok):
+    newToks = []
+    i = 0
+    while i < len(toks):
+        if i < len(toks) - 1 and (toks[i], toks[i+1]) == pair:
+            newToks.append(newTok)
+            i += 2
+        else:
+            newToks.append(toks[i])
+            i += 1
+    return newToks
+
+# vocab - key as int values from 0 - max(vocabSize) and values are bytes
+def create_vocab(paired_tokens_vocab):
+    vocab = {i: bytes([i]) for i in range(256)} # 0-255 bytes as is
+
+    for (p0, p1), i in paired_tokens_vocab.items():
+        vocab[i] = vocab[p0] + vocab[p1] # as we are iterating in order, we are deriving the values of later vocab from previous vocab value bytes
+
+    return vocab
+
+def save_paired_tokens_vocab(filepath, paired_tokens_vocab):
+     with open(filepath, 'w') as f:
+        f.write('Token version 1\n')
+        for k,v in paired_tokens_vocab.items():
+            f.write(f"{k}:{v}\n") 
+
+def save_vocab(filepath, vocab):
+     with open(filepath, 'w') as f:
+        f.write('Token version 1\n')
+        for k,v in vocab.items():
+            f.write(f"{k}:{v}\n") 
+        
+
+
+##### Following methods are used to use the already created vocab ####
+# paired_tokens_vocab - key as integer pairs (p0 & p1 would be between 0 & max_vocab_size) and value would be integer val between 0 - max_vocab_size
+# vocab - key as int values from 0 - max(vocabSize) and values are bytes
+
+def decode(intTokens, vocab):
+    vocalTokenVals = b"".join(vocab[vocabKey] for vocabKey in intTokens) # join actual byte values of the keys from vocab
+    return vocalTokenVals.decode("utf-8")  # Decode the utf-code byte sequemces back to String
+
+def encode(text, paired_tokens_vocab):
+    tokens = list(text.encode('utf-8'))
+
+    while len(tokens) > 1: # as long as the token values can be paired. It will break within loop below
+        pair_stats = get_pair_stats(tokens, {})
+        # Get the matching pair from the vocab with least value. That way, if it is not found, we can stop the iteration
+        pair = min(pair_stats, key=lambda k : paired_tokens_vocab.get(k, float("inf"))) # find min pair whose min value is decided based on the value found in paired_tokens_vocab
+        if pair not in paired_tokens_vocab:
+            return tokens
+        newTok = paired_tokens_vocab[pair]
+        # print(f"Replacing {pair} with {newTok}")
+        tokens = merge(tokens, pair, newTok)
+    
+    return tokens

tests/test_tokenizer.py

@@ -0,0 +1,59 @@
+import pytest
+from src.tokenizer import sanskrit_token_preprocessor, bpeAlgo, create_vocab, encode, decode
+
+@pytest.fixture
+def sample_text():
+    return """चीराण्यपास्याज्जनकस्य कन्या नेयं प्रतिज्ञा मम दत्तपूर्वा।
+अजीवनाहेण मया नृशंसा कृता प्रतिज्ञा नियमेन तावत्। I am going"""
+
+@pytest.fixture
+def small_vocab_size():
+    return 300  # Small vocab size for testing
+
+def test_sanskrit_token_preprocessor():
+    text = "Hello! नमस्ते 123"
+    tokens = sanskrit_token_preprocessor(text)
+    assert isinstance(tokens, list)
+    assert all(isinstance(token, str) for token in tokens)
+
+def test_encode_decode_roundtrip(sample_text, small_vocab_size):
+    # Prepare tokens
+    tokens = sanskrit_token_preprocessor(sample_text)
+    tokens = [list(tok.encode('utf-8')) for tok in tokens]
+    
+    # Train BPE
+    paired_tokens_vocab, _ = bpeAlgo(tokens, small_vocab_size - 256, 256)
+    vocab = create_vocab(paired_tokens_vocab)
+    
+    # Test encode-decode roundtrip
+    encoded = encode(sample_text, paired_tokens_vocab)
+    decoded = decode(encoded, vocab)
+    assert decoded == sample_text
+
+def test_vocab_size_limit(sample_text, small_vocab_size):
+    tokens = sanskrit_token_preprocessor(sample_text)
+    tokens = [list(tok.encode('utf-8')) for tok in tokens]
+    
+    paired_tokens_vocab, _ = bpeAlgo(tokens, small_vocab_size - 256, 256)
+    vocab = create_vocab(paired_tokens_vocab)
+    
+    # Check if vocab size is within limits
+    assert len(vocab) <= small_vocab_size
+
+def test_empty_input():
+    empty_text = ""
+    tokens = sanskrit_token_preprocessor(empty_text)
+    assert tokens == []
+    
+    # Test encode/decode with empty input
+    paired_tokens_vocab = {}
+    vocab = create_vocab(paired_tokens_vocab)
+    encoded = encode(empty_text, paired_tokens_vocab)
+    decoded = decode(encoded, vocab)
+    assert decoded == empty_text
+
+def test_special_characters():
+    special_text = "!@#$%^&*()_+ १२३"
+    tokens = sanskrit_token_preprocessor(special_text)
+    assert isinstance(tokens, list)
+    assert len(tokens) > 0