Update spaCy pipeline

README.md

@@ -9,7 +9,7 @@ language:
 | --- | --- |
 | **Name** | `fr_trf_reflex_nrp` |
 | **Version** | `0.0.0` |
-| **spaCy** | `>=3.8.3,<3.9.0` |
+| **spaCy** | `>=3.8.7,<3.9.0` |
 | **Default Pipeline** | `ner_transformer`, `ner`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
 | **Components** | `ner_transformer`, `ner`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
 | **Vectors** | 0 keys, 0 unique vectors (0 dimensions) |

base_transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9c140cc7cc6da464b19f21950129cdc7421053fb79ff5e34666fc9527905204a
-size 443537896
+oid sha256:273f8e8a1a291514e3dd91408f0e03434995c72b68fcddfb944f35897c8a0429
+size 443538293

config.cfg

@@ -16,7 +16,7 @@ before_creation = null
 after_creation = null
 after_pipeline_creation = null
 batch_size = 1000
-tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
+tokenizer = {"@tokenizers":"customize_tokenizer"}
 vectors = {"@vectors":"spacy.Vectors.v1"}
 
 [components]
@@ -90,7 +90,7 @@ set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotati
 
 [components.ner_transformer.model]
 @architectures = "spacy-transformers.TransformerModel.v3"
-name = "dbmdz/bert-base-german-cased"
+name = "almanach/camembertav2-base"
 mixed_precision = false
 
 [components.ner_transformer.model.get_spans]
@@ -228,21 +228,24 @@ eps = 0.00000001
 learn_rate = 0.001
 
 [training.score_weights]
-ents_f = 0.2
+ents_f = 0.22
 ents_p = 0.0
 ents_r = 0.0
 ents_per_type = null
-pos_acc = 0.1
-morph_acc = 0.1
+pos_acc = 0.0
+morph_acc = 0.11
 morph_per_feat = null
-tag_acc = 0.2
-dep_uas = 0.1
-dep_las = 0.1
+tag_acc = 0.22
+tag_micro_p = null
+tag_micro_r = null
+tag_micro_f = null
+dep_uas = 0.11
+dep_las = 0.11
 dep_las_per_type = null
 sents_p = null
 sents_r = null
 sents_f = 0.0
-lemma_acc = 0.2
+lemma_acc = 0.22
 
 [pretraining]
 

fr_trf_reflex_nrp-0.0.0-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a864066b3962b7185b3eaca0bf0cc55674ffc1a84e7f3a9fa4e3c41e20f50c1f
-size 822780428
+oid sha256:719a5cd2ea2461b701b8535bfbaa591227cb6a63ba60549c1c6f09efcd3c27bc
+size 824582048

meta.json

@@ -7,8 +7,8 @@
   "email":"",
   "url":"",
   "license":"",
-  "spacy_version":">=3.8.3,<3.9.0",
-  "spacy_git_version":"be0fa81",
+  "spacy_version":">=3.8.7,<3.9.0",
+  "spacy_git_version":"4b65aa7",
   "vectors":{
     "width":0,
     "vectors":0,
@@ -586,7 +586,7 @@
 
   ],
   "requirements":[
-    "spacy-transformers>=1.3.8,<1.4.0",
-    "spacy>=3.8.3,<3.9.0"
+    "spacy-transformers>=1.3.9,<1.4.0",
+    "spacy>=3.8.7,<3.9.0"
   ]
 }

ner/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:188374b00a1220f136095477c9f81dcb148c5c35b84d38c03f5bf24098c4e4f2
+oid sha256:4ed9d24f9c390a1c7c7ad6bb84a402dafa34d3210d998e2a365e2830f9590edf
 size 219690

ner/moves

@@ -1 +1 @@
-��movesٴ{"0":{},"1":{"ORG":5328,"LOC":3766,"PER":3605},"2":{"ORG":5328,"LOC":3766,"PER":3605},"3":{"ORG":5328,"LOC":3766,"PER":3605},"4":{"ORG":5328,"LOC":3766,"PER":3605,"":1},"5":{"":1}}�cfg��neg_key�
+��movesٴ{"0":{},"1":{"ORG":6029,"LOC":4464,"PER":3965},"2":{"ORG":6029,"LOC":4464,"PER":3965},"3":{"ORG":6029,"LOC":4464,"PER":3965},"4":{"ORG":6029,"LOC":4464,"PER":3965,"":1},"5":{"":1}}�cfg��neg_key�

ner_transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa876964843bdadeeaf73bebae2103e27ca655f50c96fd666d6e5ffd58d10c8f
-size 440759212
+oid sha256:755e7434a8f1df5e8dede50927501cb06e1264baa504ddb2dd909fbbf1740da6
+size 443538293

use_custom_tokenizer.py

@@ -24,17 +24,26 @@ EXTENDED_LETTER_RANGE = "A-Za-zäöüÄÖÜàòèéìù"
 DATE = r"[0-3][1-9]\.[0-1][1-9]\.[1-2][0-9]{3}"
 TOP_LEVEL_DOMAINS = "ch|at|de|com|edu|org|gov|net|fr|uk|be|es|pl|it|eu|nl|ba|cz|dk|al|ad|bg|by|fi|gr|ie|li|lu|no|pt|ro|rs|ru|se|si|sk"
 
+DOT_BEFORE_WORD = [
+    rf"(?<!www)\.(?=[{EXTENDED_LETTER_RANGE}]{{{i}}})(?!(({TOP_LEVEL_DOMAINS})\b))"
+    for i in range(3, 30)
+]
+
+
 DOT_AFTER_WORD = [
-    rf"(?<!www\.)(?<=([a-zA-ZäöüÄÖÜ]){{{i}}})\.(?!({TOP_LEVEL_DOMAINS}))"
+    rf"(?<!www\.)(?<=([{EXTENDED_LETTER_RANGE}]){{{i}}})\.(?!({TOP_LEVEL_DOMAINS}))"
     for i in range(3, 30)
 ]
 
+
 DOT_AFTER_DATE = rf"(?<=({DATE}))\."
 
 infix_res = [
-    r"[\(\[\]\)]",
+    r"[\(\[\]\)\,\»\>\«\<]",  # brackets, commas and ' that are without space
     r"(?<=\.--)\.",  # DOT after .--
-    rf"\.(?=[{EXTENDED_LETTER_RANGE}]{{3,20}})",  # DOT before word
+    r"(?<=\,)\'",
+    *DOT_BEFORE_WORD,  # DOT before word
+    # rf"(?<!www)\.(?=[{EXTENDED_LETTER_RANGE}]{{3,20}})",
     r"'\.\.",  # e.g., 'Tscheicha'.. -> "Tscheicha" "'..", then split ".." as suffix
     *DOT_AFTER_WORD,  # when there is no space after the dot
     r"[A-Z](?=\. )",  # DOT after capital letter
@@ -45,6 +54,11 @@ LETTER_DOUBLE_ENDING_DOT_VAR_LENGTH = [  # DOT after letter, e.g., A.G., or u.s.
     rf"(?<=([{EXTENDED_LETTER_RANGE}]\.){{{i}}})\." for i in range(1, 30)
 ]
 
+prefix_res = [
+    rf"\.(?=[{EXTENDED_LETTER_RANGE}]{{3,20}})",  # DOT before word
+    *DOT_BEFORE_WORD,  # DOT before word
+]
+
 suffix_res = [
     r"(?<=\d)[\.]",  # DOT after number
     r"(?<=[\.])[\]\)]",  # Closing brackets with DOT before
@@ -63,10 +77,12 @@ NUMBER_SIGN_NUMBER_FIXED = r"(?<=[0-9])[+\*^](?=[0-9])"
 
 
 # Given a nlp object, return a custom tokenizer that splits on special cases and with unwanted tokenization removed
-def custom_tokenizer(nlp):
+def custom_tokenizer(nlp) -> Tokenizer:
     nlp.tokenizer = Tokenizer(nlp.vocab)
 
-    prefix_regex = compile_prefix_regex(nlp.Defaults.prefixes)
+    prefixes = nlp.Defaults.prefixes
+    prefixes += prefix_res
+    prefix_regex = compile_prefix_regex(prefixes)
     nlp.tokenizer.prefix_search = prefix_regex.search
 
     # We use the default infixes and remove some cases that lead to unwanted tokenization.