Remove nlp() from nested_sentences
Browse files
models.py
CHANGED
@@ -10,7 +10,6 @@ def create_nest_sentences(document:str, token_max_length = 1024):
|
|
10 |
sent = []
|
11 |
length = 0
|
12 |
tokenizer = AutoTokenizer.from_pretrained('facebook/bart-large-mnli')
|
13 |
-
tokens = nlp(document)
|
14 |
|
15 |
for sentence in re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])', document.replace("\n", ' ')):
|
16 |
tokens_in_sentence = tokenizer(str(sentence), truncation=False, padding=False)[0] # hugging face transformer tokenizer
|
|
|
10 |
sent = []
|
11 |
length = 0
|
12 |
tokenizer = AutoTokenizer.from_pretrained('facebook/bart-large-mnli')
|
|
|
13 |
|
14 |
for sentence in re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])', document.replace("\n", ' ')):
|
15 |
tokens_in_sentence = tokenizer(str(sentence), truncation=False, padding=False)[0] # hugging face transformer tokenizer
|