Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -4,20 +4,29 @@ from langchain.text_splitter import (
|
|
4 |
RecursiveCharacterTextSplitter,
|
5 |
)
|
6 |
|
|
|
|
|
|
|
7 |
def extract_separators_from_string(separator_str):
|
8 |
try:
|
9 |
separators = separators_str[1:-1].split(", ")
|
10 |
return [separator.replace('"', "").replace("'", "") for separator in separators]
|
11 |
-
except:
|
|
|
12 |
raise gr.Error(f"""
|
13 |
Did not succeed in extracting seperators from string: {separator_str}.
|
14 |
Please type it in the correct format: "['separator_1', 'separator_2', etc]"
|
15 |
""")
|
|
|
|
|
|
|
|
|
|
|
16 |
|
17 |
def chunk(text, length, splitter_selection, separators_str):
|
18 |
separators = extract_separators_from_string(separators_str)
|
19 |
|
20 |
-
if splitter_selection ==
|
21 |
text_splitter = CharacterTextSplitter(
|
22 |
separator="",
|
23 |
chunk_size=length,
|
@@ -27,7 +36,7 @@ def chunk(text, length, splitter_selection, separators_str):
|
|
27 |
)
|
28 |
splits = text_splitter.create_documents([text])
|
29 |
text_splits = [split.page_content for split in splits]
|
30 |
-
elif splitter_selection ==
|
31 |
text_splitter = RecursiveCharacterTextSplitter(
|
32 |
chunk_size=length,
|
33 |
chunk_overlap=0,
|
@@ -71,12 +80,11 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
|
|
71 |
with gr.Row():
|
72 |
split_selection = gr.Dropdown(
|
73 |
choices=[
|
74 |
-
|
75 |
-
|
76 |
],
|
77 |
-
value=
|
78 |
label="Chunking method ",
|
79 |
-
info="How should we split our chunks?",
|
80 |
)
|
81 |
separator_selection = gr.Textbox(
|
82 |
value=["\n\n", "\n", ".", " ", ""],
|
|
|
4 |
RecursiveCharacterTextSplitter,
|
5 |
)
|
6 |
|
7 |
+
LABEL_TEXTSPLITTER = "LangChain's CharacterTextSplitter"
|
8 |
+
LABEL_RECURSIVE = "Langchain's RecursiveCharacterTextSplitter"
|
9 |
+
|
10 |
def extract_separators_from_string(separator_str):
|
11 |
try:
|
12 |
separators = separators_str[1:-1].split(", ")
|
13 |
return [separator.replace('"', "").replace("'", "") for separator in separators]
|
14 |
+
except Exception as e:
|
15 |
+
print(e)
|
16 |
raise gr.Error(f"""
|
17 |
Did not succeed in extracting seperators from string: {separator_str}.
|
18 |
Please type it in the correct format: "['separator_1', 'separator_2', etc]"
|
19 |
""")
|
20 |
+
|
21 |
+
def change_split_selection(text, slider_count, split_selection, separator_selection):
|
22 |
+
separator_selection.update(interactive=(split_selection==LABEL_RECURSIVE))
|
23 |
+
return chunk(text, slider_count, split_selection, separator_selection)
|
24 |
+
|
25 |
|
26 |
def chunk(text, length, splitter_selection, separators_str):
|
27 |
separators = extract_separators_from_string(separators_str)
|
28 |
|
29 |
+
if splitter_selection == LABEL_TEXTSPLITTER:
|
30 |
text_splitter = CharacterTextSplitter(
|
31 |
separator="",
|
32 |
chunk_size=length,
|
|
|
36 |
)
|
37 |
splits = text_splitter.create_documents([text])
|
38 |
text_splits = [split.page_content for split in splits]
|
39 |
+
elif splitter_selection == LABEL_RECURSIVE:
|
40 |
text_splitter = RecursiveCharacterTextSplitter(
|
41 |
chunk_size=length,
|
42 |
chunk_overlap=0,
|
|
|
80 |
with gr.Row():
|
81 |
split_selection = gr.Dropdown(
|
82 |
choices=[
|
83 |
+
LABEL_TEXTSPLITTER,
|
84 |
+
LABEL_RECURSIVE,
|
85 |
],
|
86 |
+
value=LABEL_TEXTSPLITTER,
|
87 |
label="Chunking method ",
|
|
|
88 |
)
|
89 |
separator_selection = gr.Textbox(
|
90 |
value=["\n\n", "\n", ".", " ", ""],
|