Spaces:
Sleeping
Sleeping
FauziIsyrinApridal
commited on
Commit
·
6a6756a
1
Parent(s):
0182950
add from_language
Browse files
app/document_processor.py
CHANGED
@@ -103,7 +103,7 @@ def process_documents(docs):
|
|
103 |
encode_kwargs={"normalize_embeddings": True}
|
104 |
)
|
105 |
|
106 |
-
text_splitter = RecursiveCharacterTextSplitter(
|
107 |
chunk_size=1500,
|
108 |
chunk_overlap=300
|
109 |
)
|
|
|
103 |
encode_kwargs={"normalize_embeddings": True}
|
104 |
)
|
105 |
|
106 |
+
text_splitter = RecursiveCharacterTextSplitter.from_language("id")(
|
107 |
chunk_size=1500,
|
108 |
chunk_overlap=300
|
109 |
)
|