FauziIsyrinApridal commited on
Commit
6a6756a
·
1 Parent(s): 0182950

add from_language

Browse files
Files changed (1) hide show
  1. app/document_processor.py +1 -1
app/document_processor.py CHANGED
@@ -103,7 +103,7 @@ def process_documents(docs):
103
  encode_kwargs={"normalize_embeddings": True}
104
  )
105
 
106
- text_splitter = RecursiveCharacterTextSplitter(
107
  chunk_size=1500,
108
  chunk_overlap=300
109
  )
 
103
  encode_kwargs={"normalize_embeddings": True}
104
  )
105
 
106
+ text_splitter = RecursiveCharacterTextSplitter.from_language("id")(
107
  chunk_size=1500,
108
  chunk_overlap=300
109
  )