Change to semantic chunking
Browse files
apps/ai_tutor/config/config.yml
CHANGED
@@ -51,9 +51,9 @@ splitter_options:
|
|
51 |
split_by_token : True # bool
|
52 |
remove_leftover_delimiters: True # bool
|
53 |
remove_chunks: False # bool
|
54 |
-
chunking_mode: '
|
55 |
-
chunk_size :
|
56 |
-
chunk_overlap :
|
57 |
chunk_separators : ["\n\n", "\n", " ", ""] # list of strings
|
58 |
front_chunks_to_remove : null # int or None
|
59 |
last_chunks_to_remove : null # int or None
|
|
|
51 |
split_by_token : True # bool
|
52 |
remove_leftover_delimiters: True # bool
|
53 |
remove_chunks: False # bool
|
54 |
+
chunking_mode: 'fixed' # str [fixed, semantic]
|
55 |
+
chunk_size : 1000 # int
|
56 |
+
chunk_overlap : 100 # int
|
57 |
chunk_separators : ["\n\n", "\n", " ", ""] # list of strings
|
58 |
front_chunks_to_remove : null # int or None
|
59 |
last_chunks_to_remove : null # int or None
|