faridkarimli commited on
Commit
96ec12a
·
verified ·
1 Parent(s): e2016ea

Change to semantic chunking

Browse files
Files changed (1) hide show
  1. apps/ai_tutor/config/config.yml +3 -3
apps/ai_tutor/config/config.yml CHANGED
@@ -51,9 +51,9 @@ splitter_options:
51
  split_by_token : True # bool
52
  remove_leftover_delimiters: True # bool
53
  remove_chunks: False # bool
54
- chunking_mode: 'semantic' # str [fixed, semantic]
55
- chunk_size : 300 # int
56
- chunk_overlap : 30 # int
57
  chunk_separators : ["\n\n", "\n", " ", ""] # list of strings
58
  front_chunks_to_remove : null # int or None
59
  last_chunks_to_remove : null # int or None
 
51
  split_by_token : True # bool
52
  remove_leftover_delimiters: True # bool
53
  remove_chunks: False # bool
54
+ chunking_mode: 'fixed' # str [fixed, semantic]
55
+ chunk_size : 1000 # int
56
+ chunk_overlap : 100 # int
57
  chunk_separators : ["\n\n", "\n", " ", ""] # list of strings
58
  front_chunks_to_remove : null # int or None
59
  last_chunks_to_remove : null # int or None