balibabu commited on
Commit
8191450
·
1 Parent(s): ebd2f75

For any type of file, if the parsing method is general, the chunk token number needs to be displayed. #217 (#218)

Browse files

…k token number needs to be displayed. #217

### What problem does this PR solve?

For any type of file, if the parsing method is general, the chunk token
number needs to be displayed. #217

Issue link: #217

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

web/src/components/chunk-method-modal/index.tsx CHANGED
@@ -68,13 +68,26 @@ const ChunkMethodModal: React.FC<IProps> = ({
68
  onOk(selectedTag, parser_config);
69
  };
70
 
 
 
71
  const showPages = useMemo(() => {
72
- return hidePagesChunkMethods.every((x) => x !== selectedTag);
73
- }, [selectedTag]);
74
 
75
  const showOne = useMemo(() => {
76
- return showPages || selectedTag === 'one';
77
- }, [showPages, selectedTag]);
 
 
 
 
 
 
 
 
 
 
 
78
 
79
  const afterClose = () => {
80
  form.resetFields();
@@ -116,166 +129,161 @@ const ChunkMethodModal: React.FC<IProps> = ({
116
  })}
117
  </div>
118
  </Space>
119
- <Divider></Divider>
120
-
121
- {documentExtension === 'pdf' && (
122
- <Form name="dynamic_form_nest_item" autoComplete="off" form={form}>
123
- {showPages && (
124
- <>
125
- <Space>
126
- <p>Page Ranges:</p>
127
- <Tooltip
128
- title={
129
- 'page ranges: Define the page ranges that need to be parsed. The pages that not included in these ranges will be ignored.'
130
- }
131
- >
132
- <QuestionCircleOutlined
133
- className={styles.questionIcon}
134
- ></QuestionCircleOutlined>
135
- </Tooltip>
136
- </Space>
137
- <Form.List name="pages">
138
- {(fields, { add, remove }) => (
139
- <>
140
- {fields.map(({ key, name, ...restField }) => (
141
- <Space
142
- key={key}
143
- style={{
144
- display: 'flex',
145
- }}
146
- align="baseline"
147
- >
148
- <Form.Item
149
- {...restField}
150
- name={[name, 'from']}
151
- dependencies={name > 0 ? [name - 1, 'to'] : []}
152
- rules={[
153
- {
154
- required: true,
155
- message: 'Missing start page number',
 
 
 
 
 
 
 
 
 
 
 
 
 
156
  },
157
- ({ getFieldValue }) => ({
158
- validator(_, value) {
159
- if (
160
- name === 0 ||
161
- !value ||
162
- getFieldValue(['pages', name - 1, 'to']) <
163
- value
164
- ) {
165
- return Promise.resolve();
166
- }
167
- return Promise.reject(
168
- new Error(
169
- 'The current value must be greater than the previous to!',
170
- ),
171
- );
172
- },
173
- }),
174
- ]}
175
- >
176
- <InputNumber
177
- placeholder="from"
178
- min={0}
179
- precision={0}
180
- className={styles.pageInputNumber}
181
- />
182
- </Form.Item>
183
- <Form.Item
184
- {...restField}
185
- name={[name, 'to']}
186
- dependencies={[name, 'from']}
187
- rules={[
188
- {
189
- required: true,
190
- message: 'Missing end page number(excluded)',
191
  },
192
- ({ getFieldValue }) => ({
193
- validator(_, value) {
194
- if (
195
- !value ||
196
- getFieldValue(['pages', name, 'from']) < value
197
- ) {
198
- return Promise.resolve();
199
- }
200
- return Promise.reject(
201
- new Error(
202
- 'The current value must be greater than to!',
203
- ),
204
- );
205
- },
206
- }),
207
- ]}
208
- >
209
- <InputNumber
210
- placeholder="to"
211
- min={0}
212
- precision={0}
213
- className={styles.pageInputNumber}
214
- />
215
- </Form.Item>
216
- {name > 0 && (
217
- <MinusCircleOutlined onClick={() => remove(name)} />
218
- )}
219
- </Space>
220
- ))}
221
- <Form.Item>
222
- <Button
223
- type="dashed"
224
- onClick={() => add()}
225
- block
226
- icon={<PlusOutlined />}
227
  >
228
- Add page
229
- </Button>
230
- </Form.Item>
231
- </>
232
- )}
233
- </Form.List>
234
- </>
235
- )}
236
- {showOne && (
237
- <Form.Item
238
- name={['parser_config', 'layout_recognize']}
239
- label="Layout recognize"
240
- initialValue={true}
241
- valuePropName="checked"
242
- tooltip={
243
- 'Use visual models for layout analysis to better identify document structure, find where the titles, text blocks, images, and tables are. Without this feature, only the plain text of the PDF can be obtained.'
244
- }
245
- >
246
- <Switch />
247
- </Form.Item>
248
- )}
249
- {showPages && (
250
- <Form.Item
251
- noStyle
252
- dependencies={[['parser_config', 'layout_recognize']]}
253
- >
254
- {({ getFieldValue }) =>
255
- getFieldValue(['parser_config', 'layout_recognize']) && (
256
- <Form.Item
257
- name={['parser_config', 'task_page_size']}
258
- label="Task page size"
259
- tooltip={`If using layout recognize, the PDF file will be split into groups of successive. Layout analysis will be performed parallelly between groups to increase the processing speed.
260
- The 'Task page size' determines the size of groups. The larger the page size is, the lower the chance of splitting continuous text between pages into different chunks.`}
261
- initialValue={12}
262
- rules={[
263
- {
264
- required: true,
265
- message: 'Please input your task page size!',
266
- },
267
- ]}
268
- >
269
- <InputNumber min={1} max={128} />
270
  </Form.Item>
271
- )
272
- }
273
- </Form.Item>
274
- )}
275
-
276
- {selectedTag === 'naive' && <MaxTokenNumber></MaxTokenNumber>}
277
- </Form>
278
- )}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
279
  </Modal>
280
  );
281
  };
 
68
  onOk(selectedTag, parser_config);
69
  };
70
 
71
+ const isPdf = documentExtension === 'pdf';
72
+
73
  const showPages = useMemo(() => {
74
+ return isPdf && hidePagesChunkMethods.every((x) => x !== selectedTag);
75
+ }, [selectedTag, isPdf]);
76
 
77
  const showOne = useMemo(() => {
78
+ return (
79
+ isPdf &&
80
+ hidePagesChunkMethods
81
+ .filter((x) => x !== 'one')
82
+ .every((x) => x !== selectedTag)
83
+ );
84
+ }, [selectedTag, isPdf]);
85
+
86
+ const showMaxTokenNumber = selectedTag === 'naive';
87
+
88
+ const hideDivider = [showPages, showOne, showMaxTokenNumber].every(
89
+ (x) => x === false,
90
+ );
91
 
92
  const afterClose = () => {
93
  form.resetFields();
 
129
  })}
130
  </div>
131
  </Space>
132
+ {hideDivider || <Divider></Divider>}
133
+ <Form name="dynamic_form_nest_item" autoComplete="off" form={form}>
134
+ {showPages && (
135
+ <>
136
+ <Space>
137
+ <p>Page Ranges:</p>
138
+ <Tooltip
139
+ title={
140
+ 'page ranges: Define the page ranges that need to be parsed. The pages that not included in these ranges will be ignored.'
141
+ }
142
+ >
143
+ <QuestionCircleOutlined
144
+ className={styles.questionIcon}
145
+ ></QuestionCircleOutlined>
146
+ </Tooltip>
147
+ </Space>
148
+ <Form.List name="pages">
149
+ {(fields, { add, remove }) => (
150
+ <>
151
+ {fields.map(({ key, name, ...restField }) => (
152
+ <Space
153
+ key={key}
154
+ style={{
155
+ display: 'flex',
156
+ }}
157
+ align="baseline"
158
+ >
159
+ <Form.Item
160
+ {...restField}
161
+ name={[name, 'from']}
162
+ dependencies={name > 0 ? [name - 1, 'to'] : []}
163
+ rules={[
164
+ {
165
+ required: true,
166
+ message: 'Missing start page number',
167
+ },
168
+ ({ getFieldValue }) => ({
169
+ validator(_, value) {
170
+ if (
171
+ name === 0 ||
172
+ !value ||
173
+ getFieldValue(['pages', name - 1, 'to']) < value
174
+ ) {
175
+ return Promise.resolve();
176
+ }
177
+ return Promise.reject(
178
+ new Error(
179
+ 'The current value must be greater than the previous to!',
180
+ ),
181
+ );
182
  },
183
+ }),
184
+ ]}
185
+ >
186
+ <InputNumber
187
+ placeholder="from"
188
+ min={0}
189
+ precision={0}
190
+ className={styles.pageInputNumber}
191
+ />
192
+ </Form.Item>
193
+ <Form.Item
194
+ {...restField}
195
+ name={[name, 'to']}
196
+ dependencies={[name, 'from']}
197
+ rules={[
198
+ {
199
+ required: true,
200
+ message: 'Missing end page number(excluded)',
201
+ },
202
+ ({ getFieldValue }) => ({
203
+ validator(_, value) {
204
+ if (
205
+ !value ||
206
+ getFieldValue(['pages', name, 'from']) < value
207
+ ) {
208
+ return Promise.resolve();
209
+ }
210
+ return Promise.reject(
211
+ new Error(
212
+ 'The current value must be greater than to!',
213
+ ),
214
+ );
 
 
215
  },
216
+ }),
217
+ ]}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
218
  >
219
+ <InputNumber
220
+ placeholder="to"
221
+ min={0}
222
+ precision={0}
223
+ className={styles.pageInputNumber}
224
+ />
225
+ </Form.Item>
226
+ {name > 0 && (
227
+ <MinusCircleOutlined onClick={() => remove(name)} />
228
+ )}
229
+ </Space>
230
+ ))}
231
+ <Form.Item>
232
+ <Button
233
+ type="dashed"
234
+ onClick={() => add()}
235
+ block
236
+ icon={<PlusOutlined />}
237
+ >
238
+ Add page
239
+ </Button>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
240
  </Form.Item>
241
+ </>
242
+ )}
243
+ </Form.List>
244
+ </>
245
+ )}
246
+ {showOne && (
247
+ <Form.Item
248
+ name={['parser_config', 'layout_recognize']}
249
+ label="Layout recognize"
250
+ initialValue={true}
251
+ valuePropName="checked"
252
+ tooltip={
253
+ 'Use visual models for layout analysis to better identify document structure, find where the titles, text blocks, images, and tables are. Without this feature, only the plain text of the PDF can be obtained.'
254
+ }
255
+ >
256
+ <Switch />
257
+ </Form.Item>
258
+ )}
259
+ {showPages && (
260
+ <Form.Item
261
+ noStyle
262
+ dependencies={[['parser_config', 'layout_recognize']]}
263
+ >
264
+ {({ getFieldValue }) =>
265
+ getFieldValue(['parser_config', 'layout_recognize']) && (
266
+ <Form.Item
267
+ name={['parser_config', 'task_page_size']}
268
+ label="Task page size"
269
+ tooltip={`If using layout recognize, the PDF file will be split into groups of successive. Layout analysis will be performed parallelly between groups to increase the processing speed.
270
+ The 'Task page size' determines the size of groups. The larger the page size is, the lower the chance of splitting continuous text between pages into different chunks.`}
271
+ initialValue={12}
272
+ rules={[
273
+ {
274
+ required: true,
275
+ message: 'Please input your task page size!',
276
+ },
277
+ ]}
278
+ >
279
+ <InputNumber min={1} max={128} />
280
+ </Form.Item>
281
+ )
282
+ }
283
+ </Form.Item>
284
+ )}
285
+ {showMaxTokenNumber && <MaxTokenNumber></MaxTokenNumber>}
286
+ </Form>
287
  </Modal>
288
  );
289
  };