balibabu
commited on
Commit
·
8191450
1
Parent(s):
ebd2f75
For any type of file, if the parsing method is general, the chunk token number needs to be displayed. #217 (#218)
Browse files…k token number needs to be displayed. #217
### What problem does this PR solve?
For any type of file, if the parsing method is general, the chunk token
number needs to be displayed. #217
Issue link: #217
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
web/src/components/chunk-method-modal/index.tsx
CHANGED
@@ -68,13 +68,26 @@ const ChunkMethodModal: React.FC<IProps> = ({
|
|
68 |
onOk(selectedTag, parser_config);
|
69 |
};
|
70 |
|
|
|
|
|
71 |
const showPages = useMemo(() => {
|
72 |
-
return hidePagesChunkMethods.every((x) => x !== selectedTag);
|
73 |
-
}, [selectedTag]);
|
74 |
|
75 |
const showOne = useMemo(() => {
|
76 |
-
return
|
77 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
78 |
|
79 |
const afterClose = () => {
|
80 |
form.resetFields();
|
@@ -116,166 +129,161 @@ const ChunkMethodModal: React.FC<IProps> = ({
|
|
116 |
})}
|
117 |
</div>
|
118 |
</Space>
|
119 |
-
<Divider></Divider>
|
120 |
-
|
121 |
-
|
122 |
-
|
123 |
-
|
124 |
-
|
125 |
-
<
|
126 |
-
|
127 |
-
|
128 |
-
|
129 |
-
|
130 |
-
|
131 |
-
|
132 |
-
|
133 |
-
|
134 |
-
|
135 |
-
|
136 |
-
|
137 |
-
|
138 |
-
|
139 |
-
|
140 |
-
|
141 |
-
|
142 |
-
|
143 |
-
|
144 |
-
|
145 |
-
|
146 |
-
|
147 |
-
|
148 |
-
|
149 |
-
|
150 |
-
|
151 |
-
|
152 |
-
|
153 |
-
|
154 |
-
|
155 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
156 |
},
|
157 |
-
|
158 |
-
|
159 |
-
|
160 |
-
|
161 |
-
|
162 |
-
|
163 |
-
|
164 |
-
|
165 |
-
|
166 |
-
|
167 |
-
|
168 |
-
|
169 |
-
|
170 |
-
|
171 |
-
|
172 |
-
|
173 |
-
|
174 |
-
|
175 |
-
|
176 |
-
|
177 |
-
|
178 |
-
|
179 |
-
|
180 |
-
|
181 |
-
|
182 |
-
|
183 |
-
|
184 |
-
|
185 |
-
|
186 |
-
|
187 |
-
|
188 |
-
|
189 |
-
required: true,
|
190 |
-
message: 'Missing end page number(excluded)',
|
191 |
},
|
192 |
-
|
193 |
-
|
194 |
-
if (
|
195 |
-
!value ||
|
196 |
-
getFieldValue(['pages', name, 'from']) < value
|
197 |
-
) {
|
198 |
-
return Promise.resolve();
|
199 |
-
}
|
200 |
-
return Promise.reject(
|
201 |
-
new Error(
|
202 |
-
'The current value must be greater than to!',
|
203 |
-
),
|
204 |
-
);
|
205 |
-
},
|
206 |
-
}),
|
207 |
-
]}
|
208 |
-
>
|
209 |
-
<InputNumber
|
210 |
-
placeholder="to"
|
211 |
-
min={0}
|
212 |
-
precision={0}
|
213 |
-
className={styles.pageInputNumber}
|
214 |
-
/>
|
215 |
-
</Form.Item>
|
216 |
-
{name > 0 && (
|
217 |
-
<MinusCircleOutlined onClick={() => remove(name)} />
|
218 |
-
)}
|
219 |
-
</Space>
|
220 |
-
))}
|
221 |
-
<Form.Item>
|
222 |
-
<Button
|
223 |
-
type="dashed"
|
224 |
-
onClick={() => add()}
|
225 |
-
block
|
226 |
-
icon={<PlusOutlined />}
|
227 |
>
|
228 |
-
|
229 |
-
|
230 |
-
|
231 |
-
|
232 |
-
|
233 |
-
|
234 |
-
|
235 |
-
|
236 |
-
|
237 |
-
|
238 |
-
|
239 |
-
|
240 |
-
|
241 |
-
|
242 |
-
|
243 |
-
|
244 |
-
|
245 |
-
|
246 |
-
|
247 |
-
|
248 |
-
|
249 |
-
{showPages && (
|
250 |
-
<Form.Item
|
251 |
-
noStyle
|
252 |
-
dependencies={[['parser_config', 'layout_recognize']]}
|
253 |
-
>
|
254 |
-
{({ getFieldValue }) =>
|
255 |
-
getFieldValue(['parser_config', 'layout_recognize']) && (
|
256 |
-
<Form.Item
|
257 |
-
name={['parser_config', 'task_page_size']}
|
258 |
-
label="Task page size"
|
259 |
-
tooltip={`If using layout recognize, the PDF file will be split into groups of successive. Layout analysis will be performed parallelly between groups to increase the processing speed.
|
260 |
-
The 'Task page size' determines the size of groups. The larger the page size is, the lower the chance of splitting continuous text between pages into different chunks.`}
|
261 |
-
initialValue={12}
|
262 |
-
rules={[
|
263 |
-
{
|
264 |
-
required: true,
|
265 |
-
message: 'Please input your task page size!',
|
266 |
-
},
|
267 |
-
]}
|
268 |
-
>
|
269 |
-
<InputNumber min={1} max={128} />
|
270 |
</Form.Item>
|
271 |
-
|
272 |
-
}
|
273 |
-
</Form.
|
274 |
-
|
275 |
-
|
276 |
-
|
277 |
-
|
278 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
279 |
</Modal>
|
280 |
);
|
281 |
};
|
|
|
68 |
onOk(selectedTag, parser_config);
|
69 |
};
|
70 |
|
71 |
+
const isPdf = documentExtension === 'pdf';
|
72 |
+
|
73 |
const showPages = useMemo(() => {
|
74 |
+
return isPdf && hidePagesChunkMethods.every((x) => x !== selectedTag);
|
75 |
+
}, [selectedTag, isPdf]);
|
76 |
|
77 |
const showOne = useMemo(() => {
|
78 |
+
return (
|
79 |
+
isPdf &&
|
80 |
+
hidePagesChunkMethods
|
81 |
+
.filter((x) => x !== 'one')
|
82 |
+
.every((x) => x !== selectedTag)
|
83 |
+
);
|
84 |
+
}, [selectedTag, isPdf]);
|
85 |
+
|
86 |
+
const showMaxTokenNumber = selectedTag === 'naive';
|
87 |
+
|
88 |
+
const hideDivider = [showPages, showOne, showMaxTokenNumber].every(
|
89 |
+
(x) => x === false,
|
90 |
+
);
|
91 |
|
92 |
const afterClose = () => {
|
93 |
form.resetFields();
|
|
|
129 |
})}
|
130 |
</div>
|
131 |
</Space>
|
132 |
+
{hideDivider || <Divider></Divider>}
|
133 |
+
<Form name="dynamic_form_nest_item" autoComplete="off" form={form}>
|
134 |
+
{showPages && (
|
135 |
+
<>
|
136 |
+
<Space>
|
137 |
+
<p>Page Ranges:</p>
|
138 |
+
<Tooltip
|
139 |
+
title={
|
140 |
+
'page ranges: Define the page ranges that need to be parsed. The pages that not included in these ranges will be ignored.'
|
141 |
+
}
|
142 |
+
>
|
143 |
+
<QuestionCircleOutlined
|
144 |
+
className={styles.questionIcon}
|
145 |
+
></QuestionCircleOutlined>
|
146 |
+
</Tooltip>
|
147 |
+
</Space>
|
148 |
+
<Form.List name="pages">
|
149 |
+
{(fields, { add, remove }) => (
|
150 |
+
<>
|
151 |
+
{fields.map(({ key, name, ...restField }) => (
|
152 |
+
<Space
|
153 |
+
key={key}
|
154 |
+
style={{
|
155 |
+
display: 'flex',
|
156 |
+
}}
|
157 |
+
align="baseline"
|
158 |
+
>
|
159 |
+
<Form.Item
|
160 |
+
{...restField}
|
161 |
+
name={[name, 'from']}
|
162 |
+
dependencies={name > 0 ? [name - 1, 'to'] : []}
|
163 |
+
rules={[
|
164 |
+
{
|
165 |
+
required: true,
|
166 |
+
message: 'Missing start page number',
|
167 |
+
},
|
168 |
+
({ getFieldValue }) => ({
|
169 |
+
validator(_, value) {
|
170 |
+
if (
|
171 |
+
name === 0 ||
|
172 |
+
!value ||
|
173 |
+
getFieldValue(['pages', name - 1, 'to']) < value
|
174 |
+
) {
|
175 |
+
return Promise.resolve();
|
176 |
+
}
|
177 |
+
return Promise.reject(
|
178 |
+
new Error(
|
179 |
+
'The current value must be greater than the previous to!',
|
180 |
+
),
|
181 |
+
);
|
182 |
},
|
183 |
+
}),
|
184 |
+
]}
|
185 |
+
>
|
186 |
+
<InputNumber
|
187 |
+
placeholder="from"
|
188 |
+
min={0}
|
189 |
+
precision={0}
|
190 |
+
className={styles.pageInputNumber}
|
191 |
+
/>
|
192 |
+
</Form.Item>
|
193 |
+
<Form.Item
|
194 |
+
{...restField}
|
195 |
+
name={[name, 'to']}
|
196 |
+
dependencies={[name, 'from']}
|
197 |
+
rules={[
|
198 |
+
{
|
199 |
+
required: true,
|
200 |
+
message: 'Missing end page number(excluded)',
|
201 |
+
},
|
202 |
+
({ getFieldValue }) => ({
|
203 |
+
validator(_, value) {
|
204 |
+
if (
|
205 |
+
!value ||
|
206 |
+
getFieldValue(['pages', name, 'from']) < value
|
207 |
+
) {
|
208 |
+
return Promise.resolve();
|
209 |
+
}
|
210 |
+
return Promise.reject(
|
211 |
+
new Error(
|
212 |
+
'The current value must be greater than to!',
|
213 |
+
),
|
214 |
+
);
|
|
|
|
|
215 |
},
|
216 |
+
}),
|
217 |
+
]}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
218 |
>
|
219 |
+
<InputNumber
|
220 |
+
placeholder="to"
|
221 |
+
min={0}
|
222 |
+
precision={0}
|
223 |
+
className={styles.pageInputNumber}
|
224 |
+
/>
|
225 |
+
</Form.Item>
|
226 |
+
{name > 0 && (
|
227 |
+
<MinusCircleOutlined onClick={() => remove(name)} />
|
228 |
+
)}
|
229 |
+
</Space>
|
230 |
+
))}
|
231 |
+
<Form.Item>
|
232 |
+
<Button
|
233 |
+
type="dashed"
|
234 |
+
onClick={() => add()}
|
235 |
+
block
|
236 |
+
icon={<PlusOutlined />}
|
237 |
+
>
|
238 |
+
Add page
|
239 |
+
</Button>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
240 |
</Form.Item>
|
241 |
+
</>
|
242 |
+
)}
|
243 |
+
</Form.List>
|
244 |
+
</>
|
245 |
+
)}
|
246 |
+
{showOne && (
|
247 |
+
<Form.Item
|
248 |
+
name={['parser_config', 'layout_recognize']}
|
249 |
+
label="Layout recognize"
|
250 |
+
initialValue={true}
|
251 |
+
valuePropName="checked"
|
252 |
+
tooltip={
|
253 |
+
'Use visual models for layout analysis to better identify document structure, find where the titles, text blocks, images, and tables are. Without this feature, only the plain text of the PDF can be obtained.'
|
254 |
+
}
|
255 |
+
>
|
256 |
+
<Switch />
|
257 |
+
</Form.Item>
|
258 |
+
)}
|
259 |
+
{showPages && (
|
260 |
+
<Form.Item
|
261 |
+
noStyle
|
262 |
+
dependencies={[['parser_config', 'layout_recognize']]}
|
263 |
+
>
|
264 |
+
{({ getFieldValue }) =>
|
265 |
+
getFieldValue(['parser_config', 'layout_recognize']) && (
|
266 |
+
<Form.Item
|
267 |
+
name={['parser_config', 'task_page_size']}
|
268 |
+
label="Task page size"
|
269 |
+
tooltip={`If using layout recognize, the PDF file will be split into groups of successive. Layout analysis will be performed parallelly between groups to increase the processing speed.
|
270 |
+
The 'Task page size' determines the size of groups. The larger the page size is, the lower the chance of splitting continuous text between pages into different chunks.`}
|
271 |
+
initialValue={12}
|
272 |
+
rules={[
|
273 |
+
{
|
274 |
+
required: true,
|
275 |
+
message: 'Please input your task page size!',
|
276 |
+
},
|
277 |
+
]}
|
278 |
+
>
|
279 |
+
<InputNumber min={1} max={128} />
|
280 |
+
</Form.Item>
|
281 |
+
)
|
282 |
+
}
|
283 |
+
</Form.Item>
|
284 |
+
)}
|
285 |
+
{showMaxTokenNumber && <MaxTokenNumber></MaxTokenNumber>}
|
286 |
+
</Form>
|
287 |
</Modal>
|
288 |
);
|
289 |
};
|