Feat: Scrolling knowledge base list and set the number of entries per page to 30 #3695 (#3712)

### What problem does this PR solve?

Feat: Scrolling knowledge base list and set the number of entries per
page to 30 #3695

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

api/apps/kb_app.py

@@ -125,15 +125,16 @@ def detail():
 @manager.route('/list', methods=['GET'])
 @login_required
 def list_kbs():
+    keywords = request.args.get("keywords", "")
     page_number = int(request.args.get("page", 1))
     items_per_page = int(request.args.get("page_size", 150))
     orderby = request.args.get("orderby", "create_time")
     desc = request.args.get("desc", True)
     try:
         tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
-        kbs = KnowledgebaseService.get_by_tenant_ids(
-            [m["tenant_id"] for m in tenants], current_user.id, page_number, items_per_page, orderby, desc)
-        return get_json_result(data=kbs)
+        kbs, total = KnowledgebaseService.get_by_tenant_ids(
+            [m["tenant_id"] for m in tenants], current_user.id, page_number, items_per_page, orderby, desc, keywords)
+        return get_json_result(data={"kbs": kbs, "total": total})
     except Exception as e:
         return server_error_response(e)
 

api/db/services/knowledgebase_service.py

@@ -16,6 +16,7 @@
 from api.db import StatusEnum, TenantPermission
 from api.db.db_models import Knowledgebase, DB, Tenant, User, UserTenant,Document
 from api.db.services.common_service import CommonService
+from peewee import fn
 
 
 class KnowledgebaseService(CommonService):
@@ -34,7 +35,7 @@ class KnowledgebaseService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_by_tenant_ids(cls, joined_tenant_ids, user_id,
-                          page_number, items_per_page, orderby, desc):
+                          page_number, items_per_page, orderby, desc, keywords):
         fields = [
             cls.model.id,
             cls.model.avatar,
@@ -51,20 +52,31 @@ class KnowledgebaseService(CommonService):
             User.avatar.alias('tenant_avatar'),
             cls.model.update_time
         ]
-        kbs = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id)).where(
-            ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission ==
-                                                            TenantPermission.TEAM.value)) | (
-                     cls.model.tenant_id == user_id))
-            & (cls.model.status == StatusEnum.VALID.value)
-        )
+        if keywords:
+            kbs = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id)).where(
+                ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission ==
+                                                                TenantPermission.TEAM.value)) | (
+                    cls.model.tenant_id == user_id))
+                & (cls.model.status == StatusEnum.VALID.value),
+                (fn.LOWER(cls.model.name).contains(keywords.lower()))
+            )
+        else:
+            kbs = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id)).where(
+                ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission ==
+                                                                TenantPermission.TEAM.value)) | (
+                    cls.model.tenant_id == user_id))
+                & (cls.model.status == StatusEnum.VALID.value)
+            )
         if desc:
             kbs = kbs.order_by(cls.model.getter_by(orderby).desc())
         else:
             kbs = kbs.order_by(cls.model.getter_by(orderby).asc())
 
+        count = kbs.count()
+
         kbs = kbs.paginate(page_number, items_per_page)
 
-        return list(kbs.dicts())
+        return list(kbs.dicts()), count
 
     @classmethod
     @DB.connection_context()

sdk/python/test/test_frontend_api/test_dataset.py

@@ -13,7 +13,7 @@ def test_dataset(get_auth):
     while True:
         res = list_dataset(get_auth, page_number)
         data = res.get("data")
-        for item in data:
+        for item in data.get("kbs"):
             dataset_id = item.get("id")
             dataset_list.append(dataset_id)
         if len(dataset_list) < page_number * 150:
@@ -42,7 +42,7 @@ def test_dataset_1k_dataset(get_auth):
     while True:
         res = list_dataset(get_auth, page_number)
         data = res.get("data")
-        for item in data:
+        for item in data.get("kbs"):
             dataset_id = item.get("id")
             dataset_list.append(dataset_id)
         if len(dataset_list) < page_number * 150:

web/.umirc.ts

@@ -34,7 +34,7 @@ export default defineConfig({
   proxy: [
     {
       context: ['/api', '/v1'],
-      target: 'http://127.0.0.1:9456/',
+      target: 'http://127.0.0.1:9380/',
       changeOrigin: true,
       ws: true,
       logger: console,

web/package-lock.json

@@ -57,6 +57,7 @@
         "react-force-graph": "^1.44.4",
         "react-hook-form": "^7.53.1",
         "react-i18next": "^14.0.0",
+        "react-infinite-scroll-component": "^6.1.0",
         "react-markdown": "^9.0.1",
         "react-pdf-highlighter": "^6.1.0",
         "react-string-replace": "^1.1.1",
@@ -24705,6 +24706,25 @@
         }
       }
     },
+    "node_modules/react-infinite-scroll-component": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmmirror.com/react-infinite-scroll-component/-/react-infinite-scroll-component-6.1.0.tgz",
+      "integrity": "sha512-SQu5nCqy8DxQWpnUVLx7V7b7LcA37aM7tvoWjTLZp1dk6EJibM5/4EJKzOnl07/BsM1Y40sKLuqjCwwH/xV0TQ==",
+      "dependencies": {
+        "throttle-debounce": "^2.1.0"
+      },
+      "peerDependencies": {
+        "react": ">=16.0.0"
+      }
+    },
+    "node_modules/react-infinite-scroll-component/node_modules/throttle-debounce": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmmirror.com/throttle-debounce/-/throttle-debounce-2.3.0.tgz",
+      "integrity": "sha512-H7oLPV0P7+jgvrk+6mwwwBDmxTaxnu9HMXmloNLXwnNO0ZxZ31Orah2n8lU1eMPvsaowP2CX+USCgyovXfdOFQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/react-is": {
       "version": "18.2.0",
       "resolved": "https://registry.npmmirror.com/react-is/-/react-is-18.2.0.tgz",

web/package.json

@@ -68,6 +68,7 @@
     "react-force-graph": "^1.44.4",
     "react-hook-form": "^7.53.1",
     "react-i18next": "^14.0.0",
+    "react-infinite-scroll-component": "^6.1.0",
     "react-markdown": "^9.0.1",
     "react-pdf-highlighter": "^6.1.0",
     "react-string-replace": "^1.1.1",

web/src/hooks/knowledge-hooks.ts

@@ -3,14 +3,17 @@ import { IKnowledge, ITestingResult } from '@/interfaces/database/knowledge';
 import i18n from '@/locales/config';
 import kbService from '@/services/knowledge-service';
 import {
+  useInfiniteQuery,
   useIsMutating,
   useMutation,
   useMutationState,
   useQuery,
   useQueryClient,
 } from '@tanstack/react-query';
+import { useDebounce } from 'ahooks';
 import { message } from 'antd';
 import { useSearchParams } from 'umi';
+import { useHandleSearchChange } from './logic-hooks';
 import { useSetPaginationParams } from './route-hook';
 
 export const useKnowledgeBaseId = (): string => {
@@ -50,7 +53,7 @@ export const useNextFetchKnowledgeList = (
     gcTime: 0, // https://tanstack.com/query/latest/docs/framework/react/guides/caching?from=reactQueryV3
     queryFn: async () => {
       const { data } = await kbService.getList();
-      const list = data?.data ?? [];
+      const list = data?.data?.kbs ?? [];
       return shouldFilterListWithoutDocument
         ? list.filter((x: IKnowledge) => x.chunk_num > 0)
         : list;
@@ -60,6 +63,52 @@ export const useNextFetchKnowledgeList = (
   return { list: data, loading };
 };
 
+export const useInfiniteFetchKnowledgeList = () => {
+  const { searchString, handleInputChange } = useHandleSearchChange();
+  const debouncedSearchString = useDebounce(searchString, { wait: 500 });
+
+  const PageSize = 30;
+  const {
+    data,
+    error,
+    fetchNextPage,
+    hasNextPage,
+    isFetching,
+    isFetchingNextPage,
+    status,
+  } = useInfiniteQuery({
+    queryKey: ['infiniteFetchKnowledgeList', debouncedSearchString],
+    queryFn: async ({ pageParam }) => {
+      const { data } = await kbService.getList({
+        page: pageParam,
+        page_size: PageSize,
+        keywords: debouncedSearchString,
+      });
+      const list = data?.data ?? [];
+      return list;
+    },
+    initialPageParam: 1,
+    getNextPageParam: (lastPage, pages, lastPageParam) => {
+      if (lastPageParam * PageSize <= lastPage.total) {
+        return lastPageParam + 1;
+      }
+      return undefined;
+    },
+  });
+  return {
+    data,
+    loading: isFetching,
+    error,
+    fetchNextPage,
+    hasNextPage,
+    isFetching,
+    isFetchingNextPage,
+    status,
+    handleInputChange,
+    searchString,
+  };
+};
+
 export const useCreateKnowledge = () => {
   const queryClient = useQueryClient();
   const {
@@ -95,7 +144,9 @@ export const useDeleteKnowledge = () => {
       const { data } = await kbService.rmKb({ kb_id: id });
       if (data.code === 0) {
         message.success(i18n.t(`message.deleted`));
-        queryClient.invalidateQueries({ queryKey: ['fetchKnowledgeList'] });
+        queryClient.invalidateQueries({
+          queryKey: ['infiniteFetchKnowledgeList'],
+        });
       }
       return data?.data ?? [];
     },

web/src/locales/en.ts

@@ -75,6 +75,7 @@ export default {
       namePlaceholder: 'Please input name!',
       doc: 'Docs',
       searchKnowledgePlaceholder: 'Search',
+      noMoreData: 'It is all, nothing more',
     },
     knowledgeDetails: {
       dataset: 'Dataset',

web/src/locales/zh-traditional.ts

@@ -75,6 +75,7 @@ export default {
       namePlaceholder: '請輸入名稱',
       doc: '文件',
       searchKnowledgePlaceholder: '搜索',
+      noMoreData: 'It is all, nothing more',
     },
     knowledgeDetails: {
       dataset: '數據集',

web/src/locales/zh.ts

@@ -75,6 +75,7 @@ export default {
       namePlaceholder: '请输入名称',
       doc: '文档',
       searchKnowledgePlaceholder: '搜索',
+      noMoreData: '沒有更多的數據了',
     },
     knowledgeDetails: {
       dataset: '数据集',

web/src/pages/knowledge/index.less

@@ -2,6 +2,7 @@
 
 .knowledge {
   padding: 48px 0;
+  overflow: auto;
 }
 
 .topWrapper {

web/src/pages/knowledge/index.tsx

@@ -1,18 +1,26 @@
-import { useNextFetchKnowledgeList } from '@/hooks/knowledge-hooks';
+import { useInfiniteFetchKnowledgeList } from '@/hooks/knowledge-hooks';
 import { useFetchUserInfo } from '@/hooks/user-setting-hooks';
 import { PlusOutlined, SearchOutlined } from '@ant-design/icons';
-import { Button, Empty, Flex, Input, Space, Spin } from 'antd';
+import {
+  Button,
+  Divider,
+  Empty,
+  Flex,
+  Input,
+  Skeleton,
+  Space,
+  Spin,
+} from 'antd';
+import { useTranslation } from 'react-i18next';
+import InfiniteScroll from 'react-infinite-scroll-component';
+import { useSaveKnowledge } from './hooks';
 import KnowledgeCard from './knowledge-card';
 import KnowledgeCreatingModal from './knowledge-creating-modal';
 
-import { useTranslation } from 'react-i18next';
-import { useSaveKnowledge, useSearchKnowledge } from './hooks';
+import { useMemo } from 'react';
 import styles from './index.less';
 
 const KnowledgeList = () => {
-  const { searchString, handleInputChange } = useSearchKnowledge();
-  const { loading, list: data } = useNextFetchKnowledgeList();
-  const list = data.filter((x) => x.name.includes(searchString));
   const { data: userInfo } = useFetchUserInfo();
   const { t } = useTranslation('translation', { keyPrefix: 'knowledgeList' });
   const {
@@ -22,9 +30,23 @@ const KnowledgeList = () => {
     onCreateOk,
     loading: creatingLoading,
   } = useSaveKnowledge();
+  const {
+    fetchNextPage,
+    data,
+    hasNextPage,
+    searchString,
+    handleInputChange,
+    loading,
+  } = useInfiniteFetchKnowledgeList();
+  console.log('🚀 ~ KnowledgeList ~ data:', data);
+  const nextList = data?.pages?.flatMap((x) => x.kbs) ?? [];
+
+  const total = useMemo(() => {
+    return data?.pages.at(-1).total ?? 0;
+  }, [data?.pages]);
 
   return (
-    <Flex className={styles.knowledge} vertical flex={1}>
+    <Flex className={styles.knowledge} vertical flex={1} id="scrollableDiv">
       <div className={styles.topWrapper}>
         <div>
           <span className={styles.title}>
@@ -53,21 +75,30 @@ const KnowledgeList = () => {
         </Space>
       </div>
       <Spin spinning={loading}>
-        <Flex
-          gap={'large'}
-          wrap="wrap"
-          className={styles.knowledgeCardContainer}
+        <InfiniteScroll
+          dataLength={nextList?.length ?? 0}
+          next={fetchNextPage}
+          hasMore={hasNextPage}
+          loader={<Skeleton avatar paragraph={{ rows: 1 }} active />}
+          endMessage={total && <Divider plain>{t('noMoreData')} 🤐</Divider>}
+          scrollableTarget="scrollableDiv"
         >
-          {list.length > 0 ? (
-            list.map((item: any) => {
-              return (
-                <KnowledgeCard item={item} key={item.name}></KnowledgeCard>
-              );
-            })
-          ) : (
-            <Empty className={styles.knowledgeEmpty}></Empty>
-          )}
-        </Flex>
+          <Flex
+            gap={'large'}
+            wrap="wrap"
+            className={styles.knowledgeCardContainer}
+          >
+            {nextList?.length > 0 ? (
+              nextList.map((item: any) => {
+                return (
+                  <KnowledgeCard item={item} key={item.name}></KnowledgeCard>
+                );
+              })
+            ) : (
+              <Empty className={styles.knowledgeEmpty}></Empty>
+            )}
+          </Flex>
+        </InfiniteScroll>
       </Spin>
       <KnowledgeCreatingModal
         loading={creatingLoading}