Add test for document (#3548)

### What problem does this PR solve?

Add test for document

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

Co-authored-by: liuhua <[email protected]>

api/apps/sdk/doc.py

@@ -115,6 +115,7 @@ def upload(dataset_id, tenant_id):
             return get_result(
                 message="No file selected!", code=settings.RetCode.ARGUMENT_ERROR
             )
+    '''
     # total size
     total_size = 0
     for file_obj in file_objs:
@@ -127,6 +128,7 @@ def upload(dataset_id, tenant_id):
             message=f"Total file size exceeds 10MB limit! ({total_size / (1024 * 1024):.2f} MB)",
             code=settings.RetCode.ARGUMENT_ERROR,
         )
+    '''
     e, kb = KnowledgebaseService.get_by_id(dataset_id)
     if not e:
         raise LookupError(f"Can't find the dataset with ID {dataset_id}!")

sdk/python/test/t_document.py

@@ -1,6 +1,6 @@
-from ragflow_sdk import RAGFlow, DataSet, Document, Chunk
+from ragflow_sdk import RAGFlow
 from common import HOST_ADDRESS
-
+import pytest
 
 def test_upload_document_with_success(get_api_key_fixture):
     API_KEY = get_api_key_fixture
@@ -48,7 +48,6 @@ def test_list_documents_in_dataset_with_success(get_api_key_fixture):
     ds.list_documents(keywords="test", page=1, page_size=12)
 
 
-
 def test_delete_documents_in_dataset_with_success(get_api_key_fixture):
     API_KEY = get_api_key_fixture
     rag = RAGFlow(API_KEY, HOST_ADDRESS)
@@ -59,4 +58,109 @@ def test_delete_documents_in_dataset_with_success(get_api_key_fixture):
     docs = ds.upload_documents(document_infos)
     ds.delete_documents([docs[0].id])
 
+# upload and parse the document with different in different parse method.
+def test_upload_and_parse_pdf_documents_with_general_parse_method(get_api_key_fixture):
+    API_KEY = get_api_key_fixture
+    rag = RAGFlow(API_KEY, HOST_ADDRESS)
+    ds = rag.create_dataset(name="test_pdf_document")
+    with open("test_data/test.pdf","rb") as file:
+        blob=file.read()
+    document_infos = [{"displayed_name": "test.pdf","blob": blob}]
+    docs=ds.upload_documents(document_infos)
+    doc = docs[0]
+    ds.async_parse_documents([doc.id])
 
+def test_upload_and_parse_docx_documents_with_general_parse_method(get_api_key_fixture):
+    API_KEY = get_api_key_fixture
+    rag = RAGFlow(API_KEY, HOST_ADDRESS)
+    ds = rag.create_dataset(name="test_docx_document")
+    with open("test_data/test.docx","rb") as file:
+        blob=file.read()
+    document_infos = [{"displayed_name": "test.docx","blob": blob}]
+    docs=ds.upload_documents(document_infos)
+    doc = docs[0]
+    ds.async_parse_documents([doc.id])
+def test_upload_and_parse_excel_documents_with_general_parse_method(get_api_key_fixture):
+    API_KEY = get_api_key_fixture
+    rag = RAGFlow(API_KEY, HOST_ADDRESS)
+    ds = rag.create_dataset(name="test_excel_document")
+    with open("test_data/test.xlsx","rb") as file:
+        blob=file.read()
+    document_infos = [{"displayed_name": "test.xlsx","blob": blob}]
+    docs=ds.upload_documents(document_infos)
+    doc = docs[0]
+    ds.async_parse_documents([doc.id])
+def test_upload_and_parse_ppt_documents_with_general_parse_method(get_api_key_fixture):
+    API_KEY = get_api_key_fixture
+    rag = RAGFlow(API_KEY, HOST_ADDRESS)
+    ds = rag.create_dataset(name="test_ppt_document")
+    with open("test_data/test.ppt","rb") as file:
+        blob=file.read()
+    document_infos = [{"displayed_name": "test.ppt","blob": blob}]
+    docs=ds.upload_documents(document_infos)
+    doc = docs[0]
+    ds.async_parse_documents([doc.id])
+def test_upload_and_parse_image_documents_with_general_parse_method(get_api_key_fixture):
+    API_KEY = get_api_key_fixture
+    rag = RAGFlow(API_KEY, HOST_ADDRESS)
+    ds = rag.create_dataset(name="test_image_document")
+    with open("test_data/test.jpg","rb") as file:
+        blob=file.read()
+    document_infos = [{"displayed_name": "test.jpg","blob": blob}]
+    docs=ds.upload_documents(document_infos)
+    doc = docs[0]
+    ds.async_parse_documents([doc.id])
+def test_upload_and_parse_txt_documents_with_general_parse_method(get_api_key_fixture):
+    API_KEY = get_api_key_fixture
+    rag = RAGFlow(API_KEY, HOST_ADDRESS)
+    ds = rag.create_dataset(name="test_txt_document")
+    with open("test_data/test.txt","rb") as file:
+        blob=file.read()
+    document_infos = [{"displayed_name": "test.txt","blob": blob}]
+    docs=ds.upload_documents(document_infos)
+    doc = docs[0]
+    ds.async_parse_documents([doc.id])
+def test_upload_and_parse_md_documents_with_general_parse_method(get_api_key_fixture):
+    API_KEY = get_api_key_fixture
+    rag = RAGFlow(API_KEY, HOST_ADDRESS)
+    ds = rag.create_dataset(name="test_md_document")
+    with open("test_data/test.md","rb") as file:
+        blob=file.read()
+    document_infos = [{"displayed_name": "test.md","blob": blob}]
+    docs=ds.upload_documents(document_infos)
+    doc = docs[0]
+    ds.async_parse_documents([doc.id])
+
+def test_upload_and_parse_json_documents_with_general_parse_method(get_api_key_fixture):
+    API_KEY = get_api_key_fixture
+    rag = RAGFlow(API_KEY, HOST_ADDRESS)
+    ds = rag.create_dataset(name="test_json_document")
+    with open("test_data/test.json","rb") as file:
+        blob=file.read()
+    document_infos = [{"displayed_name": "test.json","blob": blob}]
+    docs=ds.upload_documents(document_infos)
+    doc = docs[0]
+    ds.async_parse_documents([doc.id])
+
+@pytest.mark.skip(reason="")
+def test_upload_and_parse_eml_documents_with_general_parse_method(get_api_key_fixture):
+    API_KEY = get_api_key_fixture
+    rag = RAGFlow(API_KEY, HOST_ADDRESS)
+    ds = rag.create_dataset(name="test_eml_document")
+    with open("test_data/test.eml","rb") as file:
+        blob=file.read()
+    document_infos = [{"displayed_name": "test.eml","blob": blob}]
+    docs=ds.upload_documents(document_infos)
+    doc = docs[0]
+    ds.async_parse_documents([doc.id])
+
+def test_upload_and_parse_html_documents_with_general_parse_method(get_api_key_fixture):
+    API_KEY = get_api_key_fixture
+    rag = RAGFlow(API_KEY, HOST_ADDRESS)
+    ds = rag.create_dataset(name="test_html_document")
+    with open("test_data/test.html","rb") as file:
+        blob=file.read()
+    document_infos = [{"displayed_name": "test.html","blob": blob}]
+    docs=ds.upload_documents(document_infos)
+    doc = docs[0]
+    ds.async_parse_documents([doc.id])

sdk/python/test/test_data/.txt

@@ -1,2 +0,0 @@
-hhh
-hhh

sdk/python/test/test_data/lol.txt

@@ -1,3 +0,0 @@
-llll
-ooooo
-llll

sdk/python/test/test_data/story.txt

@@ -1,8 +0,0 @@
-Once upon a time, in a small village nestled at the foot of a towering mountain, lived a young girl named Lily. Lily had a heart as pure as the mountain's snowcaps and a spirit as adventurous as the winding trails that led to its peak.
-One day, as Lily was gathering berries in the forest's edge, she stumbled upon an old, weathered map hidden beneath a fallen tree. The map was covered in strange symbols and a single, glowing word: "Treasure." Curiousity piqued, Lily decided to embark on a quest to uncover the mystery of the treasure.
-With nothing more than her trusty basket of berries, a few pieces of bread, and the map, Lily set off into the unknown. As she climbed higher and higher into the mountains, the air grew crisp, and the scenery transformed into a breathtaking tapestry of lush greenery and sparkling streams.
-Along the way, Lily encountered all sorts of challenges. She had to navigate treacherous rivers using fallen logs as bridges, climb steep cliffs with nothing but her agility and determination, and even outsmart a mischievous pack of foxes that tried to lead her astray. But through it all, Lily remained steadfast, her heart filled with hope and a sense of purpose.
-Finally, after what seemed like an eternity of trekking, Lily arrived at a hidden valley. At its center stood an ancient tree, its roots entwined with glittering jewels and a chest made of pure gold. This, the map had revealed, was the source of the treasure.
-But as Lily approached the chest, she realized that the true treasure was not the riches before her. It was the journey itself—the friendships she had forged with the animals she encountered, the strength she had gained from overcoming obstacles, and the sense of wonder and discovery that filled her heart.
-With a smile on her face, Lily gently closed the chest and left it where it was, content in the knowledge that the greatest treasures in life are not always found in gold or jewels. She turned back towards home, her heart full of stories to share and a spirit that had been forever changed by her adventure.
-And so, Lily returned to her village, a hero in her own right, with a tale that would be whispered around firesides for generations to come.

sdk/python/test/test_data/test.html

@@ -0,0 +1,148 @@
+<!DOCTYPE html><html><head><script>var __ezHttpConsent={setByCat:function(src,tagType,attributes,category,force){var setScript=function(){if(force||window.ezTcfConsent[category]){var scriptElement=document.createElement(tagType);scriptElement.src=src;attributes.forEach(function(attr){for(var key in attr){if(attr.hasOwnProperty(key)){scriptElement.setAttribute(key,attr[key]);}}});var firstScript=document.getElementsByTagName(tagType)[0];firstScript.parentNode.insertBefore(scriptElement,firstScript);}};if(force||(window.ezTcfConsent&&window.ezTcfConsent.loaded)){setScript();}else if(typeof getEzConsentData==="function"){getEzConsentData().then(function(ezTcfConsent){if(ezTcfConsent&&ezTcfConsent.loaded){setScript();}else{console.error("cannot get ez consent data");force=true;setScript();}});}else{force=true;setScript();console.error("getEzConsentData is not a function");}},};</script>
+<script>var ezTcfConsent=window.ezTcfConsent?window.ezTcfConsent:{loaded:false,store_info:false,develop_and_improve_services:false,measure_ad_performance:false,measure_content_performance:false,select_basic_ads:false,create_ad_profile:false,select_personalized_ads:false,create_content_profile:false,select_personalized_content:false,understand_audiences:false,use_limited_data_to_select_content:false,};function getEzConsentData(){return new Promise(function(resolve){document.addEventListener("ezConsentEvent",function(event){var ezTcfConsent=event.detail.ezTcfConsent;resolve(ezTcfConsent);});});}</script>
+<script>function _setEzCookies(ezConsentData){var cookies=[{name:"ezosuibasgeneris-1",value:"fd2165e4-6c59-4ab4-7fca-f093aabdcd0a; Path=/; Domain=filesamples.com; Expires=Wed, 19 Nov 2025 09:03:25 UTC; Secure; SameSite=None",tcfCategory:"understand_audiences",isEzoic:"true",},{name:"ezopvc_176527",value:"20; Path=/; Domain=filesamples.com; Expires=Tue, 19 Nov 2024 09:33:26 UTC",tcfCategory:"understand_audiences",isEzoic:"true",},{name:"ezoab_176527",value:"mod274; Path=/; Domain=filesamples.com; Max-Age=7200",tcfCategory:"store_info",isEzoic:"true",},{name:"active_template::176527",value:"pub_site.1732007006; Path=/; Domain=filesamples.com; Expires=Thu, 21 Nov 2024 09:03:26 UTC",tcfCategory:"store_info",isEzoic:"true",},{name:"ezoadgid_176527",value:"-1; Path=/; Domain=filesamples.com; Max-Age=1800",tcfCategory:"understand_audiences",isEzoic:"true",}];for(var i=0;i<cookies.length;i++){var cookie=cookies[i];if(ezConsentData&&ezConsentData.loaded&&ezConsentData[cookie.tcfCategory]){document.cookie=cookie.name+"="+cookie.value;}}}
+if(window.ezTcfConsent&&window.ezTcfConsent.loaded){_setEzCookies(window.ezTcfConsent);}else if(typeof getEzConsentData==="function"){getEzConsentData().then(function(ezTcfConsent){if(ezTcfConsent&&ezTcfConsent.loaded){_setEzCookies(window.ezTcfConsent);}else{console.error("cannot get ez consent data");_setEzCookies(window.ezTcfConsent);}});}else{console.error("getEzConsentData is not a function");_setEzCookies(window.ezTcfConsent);}</script><script type="text/javascript" data-ezscrex='false' data-cfasync='false'>window._ezaq = Object.assign({"edge_cache_status":11,"edge_response_time":740,"url":"https://filesamples.com/samples/code/html/sample1.html"}, typeof window._ezaq !== "undefined" ? window._ezaq : {});</script><script type="text/javascript" data-ezscrex='false' data-cfasync='false'>window._ezaq = Object.assign({"ab_test_id":"mod274"}, typeof window._ezaq !== "undefined" ? window._ezaq : {});window.__ez=window.__ez||{};window.__ez.tf={"vidSig":"true"};</script><script type="text/javascript" data-ezscrex='false' data-cfasync='false'>window.ezDisableAds = true;</script><script data-ezscrex='false' data-cfasync='false' data-pagespeed-no-defer>var __ez=__ez||{};__ez.stms=Date.now();__ez.evt={};__ez.script={};__ez.ck=__ez.ck||{};__ez.template={};__ez.template.isOrig=false;__ez.queue=function(){var e=0,i=0,t=[],n=!1,o=[],r=[],s=!0,a=function(e,i,n,o,r,s,a){var l=arguments.length>7&&void 0!==arguments[7]?arguments[7]:window,d=this;this.name=e,this.funcName=i,this.parameters=null===n?null:w(n)?n:[n],this.isBlock=o,this.blockedBy=r,this.deleteWhenComplete=s,this.isError=!1,this.isComplete=!1,this.isInitialized=!1,this.proceedIfError=a,this.fWindow=l,this.isTimeDelay=!1,this.process=function(){u("... func = "+e),d.isInitialized=!0,d.isComplete=!0,u("... func.apply: "+e);var i=d.funcName.split("."),n=null,o=this.fWindow||window;i.length>3||(n=3===i.length?o[i[0]][i[1]][i[2]]:2===i.length?o[i[0]][i[1]]:o[d.funcName]),null!=n&&n.apply(null,this.parameters),!0===d.deleteWhenComplete&&delete t[e],!0===d.isBlock&&(u("----- F'D: "+d.name),m())}},l=function(e,i,t,n,o,r,s){var a=arguments.length>7&&void 0!==arguments[7]?arguments[7]:window,l=this;this.name=e,this.path=i,this.async=o,this.defer=r,this.isBlock=t,this.blockedBy=n,this.isInitialized=!1,this.isError=!1,this.isComplete=!1,this.proceedIfError=s,this.fWindow=a,this.isTimeDelay=!1,this.isPath=function(e){return"/"===e[0]&&"/"!==e[1]},this.getSrc=function(e){return void 0!==window.__ezScriptHost&&this.isPath(e)&&"banger.js"!==this.name?window.__ezScriptHost+e:e},this.process=function(){l.isInitialized=!0,u("... file = "+e);var i=this.fWindow?this.fWindow.document:document,t=i.createElement("script");t.src=this.getSrc(this.path),!0===o?t.async=!0:!0===r&&(t.defer=!0),t.onerror=function(){var e={url:window.location.href,name:l.name,path:l.path,user_agent:window.navigator.userAgent};"undefined"!=typeof _ezaq&&(e.pageview_id=_ezaq.page_view_id);var i=encodeURIComponent(JSON.stringify(e)),t=new XMLHttpRequest;t.open("GET","//g.ezoic.net/ezqlog?d="+i,!0),t.send(),u("----- ERR'D: "+l.name),l.isError=!0,!0===l.isBlock&&m()},t.onreadystatechange=t.onload=function(){var e=t.readyState;u("----- F'D: "+l.name),e&&!/loaded|complete/.test(e)||(l.isComplete=!0,!0===l.isBlock&&m())},i.getElementsByTagName("head")[0].appendChild(t)}},d=function(e,i){this.name=e,this.path="",this.async=!1,this.defer=!1,this.isBlock=!1,this.blockedBy=[],this.isInitialized=!0,this.isError=!1,this.isComplete=i,this.proceedIfError=!1,this.isTimeDelay=!1,this.process=function(){}};function c(e,i,n,s,a,d,c,f,u){var m=new l(e,i,n,s,a,d,c,u);!0===f?o[e]=m:r[e]=m,t[e]=m,h(m)}function h(e){!0!==f(e)&&0!=s&&e.process()}function f(e){if(!0===e.isTimeDelay&&!1===n)return u(e.name+" blocked = TIME DELAY!"),!0;if(w(e.blockedBy))for(var i=0;i<e.blockedBy.length;i++){var o=e.blockedBy[i];if(!1===t.hasOwnProperty(o))return u(e.name+" blocked = "+o),!0;if(!0===e.proceedIfError&&!0===t[o].isError)return!1;if(!1===t[o].isComplete)return u(e.name+" blocked = "+o),!0}return!1}function u(e){var i=window.location.href,t=new RegExp("[?&]ezq=([^&#]*)","i").exec(i);"1"===(t?t[1]:null)&&console.debug(e)}function m(){++e>200||(u("let's go"),p(o),p(r))}function p(e){for(var i in e)if(!1!==e.hasOwnProperty(i)){var t=e[i];!0===t.isComplete||f(t)||!0===t.isInitialized||!0===t.isError?!0===t.isError?u(t.name+": error"):!0===t.isComplete?u(t.name+": complete already"):!0===t.isInitialized&&u(t.name+": initialized already"):t.process()}}function w(e){return"[object Array]"==Object.prototype.toString.call(e)}return window.addEventListener("load",(function(){setTimeout((function(){n=!0,u("TDELAY -----"),m()}),5e3)}),!1),{addFile:c,addFileOnce:function(e,i,n,o,r,s,a,l,d){t[e]||c(e,i,n,o,r,s,a,l,d)},addDelayFile:function(e,i){var n=new l(e,i,!1,[],!1,!1,!0);n.isTimeDelay=!0,u(e+" ...  FILE! TDELAY"),r[e]=n,t[e]=n,h(n)},addFunc:function(e,n,s,l,d,c,f,u,m,p){!0===c&&(e=e+"_"+i++);var w=new a(e,n,s,l,d,f,u,p);!0===m?o[e]=w:r[e]=w,t[e]=w,h(w)},addDelayFunc:function(e,i,n){var o=new a(e,i,n,!1,[],!0,!0);o.isTimeDelay=!0,u(e+" ...  FUNCTION! TDELAY"),r[e]=o,t[e]=o,h(o)},items:t,processAll:m,setallowLoad:function(e){s=e},markLoaded:function(e){if(e&&0!==e.length){if(e in t){var i=t[e];!0===i.isComplete?u(i.name+" "+e+": error loaded duplicate"):(i.isComplete=!0,i.isInitialized=!0)}else t[e]=new d(e,!0);u("markLoaded dummyfile: "+t[e].name)}},logWhatsBlocked:function(){for(var e in t)!1!==t.hasOwnProperty(e)&&f(t[e])}}}();__ez.evt.add=function(e,t,n){e.addEventListener?e.addEventListener(t,n,!1):e.attachEvent?e.attachEvent("on"+t,n):e["on"+t]=n()},__ez.evt.remove=function(e,t,n){e.removeEventListener?e.removeEventListener(t,n,!1):e.detachEvent?e.detachEvent("on"+t,n):delete e["on"+t]};__ez.script.add=function(e){var t=document.createElement("script");t.src=e,t.async=!0,t.type="text/javascript",document.getElementsByTagName("head")[0].appendChild(t)};__ez.dot={};__ez.queue.addFile('/detroitchicago/boise.js', '/detroitchicago/boise.js?gcb=195-2&cb=5', true, [], true, false, true, false);__ez.queue.addFile('/parsonsmaize/abilene.js', '/parsonsmaize/abilene.js?gcb=195-2&cb=41', true, [], true, false, true, false);</script>
+<script data-ezscrex="false" type="text/javascript" data-cfasync="false">window._ezaq = Object.assign({"ad_cache_level":1,"adpicker_placement_cnt":0,"ai_placeholder_cache_level":1,"ai_placeholder_placement_cnt":-1,"domain_id":176527,"ezcache_level":0,"ezcache_skip_code":14,"has_bad_image":0,"has_bad_words":0,"is_sitespeed":0,"lt_cache_level":0,"response_size":7105,"response_size_orig":1717,"response_time_orig":660,"template_id":147,"url":"https://filesamples.com/samples/code/html/sample1.html","word_count":196,"worst_bad_word_level":0}, typeof window._ezaq !== "undefined" ? window._ezaq : {});__ez.queue.markLoaded('ezaqBaseReady');</script>
+<script type="text/javascript">(function(){function storageAvailable(type){var storage;try{storage=window[type];var x='__storage_test__';storage.setItem(x,x);storage.removeItem(x);return true;}
+catch(e){return e instanceof DOMException&&(e.code===22||e.code===1014||e.name==='QuotaExceededError'||e.name==='NS_ERROR_DOM_QUOTA_REACHED')&&(storage&&storage.length!==0);}}
+function remove_ama_config(){if(storageAvailable('localStorage')){localStorage.removeItem("google_ama_config");}}
+remove_ama_config()})()</script>
+<script type="text/javascript">var ezoicTestActive = true</script>
+<script type='text/javascript' data-ezscrex='false' data-cfasync='false'>
+window.ezAnalyticsStatic = true;
+
+function analyticsAddScript(script) {
+	var ezDynamic = document.createElement('script');
+	ezDynamic.type = 'text/javascript';
+	ezDynamic.innerHTML = script;
+	document.head.appendChild(ezDynamic);
+}
+function getCookiesWithPrefix() {
+    var allCookies = document.cookie.split(';');
+    var cookiesWithPrefix = {};
+
+    for (var i = 0; i < allCookies.length; i++) {
+        var cookie = allCookies[i].trim();
+
+        for (var j = 0; j < arguments.length; j++) {
+            var prefix = arguments[j];
+            if (cookie.indexOf(prefix) === 0) {
+                var cookieParts = cookie.split('=');
+                var cookieName = cookieParts[0];
+                var cookieValue = cookieParts.slice(1).join('=');
+                cookiesWithPrefix[cookieName] = decodeURIComponent(cookieValue);
+                break; // Once matched, no need to check other prefixes
+            }
+        }
+    }
+
+    return cookiesWithPrefix;
+}
+function productAnalytics() {
+	var d = {"pr":[1,6,3],"aop":{"2":0,"4":147},"omd5":"6d6bb459461be9d3f11aa7f4f6b6031d"};
+	d.u = _ezaq.url;
+	d.p = _ezaq.page_view_id;
+	d.v = _ezaq.visit_uuid;
+	d.ab = _ezaq.ab_test_id;
+	d.e = JSON.stringify(_ezaq);
+	d.ref = document.referrer;
+	d.c = getCookiesWithPrefix('active_template', 'ez', 'lp_');
+	if(typeof ez_utmParams !== 'undefined') {
+		d.utm = ez_utmParams;
+	}
+
+	var dataText = JSON.stringify(d);
+	var xhr = new XMLHttpRequest();
+	xhr.open('POST','/ezais/analytics?cb=1', true);
+	xhr.onload = function () {
+		if (xhr.status!=200) {
+            return;
+		}
+
+        if(document.readyState !== 'loading') {
+            analyticsAddScript(xhr.response);
+            return;
+        }
+
+        var eventFunc = function() {
+            if(document.readyState === 'loading') {
+                return;
+            }
+            document.removeEventListener('readystatechange', eventFunc, false);
+            analyticsAddScript(xhr.response);
+        };
+
+        document.addEventListener('readystatechange', eventFunc, false);
+	};
+	xhr.setRequestHeader('Content-Type','text/plain');
+	xhr.send(dataText);
+}
+__ez.queue.addFunc("productAnalytics", "productAnalytics", null, true, ['ezaqBaseReady'], false, false, false, true);
+</script><base href="https://filesamples.com/samples/code/html/sample1.html"/>
+	<title>Sample HTML 1</title>
+<script type='text/javascript'>
+var ezoTemplate = 'pub_site_noads';
+var ezouid = '1';
+var ezoFormfactor = '1';
+</script><script data-ezscrex="false" type='text/javascript'>
+var soc_app_id = '0';
+var did = 176527;
+var ezdomain = 'filesamples.com';
+var ezoicSearchable = 1;
+</script>
+<link rel='canonical' href='https://filesamples.com/samples/code/html/sample1.html' /></head>
+<body><script>__ez.queue.addFile('/detroitchicago/omaha.js', '/detroitchicago/omaha.js?gcb=2&cb=6', true, [], true, false, true, false);</script>
+	Sample HTML 1
+	<h1>Minime vero, inquit ille, consentit.</h1>
+
+	<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit. Inscite autem medicinae et gubernationis ultimum cum ultimo sapientiae comparatur. Cur igitur, cum de re conveniat, non malumus usitate loqui? </p>
+
+	<ol>
+		<li>Si qua in iis corrigere voluit, deteriora fecit.</li>
+		<li>At quicum ioca seria, ut dicitur, quicum arcana, quicum occulta omnia?</li>
+		<li>An dolor longissimus quisque miserrimus, voluptatem non optabiliorem diuturnitas facit?</li>
+		<li>Multoque hoc melius nos veriusque quam Stoici.</li>
+		<li>Stuprata per vim Lucretia a regis filio testata civis se ipsa interemit.</li>
+		<li>Ego vero isti, inquam, permitto.</li>
+	</ol>
+
+
+	<p>Graecum enim hunc versum nostis omnes-: Suavis laborum est praeteritorum memoria. Qui enim existimabit posse se miserum esse beatus non erit. Si qua in iis corrigere voluit, deteriora fecit. <a href="http://loripsum.net/" target="_blank">Si qua in iis corrigere voluit, deteriora fecit.</a> Dic in quovis conventu te omnia facere, ne doleas. Tu quidem reddes; </p>
+
+	<ul>
+		<li>Duo Reges: constructio interrete.</li>
+		<li>Contineo me ab exemplis.</li>
+		<li>Quo plebiscito decreta a senatu est consuli quaestio Cn.</li>
+		<li>Quicquid porro animo cernimus, id omne oritur a sensibus;</li>
+		<li>Eam si varietatem diceres, intellegerem, ut etiam non dicente te intellego;</li>
+		<li>Qua ex cognitione facilior facta est investigatio rerum occultissimarum.</li>
+	</ul>
+
+
+	<blockquote cite="http://loripsum.net">
+		Me igitur ipsum ames oportet, non mea, si veri amici futuri sumus.
+	</blockquote>
+
+<script type='text/javascript' data-ezscrex='false'>var EmbedExclusionEvaluated = 'exempt'; var EzoicMagicPlayerExclusionSelectors = [".humix-off"];var EzoicMagicPlayerInclusionSelectors = [];var EzoicPreferredLocation = '1';</script>
+<script type='text/javascript' style='display:none;' async>if (typeof window.__ez !== 'undefined' && window.__ez?.queue?.addFileOnce) {window.__ez.queue.addFileOnce('identity', 'https://go.ezodn.com/detroitchicago/indy.js?cb=6&gcb=0', true, [], true, false, false, true);} </script><script data-cfasync="false">function _emitEzConsentEvent(){var customEvent=new CustomEvent("ezConsentEvent",{detail:{ezTcfConsent:window.ezTcfConsent},bubbles:true,cancelable:true,});document.dispatchEvent(customEvent);}
+(function(window,document){function _setAllEzConsentTrue(){window.ezTcfConsent.loaded=true;window.ezTcfConsent.store_info=true;window.ezTcfConsent.develop_and_improve_services=true;window.ezTcfConsent.measure_ad_performance=true;window.ezTcfConsent.measure_content_performance=true;window.ezTcfConsent.select_basic_ads=true;window.ezTcfConsent.create_ad_profile=true;window.ezTcfConsent.select_personalized_ads=true;window.ezTcfConsent.create_content_profile=true;window.ezTcfConsent.select_personalized_content=true;window.ezTcfConsent.understand_audiences=true;window.ezTcfConsent.use_limited_data_to_select_content=true;window.ezTcfConsent.select_personalized_content=true;}
+function _clearEzConsentCookie(){document.cookie="ezCMPCookieConsent=tcf2;Domain=.filesamples.com;Path=/;expires=Thu, 01 Jan 1970 00:00:00 GMT";}
+_clearEzConsentCookie();if(typeof window.__tcfapi!=="undefined"){window.ezgconsent=false;var amazonHasRun=false;function _ezAllowed(tcdata,purpose){return(tcdata.purpose.consents[purpose]||tcdata.purpose.legitimateInterests[purpose]);}
+function _reloadAds(){if(typeof window.ezorefgsl==="function"&&typeof window.ezslots==="object"){if(typeof __ezapsFetchBids=="function"&&amazonHasRun===false){ezapsFetchBids(__ezaps);if(typeof __ezapsVideo!="undefined"){ezapsFetchBids(__ezapsVideo,"video");}
+amazonHasRun=true;}
+var slots=[];for(var i=0;i<window.ezslots.length;i++){if(window[window.ezslots[i]]&&typeof window[window.ezslots[i]]==="object"){slots.push(window[window.ezslots[i]]);}else{setTimeout(_reloadAds,100);return false;}}
+for(var i=0;i<slots.length;i++){window.ezorefgsl(slots[i]);}}else if(!window.ezadtimeoutset){window.ezadtimeoutset=true;setTimeout(_reloadAds,100);}}
+function _handleConsentDecision(tcdata){window.ezTcfConsent.loaded=true;if(!tcdata.vendor.consents["347"]&&!tcdata.vendor.legitimateInterests["347"]){window._emitEzConsentEvent();return;}
+window.ezTcfConsent.store_info=_ezAllowed(tcdata,"1");window.ezTcfConsent.develop_and_improve_services=_ezAllowed(tcdata,"10");window.ezTcfConsent.measure_content_performance=_ezAllowed(tcdata,"8");window.ezTcfConsent.select_basic_ads=_ezAllowed(tcdata,"2");window.ezTcfConsent.create_ad_profile=_ezAllowed(tcdata,"3");window.ezTcfConsent.select_personalized_ads=_ezAllowed(tcdata,"4");window.ezTcfConsent.create_content_profile=_ezAllowed(tcdata,"5");window.ezTcfConsent.measure_ad_performance=_ezAllowed(tcdata,"7");window.ezTcfConsent.use_limited_data_to_select_content=_ezAllowed(tcdata,"11");window.ezTcfConsent.select_personalized_content=_ezAllowed(tcdata,"6");window.ezTcfConsent.understand_audiences=_ezAllowed(tcdata,"9");window._emitEzConsentEvent();}
+function _handleGoogleConsentV2(tcdata){if(!tcdata||!tcdata.purpose||!tcdata.purpose.consents){return;}
+var googConsentV2={};if(tcdata.purpose.consents[1]){googConsentV2.ad_storage='granted';googConsentV2.analytics_storage='granted';}
+if(tcdata.purpose.consents[3]&&tcdata.purpose.consents[4]){googConsentV2.ad_personalization='granted';}
+if(tcdata.purpose.consents[1]&&tcdata.purpose.consents[7]){googConsentV2.ad_user_data='granted';}
+if(googConsentV2.analytics_storage=='denied'){gtag('set','url_passthrough',true);}
+gtag('consent','update',googConsentV2);}
+__tcfapi("addEventListener",2,function(tcdata,success){if(!success||!tcdata){window._emitEzConsentEvent();return;}
+if(!tcdata.gdprApplies){_setAllEzConsentTrue();window._emitEzConsentEvent();return;}
+if(tcdata.eventStatus==="useractioncomplete"||tcdata.eventStatus==="tcloaded"){if(typeof gtag!='undefined'){_handleGoogleConsentV2(tcdata);}
+_handleConsentDecision(tcdata);if(tcdata.purpose.consents["1"]===true&&tcdata.vendor.consents["755"]!==false){window.ezgconsent=true;(adsbygoogle=window.adsbygoogle||[]).pauseAdRequests=0;_reloadAds();}else{_reloadAds();}
+if(window.__ezconsent){__ezconsent.setEzoicConsentSettings(ezConsentCategories);}
+__tcfapi("removeEventListener",2,function(success){return null;},tcdata.listenerId);if(!(tcdata.purpose.consents["1"]===true&&_ezAllowed(tcdata,"2")&&_ezAllowed(tcdata,"3")&&_ezAllowed(tcdata,"4"))){if(typeof __ez=="object"&&typeof __ez.bit=="object"&&typeof window["_ezaq"]=="object"&&typeof window["_ezaq"]["page_view_id"]=="string"){__ez.bit.Add(window["_ezaq"]["page_view_id"],[new __ezDotData("non_personalized_ads",true),]);}}}});}else{_setAllEzConsentTrue();window._emitEzConsentEvent();}})(window,document);</script></body></html>

sdk/python/test/test_data/test.json

@@ -0,0 +1,107 @@
+{
+    "单车": [
+        "自行车"
+    ],
+    "青禾服装": [
+        "青禾服饰"
+    ],
+    "救济灾民": [
+        "救助",
+        "灾民救济",
+        "赈济"
+    ],
+    "左移": [],
+    "低速": [],
+    "雨果网": [],
+    "钢小二": [
+        "成立于2013年,位于江苏省无锡市,是一家以从事研究和试验发展为主的企业"
+    ],
+    "第五项": [
+        "5项"
+    ],
+    "铸排机": [
+        "机排",
+        "排铸机",
+        "排铸"
+    ],
+    "金淳高分子": [],
+    "麦门冬汤": [],
+    "错位": [],
+    "佰特吉姆": [],
+    "楼体": [],
+    "展美科技": [
+        "美展"
+    ],
+    "中寮": [],
+    "贪官汙吏": [
+        "...",
+        "贪吏",
+        "贪官污吏"
+    ],
+    "掩蔽部": [
+        "掩 蔽 部"
+    ],
+    "海宏智能": [],
+    "中寰": [],
+    "万次": [],
+    "领星资本": [
+        "星领"
+    ],
+    "肯讯": [],
+    "坎肩": [],
+    "爱农人": [],
+    "易美餐": [],
+    "寸丝半粟": [],
+    "罗丹萍": [],
+    "转导物": [],
+    "泊寓": [],
+    "万欧": [
+        "欧万"
+    ],
+    "友聚惠": [
+        "友惠",
+        "惠友"
+    ],
+    "舞牙弄爪": [
+        ":形容凶猛的样子,比喻威胁、恐吓",
+        "原形容猛兽的凶相,后常用来比喻猖狂凶恶的样子",
+        "成语解释:原形容猛兽的凶相,后常用来比喻猖狂凶恶的样子",
+        "原形容猛兽的凶相,后常用来比喻猖狂(好工具hao86.com",
+        "牙舞爪",
+        "形容猛兽凶恶可怕。也比喻猖狂凶恶",
+        "舞爪"
+    ],
+    "上海致上": [
+        "上海上",
+        "上海市"
+    ],
+    "迪因加": [],
+    "李正茂": [],
+    "君来投": [],
+    "双掌空": [
+        "双掌 空",
+        "空掌",
+        "两手空空"
+    ],
+    "浩石": [
+        "石浩",
+        "皓石"
+    ],
+    "云阅文学": [],
+    "阿斯帕": [],
+    "中导": [],
+    "以诚相待": [],
+    "中融金服": [],
+    "尚股网": [],
+    "叶立钦": [
+        "叶利钦"
+    ],
+    "新信钱包": [
+        "信信"
+    ],
+    "赛苏投资": [
+        "投资者"
+    ],
+    "售价": [],
+    "帮医网": []
+}

sdk/python/test/test_data/test.md

@@ -0,0 +1,21 @@
+Quod equidem non reprehendo;
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quibus natura iure responderit non esse verum aliunde finem beate vivendi, a se principia rei gerendae peti; Quae enim adhuc protulisti, popularia sunt, ego autem a te elegantiora desidero. Duo Reges: constructio interrete. Tum Lucius: Mihi vero ista valde probata sunt, quod item fratri puto. Bestiarum vero nullum iudicium puto. Nihil enim iam habes, quod ad corpus referas; Deinde prima illa, quae in congressu solemus: Quid tu, inquit, huc? Et homini, qui ceteris animantibus plurimum praestat, praecipue a natura nihil datum esse dicemus?
+
+Iam id ipsum absurdum, maximum malum neglegi. Quod ea non occurrentia fingunt, vincunt Aristonem; Atqui perspicuum est hominem e corpore animoque constare, cum primae sint animi partes, secundae corporis. Fieri, inquam, Triari, nullo pacto potest, ut non dicas, quid non probes eius, a quo dissentias. Equidem e Cn. An dubium est, quin virtus ita maximam partem optineat in rebus humanis, ut reliquas obruat?
+
+Quis istum dolorem timet?
+Summus dolor plures dies manere non potest? Dicet pro me ipsa virtus nec dubitabit isti vestro beato M. Tubulum fuisse, qua illum, cuius is condemnatus est rogatione, P. Quod si ita sit, cur opera philosophiae sit danda nescio.
+
+Ex eorum enim scriptis et institutis cum omnis doctrina liberalis, omnis historia.
+Quod si ita est, sequitur id ipsum, quod te velle video, omnes semper beatos esse sapientes. Cum enim fertur quasi torrens oratio, quamvis multa cuiusque modi rapiat, nihil tamen teneas, nihil apprehendas, nusquam orationem rapidam coerceas. Ita redarguitur ipse a sese, convincunturque scripta eius probitate ipsius ac moribus. At quanta conantur! Mundum hunc omnem oppidum esse nostrum! Incendi igitur eos, qui audiunt, vides. Vide, ne magis, inquam, tuum fuerit, cum re idem tibi, quod mihi, videretur, non nova te rebus nomina inponere. Qui-vere falsone, quaerere mittimus-dicitur oculis se privasse; Si ista mala sunt, in quae potest incidere sapiens, sapientem esse non esse ad beate vivendum satis. At vero si ad vitem sensus accesserit, ut appetitum quendam habeat et per se ipsa moveatur, quid facturam putas?
+
+Quem si tenueris, non modo meum Ciceronem, sed etiam me ipsum abducas licebit.
+Stulti autem malorum memoria torquentur, sapientes bona praeterita grata recordatione renovata delectant.
+Esse enim quam vellet iniquus iustus poterat inpune.
+Quae autem natura suae primae institutionis oblita est?
+Verum tamen cum de rebus grandioribus dicas, ipsae res verba rapiunt;
+Hoc est non modo cor non habere, sed ne palatum quidem.
+Voluptatem cum summum bonum diceret, primum in eo ipso parum vidit, deinde hoc quoque alienum; Sed tu istuc dixti bene Latine, parum plane. Nam haec ipsa mihi erunt in promptu, quae modo audivi, nec ante aggrediar, quam te ab istis, quos dicis, instructum videro. Fatebuntur Stoici haec omnia dicta esse praeclare, neque eam causam Zenoni desciscendi fuisse. Non autem hoc: igitur ne illud quidem. Ratio quidem vestra sic cogit. Cum audissem Antiochum, Brute, ut solebam, cum M. An quod ita callida est, ut optime possit architectari voluptates?
+
+Idemne, quod iucunde?
+Haec mihi videtur delicatior, ut ita dicam, molliorque ratio, quam virtutis vis gravitasque postulat. Sed quoniam et advesperascit et mihi ad villam revertendum est, nunc quidem hactenus; Cuius ad naturam apta ratio vera illa et summa lex a philosophis dicitur. Neque solum ea communia, verum etiam paria esse dixerunt. Sed nunc, quod agimus; A mene tu?

sdk/python/test/test_data/test.txt

@@ -1,3 +1,21 @@
-test
-test
-test
+Quod equidem non reprehendo;
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quibus natura iure responderit non esse verum aliunde finem beate vivendi, a se principia rei gerendae peti; Quae enim adhuc protulisti, popularia sunt, ego autem a te elegantiora desidero. Duo Reges: constructio interrete. Tum Lucius: Mihi vero ista valde probata sunt, quod item fratri puto. Bestiarum vero nullum iudicium puto. Nihil enim iam habes, quod ad corpus referas; Deinde prima illa, quae in congressu solemus: Quid tu, inquit, huc? Et homini, qui ceteris animantibus plurimum praestat, praecipue a natura nihil datum esse dicemus?
+
+Iam id ipsum absurdum, maximum malum neglegi. Quod ea non occurrentia fingunt, vincunt Aristonem; Atqui perspicuum est hominem e corpore animoque constare, cum primae sint animi partes, secundae corporis. Fieri, inquam, Triari, nullo pacto potest, ut non dicas, quid non probes eius, a quo dissentias. Equidem e Cn. An dubium est, quin virtus ita maximam partem optineat in rebus humanis, ut reliquas obruat?
+
+Quis istum dolorem timet?
+Summus dolor plures dies manere non potest? Dicet pro me ipsa virtus nec dubitabit isti vestro beato M. Tubulum fuisse, qua illum, cuius is condemnatus est rogatione, P. Quod si ita sit, cur opera philosophiae sit danda nescio.
+
+Ex eorum enim scriptis et institutis cum omnis doctrina liberalis, omnis historia.
+Quod si ita est, sequitur id ipsum, quod te velle video, omnes semper beatos esse sapientes. Cum enim fertur quasi torrens oratio, quamvis multa cuiusque modi rapiat, nihil tamen teneas, nihil apprehendas, nusquam orationem rapidam coerceas. Ita redarguitur ipse a sese, convincunturque scripta eius probitate ipsius ac moribus. At quanta conantur! Mundum hunc omnem oppidum esse nostrum! Incendi igitur eos, qui audiunt, vides. Vide, ne magis, inquam, tuum fuerit, cum re idem tibi, quod mihi, videretur, non nova te rebus nomina inponere. Qui-vere falsone, quaerere mittimus-dicitur oculis se privasse; Si ista mala sunt, in quae potest incidere sapiens, sapientem esse non esse ad beate vivendum satis. At vero si ad vitem sensus accesserit, ut appetitum quendam habeat et per se ipsa moveatur, quid facturam putas?
+
+Quem si tenueris, non modo meum Ciceronem, sed etiam me ipsum abducas licebit.
+Stulti autem malorum memoria torquentur, sapientes bona praeterita grata recordatione renovata delectant.
+Esse enim quam vellet iniquus iustus poterat inpune.
+Quae autem natura suae primae institutionis oblita est?
+Verum tamen cum de rebus grandioribus dicas, ipsae res verba rapiunt;
+Hoc est non modo cor non habere, sed ne palatum quidem.
+Voluptatem cum summum bonum diceret, primum in eo ipso parum vidit, deinde hoc quoque alienum; Sed tu istuc dixti bene Latine, parum plane. Nam haec ipsa mihi erunt in promptu, quae modo audivi, nec ante aggrediar, quam te ab istis, quos dicis, instructum videro. Fatebuntur Stoici haec omnia dicta esse praeclare, neque eam causam Zenoni desciscendi fuisse. Non autem hoc: igitur ne illud quidem. Ratio quidem vestra sic cogit. Cum audissem Antiochum, Brute, ut solebam, cum M. An quod ita callida est, ut optime possit architectari voluptates?
+
+Idemne, quod iucunde?
+Haec mihi videtur delicatior, ut ita dicam, molliorque ratio, quam virtutis vis gravitasque postulat. Sed quoniam et advesperascit et mihi ad villam revertendum est, nunc quidem hactenus; Cuius ad naturam apta ratio vera illa et summa lex a philosophis dicitur. Neque solum ea communia, verum etiam paria esse dixerunt. Sed nunc, quod agimus; A mene tu?

sdk/python/test/test_data/test1.txt

@@ -1,4 +0,0 @@
-test1
-test1
-aaaa document  args arg
-rag document

sdk/python/test/test_data/test2.txt

@@ -1,4 +0,0 @@
-test22
-test22
-aaaa document  args arg
-rag document

sdk/python/test/test_data/test3.txt

@@ -1,4 +0,0 @@
-test3
-test333
-aaaa document  args arg
-rag document