init python part (#7)

.gitignore

@@ -2,6 +2,7 @@
 # will have compiled files and executables
 debug/
 target/
+__pycache__/
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
@@ -12,6 +13,7 @@ Cargo.lock
 
 # MSVC Windows builds of rustc generate these, which store debugging information
 *.pdb
+*.trie
 
 .idea/
 .env

python/conf/logging.json

@@ -0,0 +1,41 @@
+{
+    "version":1,
+    "disable_existing_loggers":false,
+    "formatters":{
+        "simple":{
+            "format":"%(asctime)s - %(name)s - %(levelname)s - %(filename)s - %(lineno)d - %(message)s"
+        }
+    },
+    "handlers":{
+        "console":{
+            "class":"logging.StreamHandler",
+            "level":"DEBUG",
+            "formatter":"simple",
+            "stream":"ext://sys.stdout"
+        },
+        "info_file_handler":{
+            "class":"logging.handlers.TimedRotatingFileHandler",
+            "level":"INFO",
+            "formatter":"simple",
+            "filename":"log/info.log",
+            "when": "MIDNIGHT",
+            "interval":1,
+            "backupCount":30,
+            "encoding":"utf8"
+        },
+        "error_file_handler":{
+            "class":"logging.handlers.TimedRotatingFileHandler",
+            "level":"ERROR",
+            "formatter":"simple",
+            "filename":"log/errors.log",
+            "when": "MIDNIGHT",
+            "interval":1,
+            "backupCount":30,
+            "encoding":"utf8"
+        }
+    },
+    "root":{
+        "level":"DEBUG",
+        "handlers":["console","info_file_handler","error_file_handler"]
+    }
+}

python/conf/mapping.json

@@ -0,0 +1,140 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": 4,
+      "number_of_replicas": 0,
+      "refresh_interval" : "1000ms"
+    },
+    "similarity": {
+        "scripted_sim": {
+          "type": "scripted",
+          "script": {
+            "source": "double idf = Math.log(1+(field.docCount-term.docFreq+0.5)/(term.docFreq + 0.5))/Math.log(1+((field.docCount-0.5)/1.5)); return query.boost * idf * Math.min(doc.freq, 1);"
+          }
+        }
+    }
+  },
+  "mappings": {
+      "properties": {
+        "lat_lon": {"type": "geo_point", "store":"true"}
+      },
+      "date_detection": "true",
+      "dynamic_templates": [
+        {
+          "int": {
+            "match": "*_int",
+            "mapping": {
+              "type": "integer",
+              "store": "true"
+            }
+          }
+        },
+        {
+          "numeric": {
+            "match": "*_flt",
+            "mapping": {
+              "type": "float",
+              "store": true
+            }
+          }
+        },
+        {
+          "tks": {
+            "match": "*_tks",
+            "mapping": {
+              "type": "text",
+              "similarity": "scripted_sim",
+              "analyzer": "whitespace",
+              "store": true
+            }
+          }
+        },
+        {
+          "ltks":{
+            "match": "*_ltks",
+            "mapping": {
+              "type": "text",
+              "analyzer": "whitespace",
+              "store": true
+            }
+          }
+        },
+        {
+          "kwd": {
+            "match_pattern": "regex",
+            "match": "^(.*_(kwd|id|ids|uid|uids)|uid)$",
+            "mapping": {
+              "type": "keyword",
+              "similarity": "boolean",
+              "store": true
+            }
+          }
+        },
+        {
+          "dt": {
+            "match_pattern": "regex",
+            "match": "^.*(_dt|_time|_at)$",
+            "mapping": {
+              "type": "date",
+              "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||yyyy-MM-dd_HH:mm:ss",
+              "store": true
+            }
+          }
+        },
+        {
+          "nested": {
+            "match": "*_nst",
+            "mapping": {
+              "type": "nested"
+            }
+          }
+        },
+          {
+            "object": {
+              "match": "*_obj",
+              "mapping": {
+                "type": "object",
+                "dynamic": "true"
+              }
+            }
+          },
+        {
+            "string": {
+              "match": "*_with_weight",
+              "mapping": {
+                "type": "text",
+                "index": "false",
+                "store": true
+              }
+            }
+        },
+        {
+            "string": {
+              "match": "*_fea",
+              "mapping": {
+                "type": "rank_feature"
+              }
+            }
+        },
+        {
+            "dense_vector": {
+              "match": "*_vec",
+              "mapping": {
+                "type": "dense_vector",
+                "dims": 1024,
+                "index": true,
+                "similarity": "cosine"
+              }
+            }
+        },
+        {
+            "binary": {
+              "match": "*_bin",
+              "mapping": {
+                "type": "binary"
+              }
+            }
+        }
+      ]
+    }
+}

python/conf/sys.cnf

@@ -0,0 +1,4 @@
+[online]
+es=127.0.0.1:9200
+idx_nm=toxic
+

python/nlp/huchunk.py

@@ -0,0 +1,364 @@
+import re
+import os
+import copy
+import base64
+from dataclasses import dataclass
+from typing import List
+import numpy as np
+from io import BytesIO
+
+
+class HuChunker:
+
+    def __init__(self):
+        self.MAX_LVL = 12
+        self.proj_patt = [
+            (r"第[零一二三四五六七八九十百]+章", 1),
+            (r"第[零一二三四五六七八九十百]+[条节]", 2),
+            (r"[零一二三四五六七八九十百]+[、 　]", 3),
+            (r"[\(（][零一二三四五六七八九十百]+[）\)]", 4),
+            (r"[0-9]+(、|\.[　 ]|\.[^0-9])", 5),
+            (r"[0-9]+\.[0-9]+(、|[ 　]|[^0-9])", 6),
+            (r"[0-9]+\.[0-9]+\.[0-9]+(、|[ 　]|[^0-9])", 7),
+            (r"[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(、|[ 　]|[^0-9])", 8),
+            (r".{,48}[：:?？]@", 9),
+            (r"[0-9]+）", 10),
+            (r"[\(（][0-9]+[）\)]", 11),
+            (r"[零一二三四五六七八九十百]+是", 12),
+            (r"[⚫•➢✓ ]", 12)
+        ]
+        self.lines = []
+
+    def _garbage(self, txt):
+        patt = [
+            r"(在此保证|不得以任何形式翻版|请勿传阅|仅供内部使用|未经事先书面授权)",
+            r"(版权(归本公司)*所有|免责声明|保留一切权力|承担全部责任|特别声明|报告中涉及)",
+            r"(不承担任何责任|投资者的通知事项：|任何机构和个人|本报告仅为|不构成投资)",
+            r"(不构成对任何个人或机构投资建议|联系其所在国家|本报告由从事证券交易)",
+            r"(本研究报告由|「认可投资者」|所有研究报告均以|请发邮件至)",
+            r"(本报告仅供|市场有风险，投资需谨慎|本报告中提及的)",
+            r"(本报告反映|此信息仅供|证券分析师承诺|具备证券投资咨询业务资格)",
+            r"^(时间|签字|签章)[:：]",
+            r"(参考文献|目录索引|图表索引)",
+            r"[ ]*年[ ]+月[ ]+日",
+            r"^(中国证券业协会|[0-9]+年[0-9]+月[0-9]+日)$",
+            r"\.{10,}",
+            r"(———————END|帮我转发|欢迎收藏|快来关注我吧)"
+        ]
+        return any([re.search(p, txt) for p in patt])
+
+    def _proj_match(self, line):
+        for p, j in self.proj_patt:
+            if re.match(p, line):
+                return j
+        return
+
+    def _does_proj_match(self):
+        mat = [None for _ in range(len(self.lines))]
+        for i in range(len(self.lines)):
+            mat[i] = self._proj_match(self.lines[i])
+        return mat
+
+    def naive_text_chunk(self, text, ti="", MAX_LEN=612):
+        if text:
+            self.lines = [l.strip().replace(u'\u3000', u' ')
+                          .replace(u'\xa0', u'')
+                          for l in text.split("\n\n")]
+            self.lines = [l for l in self.lines if not self._garbage(l)]
+            self.lines = [re.sub(r"([ ]+| )", " ", l)
+                          for l in self.lines if l]
+        if not self.lines:
+            return []
+        arr = self.lines
+
+        res = [""]
+        i = 0
+        while i < len(arr):
+            a = arr[i]
+            if not a:
+                i += 1
+                continue
+            if len(a) > MAX_LEN:
+                a_ = a.split("\n")
+                if len(a_) >= 2:
+                    arr.pop(i)
+                    for j in range(2, len(a_) + 1):
+                        if len("\n".join(a_[:j])) >= MAX_LEN:
+                            arr.insert(i, "\n".join(a_[:j - 1]))
+                            arr.insert(i + 1, "\n".join(a_[j - 1:]))
+                            break
+                    else:
+                        assert False, f"Can't split: {a}"
+                    continue
+
+            if len(res[-1]) < MAX_LEN / 3:
+                res[-1] += "\n" + a
+            else:
+                res.append(a)
+            i += 1
+
+        if ti:
+            for i in range(len(res)):
+                if res[i].find("——来自") >= 0:
+                    continue
+                res[i] += f"\t——来自“{ti}”"
+
+        return res
+
+    def _merge(self):
+        # merge continuous same level text
+        lines = [self.lines[0]] if self.lines else []
+        for i in range(1, len(self.lines)):
+            if self.mat[i] == self.mat[i - 1] \
+               and len(lines[-1]) < 256 \
+               and len(self.lines[i]) < 256:
+                lines[-1] += "\n" + self.lines[i]
+                continue
+            lines.append(self.lines[i])
+        self.lines = lines
+        self.mat = self._does_proj_match()
+        return self.mat
+
+    def text_chunks(self, text):
+        if text:
+            self.lines = [l.strip().replace(u'\u3000', u' ')
+                          .replace(u'\xa0', u'')
+                          for l in re.split(r"[\r\n]", text)]
+            self.lines = [l for l in self.lines if not self._garbage(l)]
+            self.lines = [l for l in self.lines if l]
+        self.mat = self._does_proj_match()
+        mat = self._merge()
+
+        tree = []
+        for i in range(len(self.lines)):
+            tree.append({"proj": mat[i],
+                         "children": [],
+                         "read": False})
+        # find all children
+        for i in range(len(self.lines) - 1):
+            if tree[i]["proj"] is None:
+                continue
+            ed = i + 1
+            while ed < len(tree) and (tree[ed]["proj"] is None or
+                                      tree[ed]["proj"] > tree[i]["proj"]):
+                ed += 1
+
+            nxt = tree[i]["proj"] + 1
+            st = set([p["proj"] for p in tree[i + 1: ed] if p["proj"]])
+            while nxt not in st:
+                nxt += 1
+                if nxt > self.MAX_LVL:
+                    break
+            if nxt <= self.MAX_LVL:
+                for j in range(i + 1, ed):
+                    if tree[j]["proj"] is not None:
+                        break
+                    tree[i]["children"].append(j)
+                for j in range(i + 1, ed):
+                    if tree[j]["proj"] != nxt:
+                        continue
+                    tree[i]["children"].append(j)
+            else:
+                for j in range(i + 1, ed):
+                    tree[i]["children"].append(j)
+
+        # get DFS combinations, find all the paths to leaf
+        paths = []
+
+        def dfs(i, path):
+            nonlocal tree, paths
+            path.append(i)
+            tree[i]["read"] = True
+            if len(self.lines[i]) > 256:
+                paths.append(path)
+                return
+            if not tree[i]["children"]:
+                if len(path) > 1 or len(self.lines[i]) >= 32:
+                    paths.append(path)
+                return
+            for j in tree[i]["children"]:
+                dfs(j, copy.deepcopy(path))
+
+        for i, t in enumerate(tree):
+            if t["read"]:
+                continue
+            dfs(i, [])
+
+        # concat txt on the path for all paths
+        res = []
+        lines = np.array(self.lines)
+        for p in paths:
+            if len(p) < 2:
+                tree[p[0]]["read"] = False
+                continue
+            txt = "\n".join(lines[p[:-1]]) + "\n" + lines[p[-1]]
+            res.append(txt)
+        # concat continuous orphans
+        assert len(tree) == len(lines)
+        ii = 0
+        while ii < len(tree):
+            if tree[ii]["read"]:
+                ii += 1
+                continue
+            txt = lines[ii]
+            e = ii + 1
+            while e < len(tree) and not tree[e]["read"] and len(txt) < 256:
+                txt += "\n" + lines[e]
+                e += 1
+            res.append(txt)
+            ii = e
+
+        # if the node has not been read, find its daddy
+        def find_daddy(st):
+            nonlocal lines, tree
+            proj = tree[st]["proj"]
+            if len(self.lines[st]) > 512:
+                return [st]
+            if proj is None:
+                proj = self.MAX_LVL + 1
+            for i in range(st - 1, -1, -1):
+                if tree[i]["proj"] and tree[i]["proj"] < proj:
+                    a = [st] + find_daddy(i)
+                    return a
+            return []
+
+        return res
+
+
+class PdfChunker(HuChunker):
+
+    @dataclass
+    class Fields:
+        text_chunks: List = None
+        table_chunks: List = None
+
+    def __init__(self, pdf_parser):
+        self.pdf = pdf_parser
+        super().__init__()
+
+    def tableHtmls(self, pdfnm):
+        _, tbls = self.pdf(pdfnm, return_html=True)
+        res = []
+        for img, arr in tbls:
+            if arr[0].find("<table>") < 0:
+                continue
+            buffered = BytesIO()
+            if img:
+                img.save(buffered, format="JPEG")
+            img_str = base64.b64encode(
+                buffered.getvalue()).decode('utf-8') if img else ""
+            res.append({"table": arr[0], "image": img_str})
+        return res
+
+    def html(self, pdfnm):
+        txts, tbls = self.pdf(pdfnm, return_html=True)
+        res = []
+        txt_cks = self.text_chunks(txts)
+        for txt, img in [(self.pdf.remove_tag(c), self.pdf.crop(c))
+                         for c in txt_cks]:
+            buffered = BytesIO()
+            if img:
+                img.save(buffered, format="JPEG")
+            img_str = base64.b64encode(
+                buffered.getvalue()).decode('utf-8') if img else ""
+            res.append({"table": "<p>%s</p>" % txt.replace("\n", "<br/>"),
+                        "image": img_str})
+
+        for img, arr in tbls:
+            if not arr:
+                continue
+            buffered = BytesIO()
+            if img:
+                img.save(buffered, format="JPEG")
+            img_str = base64.b64encode(
+                buffered.getvalue()).decode('utf-8') if img else ""
+            res.append({"table": arr[0], "image": img_str})
+
+        return res
+
+    def __call__(self, pdfnm, return_image=True, naive_chunk=False):
+        flds = self.Fields()
+        text, tbls = self.pdf(pdfnm)
+        fnm = pdfnm
+        txt_cks = self.text_chunks(text) if not naive_chunk else \
+            self.naive_text_chunk(text, ti=fnm if isinstance(fnm, str) else "")
+        flds.text_chunks = [(self.pdf.remove_tag(c),
+                             self.pdf.crop(c) if return_image else None) for c in txt_cks]
+
+        flds.table_chunks = [(arr, img if return_image else None)
+                             for img, arr in tbls]
+        return flds
+
+
+class DocxChunker(HuChunker):
+    def __init__(self, doc_parser):
+        self.doc = doc_parser
+        super().__init__()
+
+    def _does_proj_match(self):
+        mat = []
+        for s in self.styles:
+            s = s.split(" ")[-1]
+            try:
+                mat.append(int(s))
+            except Exception as e:
+                mat.append(None)
+        return mat
+
+    def _merge(self):
+        i = 1
+        while i < len(self.lines):
+            if self.mat[i] == self.mat[i - 1] \
+               and len(self.lines[i - 1]) < 256 \
+               and len(self.lines[i]) < 256:
+                self.lines[i - 1] += "\n" + self.lines[i]
+                self.styles.pop(i)
+                self.lines.pop(i)
+                self.mat.pop(i)
+                continue
+            i += 1
+        self.mat = self._does_proj_match()
+        return self.mat
+
+    def __call__(self, fnm):
+        flds = self.Fields()
+        flds.title = os.path.splitext(
+            os.path.basename(fnm))[0] if isinstance(
+            fnm, type("")) else ""
+        secs, tbls = self.doc(fnm)
+        self.lines = [l for l, s in secs]
+        self.styles = [s for l, s in secs]
+
+        txt_cks = self.text_chunks("")
+        flds.text_chunks = [(t, None) for t in txt_cks if not self._garbage(t)]
+        flds.table_chunks = [(tb, None) for tb in tbls for t in tb if t]
+        return flds
+
+
+class ExcelChunker(HuChunker):
+    def __init__(self, excel_parser):
+        self.excel = excel_parser
+        super().__init__()
+
+    def __call__(self, fnm):
+        flds = self.Fields()
+        flds.text_chunks = [(t, None) for t in self.excel(fnm)]
+        flds.table_chunks = []
+        return flds
+
+
+if __name__ == "__main__":
+    import sys
+    sys.path.append(os.path.dirname(__file__) + "/../")
+    if sys.argv[1].split(".")[-1].lower() == "pdf":
+        from parser import PdfParser
+        ckr = PdfChunker(PdfParser())
+    if sys.argv[1].split(".")[-1].lower().find("doc") >= 0:
+        from .parser import DocxParser
+        ckr = DocxChunker(DocxParser())
+    if sys.argv[1].split(".")[-1].lower().find("xlsx") >= 0:
+        from .parser import ExcelParser
+        ckr = ExcelChunker(ExcelParser())
+
+    # ckr.html(sys.argv[1])
+    print(ckr(sys.argv[1]))

python/nlp/huqie.py

@@ -0,0 +1,411 @@
+# -*- coding: utf-8 -*-
+
+import copy
+import datrie
+import math
+import os
+import re
+import string
+import sys
+from hanziconv import HanziConv
+
+
+class Huqie:
+    def key_(self, line):
+        return str(line.lower().encode("utf-8"))[2:-1]
+
+    def rkey_(self, line):
+        return str(("DD" + (line[::-1].lower())).encode("utf-8"))[2:-1]
+
+    def loadDict_(self, fnm):
+        print("[HUQIE]:Build trie", fnm, file=sys.stderr)
+        try:
+            of = open(fnm, "r")
+            while True:
+                line = of.readline()
+                if not line:
+                    break
+                line = re.sub(r"[\r\n]+", "", line)
+                line = re.split(r"[ \t]", line)
+                k = self.key_(line[0])
+                F = int(math.log(float(line[1]) / self.DENOMINATOR) + .5)
+                if k not in self.trie_ or self.trie_[k][0] < F:
+                    self.trie_[self.key_(line[0])] = (F, line[2])
+                self.trie_[self.rkey_(line[0])] = 1
+            self.trie_.save(fnm + ".trie")
+            of.close()
+        except Exception as e:
+            print("[HUQIE]:Faild to build trie, ", fnm, e, file=sys.stderr)
+
+    def __init__(self, debug=False):
+        self.DEBUG = debug
+        self.DENOMINATOR = 1000000
+        self.trie_ = datrie.Trie(string.printable)
+        self.DIR_ = ""
+        if os.path.exists("../res/huqie.txt"):
+            self.DIR_ = "../res/huqie"
+        if os.path.exists("./res/huqie.txt"):
+            self.DIR_ = "./res/huqie"
+        if os.path.exists("./huqie.txt"):
+            self.DIR_ = "./huqie"
+        assert self.DIR_, f"【Can't find huqie】"
+
+        self.SPLIT_CHAR = r"([ ,\.<>/?;'\[\]\\`!@#$%^&*\(\)\{\}\|_+=《》，。？、；‘’：“”【】~！￥%……（）——-]+|[a-z\.-]+|[0-9,\.-]+)"
+        try:
+            self.trie_ = datrie.Trie.load(self.DIR_ + ".txt.trie")
+            return
+        except Exception as e:
+            print("[HUQIE]:Build default trie", file=sys.stderr)
+            self.trie_ = datrie.Trie(string.printable)
+
+        self.loadDict_(self.DIR_ + ".txt")
+
+    def loadUserDict(self, fnm):
+        try:
+            self.trie_ = datrie.Trie.load(fnm + ".trie")
+            return
+        except Exception as e:
+            self.trie_ = datrie.Trie(string.printable)
+        self.loadDict_(fnm)
+
+    def addUserDict(self, fnm):
+        self.loadDict_(fnm)
+
+    def _strQ2B(self, ustring):
+        """把字符串全角转半角"""
+        rstring = ""
+        for uchar in ustring:
+            inside_code = ord(uchar)
+            if inside_code == 0x3000:
+                inside_code = 0x0020
+            else:
+                inside_code -= 0xfee0
+            if inside_code < 0x0020 or inside_code > 0x7e:  # 转完之后不是半角字符返回原来的字符
+                rstring += uchar
+            else:
+                rstring += chr(inside_code)
+        return rstring
+
+    def _tradi2simp(self, line):
+        return HanziConv.toSimplified(line)
+
+    def dfs_(self, chars, s, preTks, tkslist):
+        MAX_L = 10
+        res = s
+        # if s > MAX_L or s>= len(chars):
+        if s >= len(chars):
+            tkslist.append(preTks)
+            return res
+
+        # pruning
+        S = s + 1
+        if s + 2 <= len(chars):
+            t1, t2 = "".join(chars[s:s + 1]), "".join(chars[s:s + 2])
+            if self.trie_.has_keys_with_prefix(self.key_(t1)) and not self.trie_.has_keys_with_prefix(
+                    self.key_(t2)):
+                S = s + 2
+        if len(preTks) > 2 and len(
+                preTks[-1][0]) == 1 and len(preTks[-2][0]) == 1 and len(preTks[-3][0]) == 1:
+            t1 = preTks[-1][0] + "".join(chars[s:s + 1])
+            if self.trie_.has_keys_with_prefix(self.key_(t1)):
+                S = s + 2
+
+        ################
+        for e in range(S, len(chars) + 1):
+            t = "".join(chars[s:e])
+            k = self.key_(t)
+
+            if e > s + 1 and not self.trie_.has_keys_with_prefix(k):
+                break
+
+            if k in self.trie_:
+                pretks = copy.deepcopy(preTks)
+                if k in self.trie_:
+                    pretks.append((t, self.trie_[k]))
+                else:
+                    pretks.append((t, (-12, '')))
+                res = max(res, self.dfs_(chars, e, pretks, tkslist))
+
+        if res > s:
+            return res
+
+        t = "".join(chars[s:s + 1])
+        k = self.key_(t)
+        if k in self.trie_:
+            preTks.append((t, self.trie_[k]))
+        else:
+            preTks.append((t, (-12, '')))
+
+        return self.dfs_(chars, s + 1, preTks, tkslist)
+
+    def freq(self, tk):
+        k = self.key_(tk)
+        if k not in self.trie_:
+            return 0
+        return int(math.exp(self.trie_[k][0]) * self.DENOMINATOR + 0.5)
+
+    def tag(self, tk):
+        k = self.key_(tk)
+        if k not in self.trie_:
+            return ""
+        return self.trie_[k][1]
+
+    def score_(self, tfts):
+        B = 30
+        F, L, tks = 0, 0, []
+        for tk, (freq, tag) in tfts:
+            F += freq
+            L += 0 if len(tk) < 2 else 1
+            tks.append(tk)
+        F /= len(tks)
+        L /= len(tks)
+        if self.DEBUG:
+            print("[SC]", tks, len(tks), L, F, B / len(tks) + L + F)
+        return tks, B / len(tks) + L + F
+
+    def sortTks_(self, tkslist):
+        res = []
+        for tfts in tkslist:
+            tks, s = self.score_(tfts)
+            res.append((tks, s))
+        return sorted(res, key=lambda x: x[1], reverse=True)
+
+    def merge_(self, tks):
+        patts = [
+            (r"[ ]+", " "),
+            (r"([0-9\+\.,%\*=-]) ([0-9\+\.,%\*=-])", r"\1\2"),
+        ]
+        # for p,s in patts: tks = re.sub(p, s, tks)
+
+        # if split chars is part of token
+        res = []
+        tks = re.sub(r"[ ]+", " ", tks).split(" ")
+        s = 0
+        while True:
+            if s >= len(tks):
+                break
+            E = s + 1
+            for e in range(s + 2, min(len(tks) + 2, s + 6)):
+                tk = "".join(tks[s:e])
+                if re.search(self.SPLIT_CHAR, tk) and self.freq(tk):
+                    E = e
+            res.append("".join(tks[s:E]))
+            s = E
+
+        return " ".join(res)
+
+    def maxForward_(self, line):
+        res = []
+        s = 0
+        while s < len(line):
+            e = s + 1
+            t = line[s:e]
+            while e < len(line) and self.trie_.has_keys_with_prefix(
+                    self.key_(t)):
+                e += 1
+                t = line[s:e]
+
+            while e - 1 > s and self.key_(t) not in self.trie_:
+                e -= 1
+                t = line[s:e]
+
+            if self.key_(t) in self.trie_:
+                res.append((t, self.trie_[self.key_(t)]))
+            else:
+                res.append((t, (0, '')))
+
+            s = e
+
+        return self.score_(res)
+
+    def maxBackward_(self, line):
+        res = []
+        s = len(line) - 1
+        while s >= 0:
+            e = s + 1
+            t = line[s:e]
+            while s > 0 and self.trie_.has_keys_with_prefix(self.rkey_(t)):
+                s -= 1
+                t = line[s:e]
+
+            while s + 1 < e and self.key_(t) not in self.trie_:
+                s += 1
+                t = line[s:e]
+
+            if self.key_(t) in self.trie_:
+                res.append((t, self.trie_[self.key_(t)]))
+            else:
+                res.append((t, (0, '')))
+
+            s -= 1
+
+        return self.score_(res[::-1])
+
+    def qie(self, line):
+        line = self._strQ2B(line).lower()
+        line = self._tradi2simp(line)
+        arr = re.split(self.SPLIT_CHAR, line)
+        res = []
+        for L in arr:
+            if len(L) < 2 or re.match(
+                    r"[a-z\.-]+$", L) or re.match(r"[0-9\.-]+$", L):
+                res.append(L)
+                continue
+            # print(L)
+
+            # use maxforward for the first time
+            tks, s = self.maxForward_(L)
+            tks1, s1 = self.maxBackward_(L)
+            if self.DEBUG:
+                print("[FW]", tks, s)
+                print("[BW]", tks1, s1)
+
+            diff = [0 for _ in range(max(len(tks1), len(tks)))]
+            for i in range(min(len(tks1), len(tks))):
+                if tks[i] != tks1[i]:
+                    diff[i] = 1
+
+            if s1 > s:
+                tks = tks1
+
+            i = 0
+            while i < len(tks):
+                s = i
+                while s < len(tks) and diff[s] == 0:
+                    s += 1
+                if s == len(tks):
+                    res.append(" ".join(tks[i:]))
+                    break
+                if s > i:
+                    res.append(" ".join(tks[i:s]))
+
+                e = s
+                while e < len(tks) and e - s < 5 and diff[e] == 1:
+                    e += 1
+
+                tkslist = []
+                self.dfs_("".join(tks[s:e + 1]), 0, [], tkslist)
+                res.append(" ".join(self.sortTks_(tkslist)[0][0]))
+
+                i = e + 1
+
+        res = " ".join(res)
+        if self.DEBUG:
+            print("[TKS]", self.merge_(res))
+        return self.merge_(res)
+
+    def qieqie(self, tks):
+        res = []
+        for tk in tks.split(" "):
+            if len(tk) < 3 or re.match(r"[0-9,\.-]+$", tk):
+                res.append(tk)
+                continue
+            tkslist = []
+            if len(tk) > 10:
+                tkslist.append(tk)
+            else:
+                self.dfs_(tk, 0, [], tkslist)
+            if len(tkslist) < 2:
+                res.append(tk)
+                continue
+            stk = self.sortTks_(tkslist)[1][0]
+            if len(stk) == len(tk):
+                stk = tk
+            else:
+                if re.match(r"[a-z\.-]+$", tk):
+                    for t in stk:
+                        if len(t) < 3:
+                            stk = tk
+                            break
+                    else:
+                        stk = " ".join(stk)
+                else:
+                    stk = " ".join(stk)
+
+            res.append(stk)
+
+        return " ".join(res)
+
+
+def is_chinese(s):
+    if s >= u'\u4e00' and s <= u'\u9fa5':
+        return True
+    else:
+        return False
+
+
+def is_number(s):
+    if s >= u'\u0030' and s <= u'\u0039':
+        return True
+    else:
+        return False
+
+
+def is_alphabet(s):
+    if (s >= u'\u0041' and s <= u'\u005a') or (
+            s >= u'\u0061' and s <= u'\u007a'):
+        return True
+    else:
+        return False
+
+
+def naiveQie(txt):
+    tks = []
+    for t in txt.split(" "):
+        if tks and re.match(r".*[a-zA-Z]$", tks[-1]
+                            ) and re.match(r".*[a-zA-Z]$", t):
+            tks.append(" ")
+        tks.append(t)
+    return tks
+
+
+hq = Huqie()
+qie = hq.qie
+qieqie = hq.qieqie
+tag = hq.tag
+freq = hq.freq
+loadUserDict = hq.loadUserDict
+addUserDict = hq.addUserDict
+tradi2simp = hq._tradi2simp
+strQ2B = hq._strQ2B
+
+if __name__ == '__main__':
+    huqie = Huqie(debug=True)
+    # huqie.addUserDict("/tmp/tmp.new.tks.dict")
+    tks = huqie.qie(
+        "哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈哈")
+    print(huqie.qieqie(tks))
+    tks = huqie.qie(
+        "公开征求意见稿提出，境外投资者可使用自有人民币或外汇投资。使用外汇投资的，可通过债券持有人在香港人民币业务清算行及香港地区经批准可进入境内银行间外汇市场进行交易的境外人民币业务参加行（以下统称香港结算行）办理外汇资金兑换。香港结算行由此所产生的头寸可到境内银行间外汇市场平盘。使用外汇投资的，在其投资的债券到期或卖出后，原则上应兑换回外汇。")
+    print(huqie.qieqie(tks))
+    tks = huqie.qie(
+        "多校划片就是一个小区对应多个小学初中，让买了学区房的家庭也不确定到底能上哪个学校。目的是通过这种方式为学区房降温，把就近入学落到实处。南京市长江大桥")
+    print(huqie.qieqie(tks))
+    tks = huqie.qie(
+        "实际上当时他们已经将业务中心偏移到安全部门和针对政府企业的部门 Scripts are compiled and cached aaaaaaaaa")
+    print(huqie.qieqie(tks))
+    tks = huqie.qie("虽然我不怎么玩")
+    print(huqie.qieqie(tks))
+    tks = huqie.qie("蓝月亮如何在外资夹击中生存,那是全宇宙最有意思的")
+    print(huqie.qieqie(tks))
+    tks = huqie.qie(
+        "涡轮增压发动机num最大功率,不像别的共享买车锁电子化的手段,我们接过来是否有意义,黄黄爱美食,不过，今天阿奇要讲到的这家农贸市场，说实话，还真蛮有特色的！不仅环境好，还打出了")
+    print(huqie.qieqie(tks))
+    tks = huqie.qie("这周日你去吗？这周日你有空吗？")
+    print(huqie.qieqie(tks))
+    tks = huqie.qie("Unity3D开发经验 测试开发工程师 c++双11双11 985 211 ")
+    print(huqie.qieqie(tks))
+    tks = huqie.qie(
+        "数据分析项目经理|数据分析挖掘|数据分析方向|商品数据分析|搜索数据分析 sql python hive tableau Cocos2d-")
+    print(huqie.qieqie(tks))
+    if len(sys.argv) < 2:
+        sys.exit()
+    huqie.DEBUG = False
+    huqie.loadUserDict(sys.argv[1])
+    of = open(sys.argv[2], "r")
+    while True:
+        line = of.readline()
+        if not line:
+            break
+        print(huqie.qie(line))
+    of.close()

python/nlp/query.py

@@ -0,0 +1,167 @@
+import json
+import re
+import sys
+import os
+import logging
+import copy
+import math
+from elasticsearch_dsl import Q, Search
+from nlp import huqie, term_weight, synonym
+
+
+class EsQueryer:
+    def __init__(self, es):
+        self.tw = term_weight.Dealer()
+        self.es = es
+        self.syn = synonym.Dealer(None)
+        self.flds = ["ask_tks^10", "ask_small_tks"]
+
+    @staticmethod
+    def subSpecialChar(line):
+        return re.sub(r"([:\{\}/\[\]\-\*\"\(\)\|~\^])", r"\\\1", line).strip()
+
+    @staticmethod
+    def isChinese(line):
+        arr = re.split(r"[ \t]+", line)
+        if len(arr) <= 3:
+            return True
+        e = 0
+        for t in arr:
+            if not re.match(r"[a-zA-Z]+$", t):
+                e += 1
+        return e * 1. / len(arr) >= 0.8
+
+    @staticmethod
+    def rmWWW(txt):
+        txt = re.sub(
+            r"是*(什么样的|哪家|那家|啥样|咋样了|什么时候|何时|何地|何人|是否|是不是|多少|哪里|怎么|哪儿|怎么样|如何|哪些|是啥|啥是|啊|吗|呢|吧|咋|什么|有没有|呀)是*",
+            "",
+            txt)
+        return re.sub(
+            r"(what|who|how|which|where|why|(is|are|were|was) there) (is|are|were|was)*", "", txt, re.IGNORECASE)
+
+    def question(self, txt, tbl="qa", min_match="60%"):
+        txt = re.sub(
+            r"[ \t,，。？?/`!！&]+",
+            " ",
+            huqie.tradi2simp(
+                huqie.strQ2B(
+                    txt.lower()))).strip()
+        txt = EsQueryer.rmWWW(txt)
+
+        if not self.isChinese(txt):
+            tks = txt.split(" ")
+            q = []
+            for i in range(1, len(tks)):
+                q.append("\"%s %s\"~2" % (tks[i - 1], tks[i]))
+            if not q:
+                q.append(txt)
+            return Q("bool",
+                     must=Q("query_string", fields=self.flds,
+                            type="best_fields", query=" OR ".join(q),
+                            boost=1, minimum_should_match="60%")
+                     ), txt.split(" ")
+
+        def needQieqie(tk):
+            if len(tk) < 4:
+                return False
+            if re.match(r"[0-9a-z\.\+#_\*-]+$", tk):
+                return False
+            return True
+
+        qs, keywords = [], []
+        for tt in self.tw.split(txt):  # .split(" "):
+            if not tt:
+                continue
+            twts = self.tw.weights([tt])
+            syns = self.syn.lookup(tt)
+            logging.info(json.dumps(twts, ensure_ascii=False))
+            tms = []
+            for tk, w in sorted(twts, key=lambda x: x[1] * -1):
+                sm = huqie.qieqie(tk).split(" ") if needQieqie(tk) else []
+                sm = [
+                    re.sub(
+                        r"[ ,\./;'\[\]\\`~!@#$%\^&\*\(\)=\+_<>\?:\"\{\}\|，。；‘’【】、！￥……（）——《》？：“”-]+",
+                        "",
+                        m) for m in sm]
+                sm = [EsQueryer.subSpecialChar(m) for m in sm if len(m) > 1]
+                sm = [m for m in sm if len(m) > 1]
+                if len(sm) < 2:
+                    sm = []
+
+                keywords.append(re.sub(r"[ \\\"']+", "", tk))
+
+                tk_syns = self.syn.lookup(tk)
+                tk = EsQueryer.subSpecialChar(tk)
+                if tk.find(" ") > 0:
+                    tk = "\"%s\"" % tk
+                if tk_syns:
+                    tk = f"({tk} %s)" % " ".join(tk_syns)
+                if sm:
+                    tk = f"{tk} OR \"%s\" OR (\"%s\"~2)^0.5" % (
+                        " ".join(sm), " ".join(sm))
+                tms.append((tk, w))
+
+            tms = " ".join([f"({t})^{w}" for t, w in tms])
+
+            if len(twts) > 1:
+                tms += f" (\"%s\"~4)^1.5" % (" ".join([t for t, _ in twts]))
+            if re.match(r"[0-9a-z ]+$", tt):
+                tms = f"(\"{tt}\" OR \"%s\")" % huqie.qie(tt)
+
+            syns = " OR ".join(
+                ["\"%s\"^0.7" % EsQueryer.subSpecialChar(huqie.qie(s)) for s in syns])
+            if syns:
+                tms = f"({tms})^5 OR ({syns})^0.7"
+
+            qs.append(tms)
+
+        flds = copy.deepcopy(self.flds)
+        mst = []
+        if qs:
+            mst.append(
+                Q("query_string", fields=flds, type="best_fields",
+                  query=" OR ".join([f"({t})" for t in qs if t]), boost=1, minimum_should_match=min_match)
+            )
+
+        return Q("bool",
+                 must=mst,
+                 ), keywords
+
+    def hybrid_similarity(self, avec, bvecs, atks, btkss, tkweight=0.3,
+                          vtweight=0.7):
+        from sklearn.metrics.pairwise import cosine_similarity as CosineSimilarity
+        import numpy as np
+        sims = CosineSimilarity([avec], bvecs)
+
+        def toDict(tks):
+            d = {}
+            if isinstance(tks, type("")):
+                tks = tks.split(" ")
+            for t, c in self.tw.weights(tks):
+                if t not in d:
+                    d[t] = 0
+                d[t] += c
+            return d
+
+        atks = toDict(atks)
+        btkss = [toDict(tks) for tks in btkss]
+        tksim = [self.similarity(atks, btks) for btks in btkss]
+        return np.array(sims[0]) * vtweight + np.array(tksim) * tkweight
+
+    def similarity(self, qtwt, dtwt):
+        if isinstance(dtwt, type("")):
+            dtwt = {t: w for t, w in self.tw.weights(self.tw.split(dtwt))}
+        if isinstance(qtwt, type("")):
+            qtwt = {t: w for t, w in self.tw.weights(self.tw.split(qtwt))}
+        s = 1e-9
+        for k, v in qtwt.items():
+            if k in dtwt:
+                s += v * dtwt[k]
+        q = 1e-9
+        for k, v in qtwt.items():
+            q += v * v
+        d = 1e-9
+        for k, v in dtwt.items():
+            d += v * v
+        return s / math.sqrt(q) / math.sqrt(d)

python/nlp/synonym.py

@@ -0,0 +1,67 @@
+import json
+import time
+import logging
+import re
+
+
+class Dealer:
+    def __init__(self, redis=None):
+
+        self.lookup_num = 100000000
+        self.load_tm = time.time() - 1000000
+        self.dictionary = None
+        try:
+            self.dictionary = json.load(open("./synonym.json", 'r'))
+        except Exception as e:
+            pass
+        try:
+            self.dictionary = json.load(open("./res/synonym.json", 'r'))
+        except Exception as e:
+            try:
+                self.dictionary = json.load(open("../res/synonym.json", 'r'))
+            except Exception as e:
+                logging.warn("Miss synonym.json")
+                self.dictionary = {}
+
+        if not redis:
+            logging.warning(
+                "Realtime synonym is disabled, since no redis connection.")
+        if not len(self.dictionary.keys()):
+            logging.warning(f"Fail to load synonym")
+
+        self.redis = redis
+        self.load()
+
+    def load(self):
+        if not self.redis:
+            return
+
+        if self.lookup_num < 100:
+            return
+        tm = time.time()
+        if tm - self.load_tm < 3600:
+            return
+
+        self.load_tm = time.time()
+        self.lookup_num = 0
+        d = self.redis.get("kevin_synonyms")
+        if not d:
+            return
+        try:
+            d = json.loads(d)
+            self.dictionary = d
+        except Exception as e:
+            logging.error("Fail to load synonym!" + str(e))
+
+    def lookup(self, tk):
+        self.lookup_num += 1
+        self.load()
+        res = self.dictionary.get(re.sub(r"[ \t]+", " ", tk.lower()), [])
+        if isinstance(res, str):
+            res = [res]
+        return res
+
+
+if __name__ == '__main__':
+    dl = Dealer()
+    print(dl.dictionary)

python/nlp/term_weight.py

@@ -0,0 +1,216 @@
+import math
+import json
+import re
+import os
+import numpy as np
+from nlp import huqie
+
+
+class Dealer:
+    def __init__(self):
+        self.stop_words = set(["请问",
+                               "您",
+                               "你",
+                               "我",
+                               "他",
+                               "是",
+                               "的",
+                               "就",
+                               "有",
+                               "于",
+                               "及",
+                               "即",
+                               "在",
+                               "为",
+                               "最",
+                               "有",
+                               "从",
+                               "以",
+                               "了",
+                               "将",
+                               "与",
+                               "吗",
+                               "吧",
+                               "中",
+                               "#",
+                               "什么",
+                               "怎么",
+                               "哪个",
+                               "哪些",
+                               "啥",
+                               "相关"])
+
+        def load_dict(fnm):
+            res = {}
+            f = open(fnm, "r")
+            while True:
+                l = f.readline()
+                if not l:
+                    break
+                arr = l.replace("\n", "").split("\t")
+                if len(arr) < 2:
+                    res[arr[0]] = 0
+                else:
+                    res[arr[0]] = int(arr[1])
+
+            c = 0
+            for _, v in res.items():
+                c += v
+            if c == 0:
+                return set(res.keys())
+            return res
+
+        fnm = os.path.join(os.path.dirname(__file__), '../res/')
+        if not os.path.exists(fnm):
+            fnm = os.path.join(os.path.dirname(__file__), '../../res/')
+        self.ne, self.df = {}, {}
+        try:
+            self.ne = json.load(open(fnm + "ner.json", "r"))
+        except Exception as e:
+            print("[WARNING] Load ner.json FAIL!")
+        try:
+            self.df = load_dict(fnm + "term.freq")
+        except Exception as e:
+            print("[WARNING] Load term.freq FAIL!")
+
+    def pretoken(self, txt, num=False, stpwd=True):
+        patt = [
+            r"[~—\t @#%!<>,\.\?\":;'\{\}\[\]_=\(\)\|，。？》•●○↓《；‘’：“”【¥ 】…￥！、·（）×`&\\/「」\\]"
+        ]
+        rewt = [
+        ]
+        for p, r in rewt:
+            txt = re.sub(p, r, txt)
+
+        res = []
+        for t in huqie.qie(txt).split(" "):
+            tk = t
+            if (stpwd and tk in self.stop_words) or (
+                    re.match(r"[0-9]$", tk) and not num):
+                continue
+            for p in patt:
+                if re.match(p, t):
+                    tk = "#"
+                    break
+            tk = re.sub(r"([\+\\-])", r"\\\1", tk)
+            if tk != "#" and tk:
+                res.append(tk)
+        return res
+
+    def tokenMerge(self, tks):
+        def oneTerm(t): return len(t) == 1 or re.match(r"[0-9a-z]{1,2}$", t)
+
+        res, i = [], 0
+        while i < len(tks):
+            j = i
+            if i == 0 and oneTerm(tks[i]) and len(
+                    tks) > 1 and len(tks[i + 1]) > 1:  # 多 工位
+                res.append(" ".join(tks[0:2]))
+                i = 2
+                continue
+
+            while j < len(
+                    tks) and tks[j] and tks[j] not in self.stop_words and oneTerm(tks[j]):
+                j += 1
+            if j - i > 1:
+                if j - i < 5:
+                    res.append(" ".join(tks[i:j]))
+                    i = j
+                else:
+                    res.append(" ".join(tks[i:i + 2]))
+                    i = i + 2
+            else:
+                if len(tks[i]) > 0:
+                    res.append(tks[i])
+                i += 1
+        return [t for t in res if t]
+
+    def ner(self, t):
+        if not self.ne:
+            return ""
+        res = self.ne.get(t, "")
+        if res:
+            return res
+
+    def split(self, txt):
+        tks = []
+        for t in re.sub(r"[ \t]+", " ", txt).split(" "):
+            if tks and re.match(r".*[a-zA-Z]$", tks[-1]) and \
+               re.match(r".*[a-zA-Z]$", t) and tks and \
+               self.ne.get(t, "") != "func" and self.ne.get(tks[-1], "") != "func":
+                tks[-1] = tks[-1] + " " + t
+            else:
+                tks.append(t)
+        return tks
+
+    def weights(self, tks):
+        def skill(t):
+            if t not in self.sk:
+                return 1
+            return 6
+
+        def ner(t):
+            if not self.ne or t not in self.ne:
+                return 1
+            m = {"toxic": 2, "func": 1, "corp": 3, "loca": 3, "sch": 3, "stock": 3,
+                 "firstnm": 1}
+            return m[self.ne[t]]
+
+        def postag(t):
+            t = huqie.tag(t)
+            if t in set(["r", "c", "d"]):
+                return 0.3
+            if t in set(["ns", "nt"]):
+                return 3
+            if t in set(["n"]):
+                return 2
+            if re.match(r"[0-9-]+", t):
+                return 2
+            return 1
+
+        def freq(t):
+            if re.match(r"[0-9\. -]+$", t):
+                return 10000
+            s = huqie.freq(t)
+            if not s and re.match(r"[a-z\. -]+$", t):
+                return 10
+            if not s:
+                s = 0
+
+            if not s and len(t) >= 4:
+                s = [tt for tt in huqie.qieqie(t).split(" ") if len(tt) > 1]
+                if len(s) > 1:
+                    s = np.min([freq(tt) for tt in s]) / 6.
+                else:
+                    s = 0
+
+            return max(s, 10)
+
+        def df(t):
+            if re.match(r"[0-9\. -]+$", t):
+                return 100000
+            if t in self.df:
+                return self.df[t] + 3
+            elif re.match(r"[a-z\. -]+$", t):
+                return 3
+            elif len(t) >= 4:
+                s = [tt for tt in huqie.qieqie(t).split(" ") if len(tt) > 1]
+                if len(s) > 1:
+                    return max(3, np.min([df(tt) for tt in s]) / 6.)
+
+            return 3
+
+        def idf(s, N): return math.log10(10 + ((N - s + 0.5) / (s + 0.5)))
+
+        tw = []
+        for tk in tks:
+            tt = self.tokenMerge(self.pretoken(tk, True))
+            idf1 = np.array([idf(freq(t), 10000000) for t in tt])
+            idf2 = np.array([idf(df(t), 1000000000) for t in tt])
+            wts = (0.3 * idf1 + 0.7 * idf2) * \
+                np.array([ner(t) * postag(t) for t in tt])
+
+            tw.extend(zip(tt, wts))
+
+        S = np.sum([s for _, s in tw])
+        return [(t, s / S) for t, s in tw]

python/parser/__init__.py

@@ -0,0 +1,3 @@
+from .pdf_parser import HuParser as PdfParser
+from .docx_parser import HuDocxParser as DocxParser
+from .excel_parser import HuExcelParser as ExcelParser

python/parser/docx_parser.py

@@ -0,0 +1,103 @@
+from docx import Document
+import re
+import pandas as pd
+from collections import Counter
+from nlp import huqie
+
+
+class HuDocxParser:
+
+    def __extract_table_content(self, tb):
+        df = []
+        for row in tb.rows:
+            df.append([c.text for c in row.cells])
+        return self.__compose_table_content(pd.DataFrame(df))
+
+    def __compose_table_content(self, df):
+
+        def blockType(b):
+            patt = [
+                ("^(20|19)[0-9]{2}[年/-][0-9]{1,2}[月/-][0-9]{1,2}日*$", "Dt"),
+                (r"^(20|19)[0-9]{2}年$", "Dt"),
+                (r"^(20|19)[0-9]{2}[年/-][0-9]{1,2}月*$", "Dt"),
+                ("^[0-9]{1,2}[月/-][0-9]{1,2}日*$", "Dt"),
+                (r"^第*[一二三四1-4]季度$", "Dt"),
+                (r"^(20|19)[0-9]{2}年*[一二三四1-4]季度$", "Dt"),
+                (r"^(20|19)[0-9]{2}[ABCDE]$", "DT"),
+                ("^[0-9.,+%/ -]+$", "Nu"),
+                (r"^[0-9A-Z/\._~-]+$", "Ca"),
+                (r"^[A-Z]*[a-z' -]+$", "En"),
+                (r"^[0-9.,+-]+[0-9A-Za-z/$￥%<>（）()' -]+$", "NE"),
+                (r"^.{1}$", "Sg")
+            ]
+            for p, n in patt:
+                if re.search(p, b):
+                    return n
+            tks = [t for t in huqie.qie(b).split(" ") if len(t) > 1]
+            if len(tks) > 3:
+                if len(tks) < 12:
+                    return "Tx"
+                else:
+                    return "Lx"
+
+            if len(tks) == 1 and huqie.tag(tks[0]) == "nr":
+                return "Nr"
+
+            return "Ot"
+
+        if len(df) < 2:
+            return []
+        max_type = Counter([blockType(str(df.iloc[i, j])) for i in range(
+            1, len(df)) for j in range(len(df.iloc[i, :]))])
+        max_type = max(max_type.items(), key=lambda x: x[1])[0]
+
+        colnm = len(df.iloc[0, :])
+        hdrows = [0]  # header is not nessesarily appear in the first line
+        if max_type == "Nu":
+            for r in range(1, len(df)):
+                tys = Counter([blockType(str(df.iloc[r, j]))
+                              for j in range(len(df.iloc[r, :]))])
+                tys = max(tys.items(), key=lambda x: x[1])[0]
+                if tys != max_type:
+                    hdrows.append(r)
+
+        lines = []
+        for i in range(1, len(df)):
+            if i in hdrows:
+                continue
+            hr = [r - i for r in hdrows]
+            hr = [r for r in hr if r < 0]
+            t = len(hr) - 1
+            while t > 0:
+                if hr[t] - hr[t - 1] > 1:
+                    hr = hr[t:]
+                    break
+                t -= 1
+            headers = []
+            for j in range(len(df.iloc[i, :])):
+                t = []
+                for h in hr:
+                    x = str(df.iloc[i + h, j]).strip()
+                    if x in t:
+                        continue
+                    t.append(x)
+                t = ",".join(t)
+                if t:
+                    t += ": "
+                headers.append(t)
+            cells = []
+            for j in range(len(df.iloc[i, :])):
+                if not str(df.iloc[i, j]):
+                    continue
+                cells.append(headers[j] + str(df.iloc[i, j]))
+            lines.append(";".join(cells))
+
+        if colnm > 3:
+            return lines
+        return ["\n".join(lines)]
+
+    def __call__(self, fnm):
+        self.doc = Document(fnm)
+        secs = [(p.text, p.style.name) for p in self.doc.paragraphs]
+        tbls = [self.__extract_table_content(tb) for tb in self.doc.tables]
+        return secs, tbls

python/parser/excel_parser.py

@@ -0,0 +1,21 @@
+from openpyxl import load_workbook
+import sys
+
+
+class HuExcelParser:
+    def __call__(self, fnm):
+        wb = load_workbook(fnm)
+        res = []
+        for sheetname in wb.sheetnames:
+            ws = wb[sheetname]
+            lines = []
+            for r in ws.rows:
+                lines.append(
+                    "\t".join([str(c.value) if c.value is not None else "" for c in r]))
+            res.append(f"《{sheetname}》\n" + "\n".join(lines))
+        return res
+
+
+if __name__ == "__main__":
+    psr = HuExcelParser()
+    psr(sys.argv[1])

python/parser/pdf_parser.py

@@ -0,0 +1,1635 @@
+import xgboost as xgb
+import torch
+import re
+import pdfplumber
+import logging
+from PIL import Image
+import numpy as np
+from nlp import huqie
+from collections import Counter
+from copy import deepcopy
+from cv.table_recognize import TableTransformer
+from cv.ppdetection import PPDet
+from huggingface_hub import hf_hub_download
+logging.getLogger("pdfminer").setLevel(logging.WARNING)
+
+
+class HuParser:
+    def __init__(self):
+        from paddleocr import PaddleOCR
+        logging.getLogger("ppocr").setLevel(logging.ERROR)
+        self.ocr = PaddleOCR(use_angle_cls=False, lang="ch")
+        self.layouter = PPDet()
+        self.tbl_det = TableTransformer()
+
+        self.updown_cnt_mdl = xgb.Booster()
+        if torch.cuda.is_available():
+            self.updown_cnt_mdl.set_param({"device": "cuda"})
+        self.updown_cnt_mdl.load_model(hf_hub_download(repo_id="InfiniFlow/text_concat_xgb_v1.0",
+                                                       filename="updown_concat_xgb.model"))
+        """
+        If you have trouble downloading HuggingFace models, -_^ this might help!!
+
+        For Linux:
+        export HF_ENDPOINT=https://hf-mirror.com
+
+        For Windows:
+        Good luck
+        ^_-
+
+        """
+
+    def __char_width(self, c):
+        return (c["x1"] - c["x0"]) // len(c["text"])
+
+    def __height(self, c):
+        return c["bottom"] - c["top"]
+
+    def _x_dis(self, a, b):
+        return min(abs(a["x1"] - b["x0"]), abs(a["x0"] - b["x1"]),
+                   abs(a["x0"] + a["x1"] - b["x0"] - b["x1"]) / 2)
+
+    def _y_dis(
+            self, a, b):
+        return (
+                       b["top"] + b["bottom"] - a["top"] - a["bottom"]) / 2
+
+    def _match_proj(self, b):
+        proj_patt = [
+            r"第[零一二三四五六七八九十百]+章",
+            r"第[零一二三四五六七八九十百]+[条节]",
+            r"[零一二三四五六七八九十百]+[、是 　]",
+            r"[\(（][零一二三四五六七八九十百]+[）\)]",
+            r"[\(（][0-9]+[）\)]",
+            r"[0-9]+(、|\.[　 ]|）|\.[^0-9./a-zA-Z_%><-]{4,})",
+            r"[0-9]+\.[0-9.]+(、|\.[ 　])",
+            r"[⚫•➢①② ]",
+        ]
+        return any([re.match(p, b["text"]) for p in proj_patt])
+
+    def _updown_concat_features(self, up, down):
+        w = max(self.__char_width(up), self.__char_width(down))
+        h = max(self.__height(up), self.__height(down))
+        y_dis = self._y_dis(up, down)
+        LEN = 6
+        tks_down = huqie.qie(down["text"][:LEN]).split(" ")
+        tks_up = huqie.qie(up["text"][-LEN:]).split(" ")
+        tks_all = up["text"][-LEN:].strip() \
+                  + (" " if re.match(r"[a-zA-Z0-9]+",
+                                     up["text"][-1] + down["text"][0]) else "") \
+                  + down["text"][:LEN].strip()
+        tks_all = huqie.qie(tks_all).split(" ")
+        fea = [
+            up.get("R", -1) == down.get("R", -1),
+            y_dis / h,
+            down["page_number"] - up["page_number"],
+            up["layout_type"] == down["layout_type"],
+            up["layout_type"] == "text",
+            down["layout_type"] == "text",
+            up["layout_type"] == "table",
+            down["layout_type"] == "table",
+            True if re.search(
+                r"([。？！；!?;+)）]|[a-z]\.)$",
+                up["text"]) else False,
+            True if re.search(r"[，：‘“、0-9（+-]$", up["text"]) else False,
+            True if re.search(
+                r"(^.?[/,?;:\]，。；：’”？！》】）-])",
+                down["text"]) else False,
+            True if re.match(r"[\(（][^\(\)（）]+[）\)]$", up["text"]) else False,
+            True if re.search(r"[，,][^。.]+$", up["text"]) else False,
+            True if re.search(r"[，,][^。.]+$", up["text"]) else False,
+            True if re.search(r"[\(（][^\)）]+$", up["text"])
+                    and re.search(r"[\)）]", down["text"]) else False,
+            self._match_proj(down),
+            True if re.match(r"[A-Z]", down["text"]) else False,
+            True if re.match(r"[A-Z]", up["text"][-1]) else False,
+            True if re.match(r"[a-z0-9]", up["text"][-1]) else False,
+            True if re.match(r"[0-9.%,-]+$", down["text"]) else False,
+            up["text"].strip()[-2:] == down["text"].strip()[-2:] if len(up["text"].strip()
+                                                                        ) > 1 and len(
+                down["text"].strip()) > 1 else False,
+            up["x0"] > down["x1"],
+            abs(self.__height(up) - self.__height(down)) / min(self.__height(up),
+                                                               self.__height(down)),
+            self._x_dis(up, down) / max(w, 0.000001),
+            (len(up["text"]) - len(down["text"])) /
+            max(len(up["text"]), len(down["text"])),
+            len(tks_all) - len(tks_up) - len(tks_down),
+            len(tks_down) - len(tks_up),
+            tks_down[-1] == tks_up[-1],
+            max(down["in_row"], up["in_row"]),
+            abs(down["in_row"] - up["in_row"]),
+            len(tks_down) == 1 and huqie.tag(tks_down[0]).find("n") >= 0,
+            len(tks_up) == 1 and huqie.tag(tks_up[0]).find("n") >= 0
+        ]
+        return fea
+
+    @staticmethod
+    def sort_Y_firstly(arr, threashold):
+        # sort using y1 first and then x1
+        arr = sorted(arr, key=lambda r: (r["top"], r["x0"]))
+        for i in range(len(arr) - 1):
+            for j in range(i, -1, -1):
+                # restore the order using th
+                if abs(arr[j + 1]["top"] - arr[j]["top"]) < threashold \
+                        and arr[j + 1]["x0"] < arr[j]["x0"]:
+                    tmp = deepcopy(arr[j])
+                    arr[j] = deepcopy(arr[j + 1])
+                    arr[j + 1] = deepcopy(tmp)
+        return arr
+
+    @staticmethod
+    def sort_R_firstly(arr, thr=0):
+        # sort using y1 first and then x1
+        # sorted(arr, key=lambda r: (r["top"], r["x0"]))
+        arr = HuParser.sort_Y_firstly(arr, thr)
+        for i in range(len(arr) - 1):
+            for j in range(i, -1, -1):
+                if "R" not in arr[j] or "R" not in arr[j + 1]:
+                    continue
+                if arr[j + 1]["R"] < arr[j]["R"] \
+                        or (
+                        arr[j + 1]["R"] == arr[j]["R"]
+                        and arr[j + 1]["x0"] < arr[j]["x0"]
+                ):
+                    tmp = arr[j]
+                    arr[j] = arr[j + 1]
+                    arr[j + 1] = tmp
+        return arr
+
+    @staticmethod
+    def sort_X_firstly(arr, threashold, copy=True):
+        # sort using y1 first and then x1
+        arr = sorted(arr, key=lambda r: (r["x0"], r["top"]))
+        for i in range(len(arr) - 1):
+            for j in range(i, -1, -1):
+                # restore the order using th
+                if abs(arr[j + 1]["x0"] - arr[j]["x0"]) < threashold \
+                        and arr[j + 1]["top"] < arr[j]["top"]:
+                    tmp = deepcopy(arr[j]) if copy else arr[j]
+                    arr[j] = deepcopy(arr[j + 1]) if copy else arr[j + 1]
+                    arr[j + 1] = deepcopy(tmp) if copy else tmp
+        return arr
+
+    @staticmethod
+    def sort_C_firstly(arr, thr=0):
+        # sort using y1 first and then x1
+        # sorted(arr, key=lambda r: (r["x0"], r["top"]))
+        arr = HuParser.sort_X_firstly(arr, thr)
+        for i in range(len(arr) - 1):
+            for j in range(i, -1, -1):
+                # restore the order using th
+                if "C" not in arr[j] or "C" not in arr[j + 1]:
+                    continue
+                if arr[j + 1]["C"] < arr[j]["C"] \
+                        or (
+                        arr[j + 1]["C"] == arr[j]["C"]
+                        and arr[j + 1]["top"] < arr[j]["top"]
+                ):
+                    tmp = arr[j]
+                    arr[j] = arr[j + 1]
+                    arr[j + 1] = tmp
+        return arr
+
+        return sorted(arr, key=lambda r: (r.get("C", r["x0"]), r["top"]))
+
+    def _has_color(self, o):
+        if o.get("ncs", "") == "DeviceGray":
+            if o["stroking_color"] and o["stroking_color"][0] == 1 and o["non_stroking_color"] and \
+                    o["non_stroking_color"][0] == 1:
+                if re.match(r"[a-zT_\[\]\(\)-]+", o.get("text", "")):
+                    return False
+        return True
+
+    def __overlapped_area(self, a, b, ratio=True):
+        tp, btm, x0, x1 = a["top"], a["bottom"], a["x0"], a["x1"]
+        if b["x0"] > x1 or b["x1"] < x0:
+            return 0
+        if b["bottom"] < tp or b["top"] > btm:
+            return 0
+        x0_ = max(b["x0"], x0)
+        x1_ = min(b["x1"], x1)
+        assert x0_ <= x1_, "Fuckedup! T:{},B:{},X0:{},X1:{} ==> {}".format(
+            tp, btm, x0, x1, b)
+        tp_ = max(b["top"], tp)
+        btm_ = min(b["bottom"], btm)
+        assert tp_ <= btm_, "Fuckedup! T:{},B:{},X0:{},X1:{} => {}".format(
+            tp, btm, x0, x1, b)
+        ov = (btm_ - tp_) * (x1_ - x0_) if x1 - \
+                                           x0 != 0 and btm - tp != 0 else 0
+        if ov > 0 and ratio:
+            ov /= (x1 - x0) * (btm - tp)
+        return ov
+
+    def __find_overlapped_with_threashold(self, box, boxes, thr=0.3):
+        if not boxes:
+            return
+        max_overlaped_i, max_overlaped, _max_overlaped = None, thr, 0
+        s, e = 0, len(boxes)
+        for i in range(s, e):
+            ov = self.__overlapped_area(box, boxes[i])
+            _ov = self.__overlapped_area(boxes[i], box)
+            if (ov, _ov) < (max_overlaped, _max_overlaped):
+                continue
+            max_overlaped_i = i
+            max_overlaped = ov
+            _max_overlaped = _ov
+
+        return max_overlaped_i
+
+    def __find_overlapped(self, box, boxes_sorted_by_y, naive=False):
+        if not boxes_sorted_by_y:
+            return
+        bxs = boxes_sorted_by_y
+        s, e, ii = 0, len(bxs), 0
+        while s < e and not naive:
+            ii = (e + s) // 2
+            pv = bxs[ii]
+            if box["bottom"] < pv["top"]:
+                e = ii
+                continue
+            if box["top"] > pv["bottom"]:
+                s = ii + 1
+                continue
+            break
+        while s < ii:
+            if box["top"] > bxs[s]["bottom"]:
+                s += 1
+            break
+        while e - 1 > ii:
+            if box["bottom"] < bxs[e - 1]["top"]:
+                e -= 1
+            break
+
+        max_overlaped_i, max_overlaped = None, 0
+        for i in range(s, e):
+            ov = self.__overlapped_area(bxs[i], box)
+            if ov <= max_overlaped:
+                continue
+            max_overlaped_i = i
+            max_overlaped = ov
+
+        return max_overlaped_i
+
+    def _is_garbage(self, b):
+        patt = [r"^•+$", r"(版权归©|免责条款|地址[:：])", r"\.{3,}", "^[0-9]{1,2} / ?[0-9]{1,2}$",
+                r"^[0-9]{1,2} of [0-9]{1,2}$", "^http://[^ ]{12,}",
+                "(资料|数据)来源[:：]", "[0-9a-z._-]+@[a-z0-9-]+\\.[a-z]{2,3}",
+                "\\(cid *: *[0-9]+ *\\)"
+                ]
+        return any([re.search(p, b["text"]) for p in patt])
+
+    def __layouts_cleanup(self, boxes, layouts, far=2, thr=0.7):
+        def notOverlapped(a, b):
+            return any([a["x1"] < b["x0"],
+                        a["x0"] > b["x1"],
+                        a["bottom"] < b["top"],
+                        a["top"] > b["bottom"]])
+
+        i = 0
+        while i + 1 < len(layouts):
+            j = i + 1
+            while j < min(i + far, len(layouts)) \
+                    and (layouts[i].get("type", "") != layouts[j].get("type", "")
+                         or notOverlapped(layouts[i], layouts[j])):
+                j += 1
+            if j >= min(i + far, len(layouts)):
+                i += 1
+                continue
+            if self.__overlapped_area(layouts[i], layouts[j]) < thr \
+                    and self.__overlapped_area(layouts[j], layouts[i]) < thr:
+                i += 1
+                continue
+
+            if layouts[i].get("score") and layouts[j].get("score"):
+                if layouts[i]["score"] > layouts[j]["score"]:
+                    layouts.pop(j)
+                else:
+                    layouts.pop(i)
+                continue
+
+            area_i, area_i_1 = 0, 0
+            for b in boxes:
+                if not notOverlapped(b, layouts[i]):
+                    area_i += self.__overlapped_area(b, layouts[i], False)
+                if not notOverlapped(b, layouts[j]):
+                    area_i_1 += self.__overlapped_area(b, layouts[j], False)
+
+            if area_i > area_i_1:
+                layouts.pop(j)
+            else:
+                layouts.pop(i)
+
+        return layouts
+
+    def __table_paddle(self, images):
+        tbls = self.tbl_det([np.array(img) for img in images], thr=0.5)
+        res = []
+        # align left&right for rows, align top&bottom for columns
+        for tbl in tbls:
+            lts = [{"label": b["type"],
+                    "score": b["score"],
+                    "x0": b["bbox"][0], "x1": b["bbox"][2],
+                    "top": b["bbox"][1], "bottom": b["bbox"][-1]
+                    } for b in tbl]
+            if not lts:
+                continue
+
+            left = [b["x0"] for b in lts if b["label"].find(
+                "row") > 0 or b["label"].find("header") > 0]
+            right = [b["x1"] for b in lts if b["label"].find(
+                "row") > 0 or b["label"].find("header") > 0]
+            if not left:
+                continue
+            left = np.median(left) if len(left) > 4 else np.min(left)
+            right = np.median(right) if len(right) > 4 else np.max(right)
+            for b in lts:
+                if b["label"].find("row") > 0 or b["label"].find("header") > 0:
+                    if b["x0"] > left:
+                        b["x0"] = left
+                    if b["x1"] < right:
+                        b["x1"] = right
+
+            top = [b["top"] for b in lts if b["label"] == "table column"]
+            bottom = [b["bottom"] for b in lts if b["label"] == "table column"]
+            if not top:
+                res.append(lts)
+                continue
+            top = np.median(top) if len(top) > 4 else np.min(top)
+            bottom = np.median(bottom) if len(bottom) > 4 else np.max(bottom)
+            for b in lts:
+                if b["label"] == "table column":
+                    if b["top"] > top:
+                        b["top"] = top
+                    if b["bottom"] < bottom:
+                        b["bottom"] = bottom
+
+            res.append(lts)
+        return res
+
+    def __table_transformer_job(self, ZM):
+        logging.info("Table processing...")
+        imgs, pos = [], []
+        tbcnt = [0]
+        MARGIN = 10
+        self.tb_cpns = []
+        assert len(self.page_layout) == len(self.page_images)
+        for p, tbls in enumerate(self.page_layout):  # for page
+            tbls = [f for f in tbls if f["type"] == "table"]
+            tbcnt.append(len(tbls))
+            if not tbls:
+                continue
+            for tb in tbls:  # for table
+                left, top, right, bott = tb["x0"] - MARGIN, tb["top"] - MARGIN, \
+                                         tb["x1"] + MARGIN, tb["bottom"] + MARGIN
+                left *= ZM
+                top *= ZM
+                right *= ZM
+                bott *= ZM
+                pos.append((left, top))
+                imgs.append(self.page_images[p].crop((left, top, right, bott)))
+
+        assert len(self.page_images) == len(tbcnt) - 1
+        if not imgs:
+            return
+        recos = self.__table_paddle(imgs)
+        tbcnt = np.cumsum(tbcnt)
+        for i in range(len(tbcnt) - 1):  # for page
+            pg = []
+            for j, tb_items in enumerate(
+                    recos[tbcnt[i]: tbcnt[i + 1]]):  # for table
+                poss = pos[tbcnt[i]: tbcnt[i + 1]]
+                for it in tb_items:  # for table components
+                    it["x0"] = (it["x0"] + poss[j][0])
+                    it["x1"] = (it["x1"] + poss[j][0])
+                    it["top"] = (it["top"] + poss[j][1])
+                    it["bottom"] = (it["bottom"] + poss[j][1])
+                    for n in ["x0", "x1", "top", "bottom"]:
+                        it[n] /= ZM
+                    it["top"] += self.page_cum_height[i]
+                    it["bottom"] += self.page_cum_height[i]
+                    it["pn"] = i
+                    it["layoutno"] = j
+                    pg.append(it)
+            self.tb_cpns.extend(pg)
+
+    def __ocr_paddle(self, pagenum, img, chars, ZM=3):
+        bxs = self.ocr.ocr(np.array(img), cls=True)[0]
+        if not bxs:
+            self.boxes.append([])
+            return
+        bxs = [(line[0], line[1][0]) for line in bxs]
+        bxs = self.sort_Y_firstly(
+            [{"x0": b[0][0] / ZM, "x1": b[1][0] / ZM,
+              "top": b[0][1] / ZM, "text": "", "txt": t,
+              "bottom": b[-1][1] / ZM,
+              "page_number": pagenum} for b, t in bxs if b[0][0] <= b[1][0] and b[0][1] <= b[-1][1]],
+            self.mean_height[-1] / 3
+        )
+
+        # merge chars in the same rect
+        for c in self.sort_X_firstly(chars, self.mean_width[pagenum - 1] // 4):
+            ii = self.__find_overlapped(c, bxs)
+            if ii is None:
+                self.lefted_chars.append(c)
+                continue
+            ch = c["bottom"] - c["top"]
+            bh = bxs[ii]["bottom"] - bxs[ii]["top"]
+            if abs(ch - bh) / max(ch, bh) >= 0.7:
+                self.lefted_chars.append(c)
+                continue
+            bxs[ii]["text"] += c["text"]
+
+        for b in bxs:
+            if not b["text"]:
+                b["text"] = b["txt"]
+            del b["txt"]
+        if self.mean_height[-1] == 0:
+            self.mean_height[-1] = np.median([b["bottom"] - b["top"]
+                                              for b in bxs])
+
+        self.boxes.append(bxs)
+
+    def __layouts_paddle(self, ZM):
+        assert len(self.page_images) == len(self.boxes)
+        # Tag layout type
+        boxes = []
+        layouts = self.layouter([np.array(img) for img in self.page_images])
+        assert len(self.page_images) == len(layouts)
+        for pn, lts in enumerate(layouts):
+            bxs = self.boxes[pn]
+            lts = [{"type": b["type"],
+                    "score": float(b["score"]),
+                    "x0": b["bbox"][0] / ZM, "x1": b["bbox"][2] / ZM,
+                    "top": b["bbox"][1] / ZM, "bottom": b["bbox"][-1] / ZM,
+                    "page_number": pn,
+                    } for b in lts]
+            lts = self.sort_Y_firstly(lts, self.mean_height[pn] / 2)
+            lts = self.__layouts_cleanup(bxs, lts)
+            self.page_layout.append(lts)
+
+            # Tag layout type, layouts are ready
+            def findLayout(ty):
+                nonlocal bxs, lts
+                lts_ = [lt for lt in lts if lt["type"] == ty]
+                i = 0
+                while i < len(bxs):
+                    if bxs[i].get("layout_type"):
+                        i += 1
+                        continue
+                    if self._is_garbage(bxs[i]):
+                        logging.debug("GARBAGE: " + bxs[i]["text"])
+                        bxs.pop(i)
+                        continue
+
+                    ii = self.__find_overlapped_with_threashold(bxs[i], lts_,
+                                                                thr=0.4)
+                    if ii is None:  # belong to nothing
+                        bxs[i]["layout_type"] = ""
+                        i += 1
+                        continue
+                    lts_[ii]["visited"] = True
+                    if lts_[ii]["type"] in ["footer", "header", "reference"]:
+                        if lts_[ii]["type"] not in self.garbages:
+                            self.garbages[lts_[ii]["type"]] = []
+                        self.garbages[lts_[ii]["type"]].append(bxs[i]["text"])
+                        logging.debug("GARBAGE: " + bxs[i]["text"])
+                        bxs.pop(i)
+                        continue
+
+                    bxs[i]["layoutno"] = f"{ty}-{ii}"
+                    bxs[i]["layout_type"] = lts_[ii]["type"]
+                    i += 1
+
+            for lt in ["footer", "header", "reference", "figure caption",
+                       "table caption", "title", "text", "table", "figure"]:
+                findLayout(lt)
+
+            # add box to figure layouts which has not text box
+            for i, lt in enumerate(
+                    [lt for lt in lts if lt["type"] == "figure"]):
+                if lt.get("visited"):
+                    continue
+                lt = deepcopy(lt)
+                del lt["type"]
+                lt["text"] = ""
+                lt["layout_type"] = "figure"
+                lt["layoutno"] = f"figure-{i}"
+                bxs.append(lt)
+
+            boxes.extend(bxs)
+
+        self.boxes = boxes
+
+    def __text_merge(self, garbage):
+        # merge adjusted boxes
+        bxs = self.boxes
+
+        def end_with(b, txt):
+            txt = txt.strip()
+            tt = b.get("text", "").strip()
+            return tt and tt.find(txt) == len(tt) - len(txt)
+
+        def start_with(b, txts):
+            tt = b.get("text", "").strip()
+            return tt and any([tt.find(t.strip()) == 0 for t in txts])
+
+        i = 0
+        while i < len(bxs) - 1:
+            b = bxs[i]
+            b_ = bxs[i + 1]
+            if b.get("layoutno", "0") != b_.get("layoutno", "1"):
+                i += 1
+                continue
+
+            dis_thr = 1
+            dis = b["x1"] - b_["x0"]
+            if b.get("layout_type", "") != "text" or b_.get(
+                    "layout_type", "") != "text":
+                if end_with(b, "，") or start_with(b_, "（，"):
+                    dis_thr = -8
+                else:
+                    i += 1
+                    continue
+
+            if abs(self._y_dis(b, b_)) < self.mean_height[bxs[i]["page_number"] - 1] / 5 \
+                    and dis >= dis_thr and b["x1"] < b_["x1"]:
+                # merge
+                bxs[i]["x1"] = b_["x1"]
+                bxs[i]["top"] = (b["top"] + b_["top"]) / 2
+                bxs[i]["bottom"] = (b["bottom"] + b_["bottom"]) / 2
+                bxs[i]["text"] += b_["text"]
+                bxs.pop(i + 1)
+                continue
+            i += 1
+        self.boxes = bxs
+
+        # count boxes in the same row
+        for i in range(len(self.boxes)):
+            mh = self.mean_height[self.boxes[i]["page_number"] - 1]
+            self.boxes[i]["in_row"] = 0
+            j = max(0, i - 12)
+            while j < min(i + 12, len(self.boxes)):
+                if j == i:
+                    j += 1
+                    continue
+                ydis = self._y_dis(self.boxes[i], self.boxes[j]) / mh
+                if abs(ydis) < 1:
+                    self.boxes[i]["in_row"] += 1
+                elif ydis > 0:
+                    break
+                j += 1
+
+        def gather(kwd, fzy=10, ption=0.6):
+            eles = self.sort_Y_firstly(
+                [r for r in self.tb_cpns if re.match(kwd, r["label"])], fzy)
+            eles = self.__layouts_cleanup(self.boxes, eles, 5, ption)
+            return self.sort_Y_firstly(eles, 0)
+
+        headers = gather(r".*header$")
+        rows = gather(r".* (row|header)")
+        spans = gather(r".*spanning")
+        clmns = sorted([r for r in self.tb_cpns if re.match(
+            r"table column$", r["label"])], key=lambda x: (x["pn"], x["layoutno"], x["x0"]))
+        clmns = self.__layouts_cleanup(self.boxes, clmns, 5, 0.5)
+        for b in self.boxes:
+            if b.get("layout_type", "") != "table":
+                continue
+            ii = self.__find_overlapped_with_threashold(b, rows, thr=0.3)
+            if ii is not None:
+                b["R"] = ii
+                b["R_top"] = rows[ii]["top"]
+                b["R_bott"] = rows[ii]["bottom"]
+
+            ii = self.__find_overlapped_with_threashold(b, headers, thr=0.3)
+            if ii is not None:
+                b["H_top"] = headers[ii]["top"]
+                b["H_bott"] = headers[ii]["bottom"]
+                b["H_left"] = headers[ii]["x0"]
+                b["H_right"] = headers[ii]["x1"]
+                b["H"] = ii
+
+            ii = self.__find_overlapped_with_threashold(b, clmns, thr=0.3)
+            if ii is not None:
+                b["C"] = ii
+                b["C_left"] = clmns[ii]["x0"]
+                b["C_right"] = clmns[ii]["x1"]
+
+            ii = self.__find_overlapped_with_threashold(b, spans, thr=0.3)
+            if ii is not None:
+                b["H_top"] = spans[ii]["top"]
+                b["H_bott"] = spans[ii]["bottom"]
+                b["H_left"] = spans[ii]["x0"]
+                b["H_right"] = spans[ii]["x1"]
+                b["SP"] = ii
+
+        # concat between rows
+        boxes = deepcopy(self.boxes)
+        blocks = []
+        while boxes:
+            chunks = []
+
+            def dfs(up, dp):
+                if not up["text"].strip() or up["text"].strip() in garbage:
+                    return
+                chunks.append(up)
+                i = dp
+                while i < min(dp + 12, len(boxes)):
+                    ydis = self._y_dis(up, boxes[i])
+                    smpg = up["page_number"] == boxes[i]["page_number"]
+                    mh = self.mean_height[up["page_number"] - 1]
+                    mw = self.mean_width[up["page_number"] - 1]
+                    if smpg and ydis > mh * 4:
+                        break
+                    if not smpg and ydis > mh * 16:
+                        break
+                    down = boxes[i]
+
+                    if up.get("R", "") != down.get(
+                            "R", "") and up["text"][-1] != "，":
+                        i += 1
+                        continue
+
+                    if re.match(r"[0-9]{2,3}/[0-9]{3}$", up["text"]) \
+                            or re.match(r"[0-9]{2,3}/[0-9]{3}$", down["text"]):
+                        i += 1
+                        continue
+
+                    if not down["text"].strip() \
+                            or down["text"].strip() in garbage:
+                        i += 1
+                        continue
+
+                    if up["x1"] < down["x0"] - 10 * \
+                            mw or up["x0"] > down["x1"] + 10 * mw:
+                        i += 1
+                        continue
+
+                    if i - dp < 5 and up.get("layout_type") == "text":
+                        if up.get("layoutno", "1") == down.get(
+                                "layoutno", "2"):
+                            dfs(down, i + 1)
+                            boxes.pop(i)
+                            return
+                        i += 1
+                        continue
+
+                    fea = self._updown_concat_features(up, down)
+                    if self.updown_cnt_mdl.predict(
+                            xgb.DMatrix([fea]))[0] <= 0.5:
+                        i += 1
+                        continue
+                    dfs(down, i + 1)
+                    boxes.pop(i)
+                    return
+
+            dfs(boxes[0], 1)
+            boxes.pop(0)
+            if chunks:
+                blocks.append(chunks)
+
+        # concat within each block
+        boxes = []
+        for b in blocks:
+            if len(b) == 1:
+                boxes.append(b[0])
+                continue
+            t = b[0]
+            for c in b[1:]:
+                t["text"] = t["text"].strip()
+                c["text"] = c["text"].strip()
+                if not c["text"]:
+                    continue
+                if t["text"] and re.match(
+                        r"[0-9\.a-zA-Z]+$", t["text"][-1] + c["text"][-1]):
+                    t["text"] += " "
+                t["text"] += c["text"]
+                t["x0"] = min(t["x0"], c["x0"])
+                t["x1"] = max(t["x1"], c["x1"])
+                t["page_number"] = min(t["page_number"], c["page_number"])
+                t["bottom"] = c["bottom"]
+                if not t["layout_type"] \
+                        and c["layout_type"]:
+                    t["layout_type"] = c["layout_type"]
+            boxes.append(t)
+
+        self.boxes = self.sort_Y_firstly(boxes, 0)
+
+    def __filter_forpages(self):
+        if not self.boxes:
+            return
+        to = min(7, len(self.page_images) // 5)
+        pg_hits = [0 for _ in range(to)]
+
+        def possible(c):
+            if c.get("layout_type", "") == "reference":
+                return True
+            if c["bottom"] - c["top"] >= 2 * \
+                    self.mean_height[c["page_number"] - 1]:
+                return False
+            if c["text"].find("....") >= 0 \
+                    or (c["x1"] - c["x0"] > 250 and re.search(r"[0-9]+$",
+                                                              c["text"].strip())):
+                return True
+            return self.is_caption(c) and re.search(
+                r"[0-9]+$", c["text"].strip())
+
+        for c in self.boxes:
+            if c["page_number"] >= to:
+                break
+            if possible(c):
+                pg_hits[c["page_number"] - 1] += 1
+
+        st, ed = -1, -1
+        for i in range(len(self.boxes)):
+            c = self.boxes[i]
+            if c["page_number"] >= to:
+                break
+            if pg_hits[c["page_number"] - 1] >= 3 and possible(c):
+                if st < 0:
+                    st = i
+                else:
+                    ed = i
+        for _ in range(st, ed + 1):
+            self.boxes.pop(st)
+
+    def _blockType(self, b):
+        patt = [
+            ("^(20|19)[0-9]{2}[年/-][0-9]{1,2}[月/-][0-9]{1,2}日*$", "Dt"),
+            (r"^(20|19)[0-9]{2}年$", "Dt"),
+            (r"^(20|19)[0-9]{2}[年-][0-9]{1,2}月*$", "Dt"),
+            ("^[0-9]{1,2}[月-][0-9]{1,2}日*$", "Dt"),
+            (r"^第*[一二三四1-4]季度$", "Dt"),
+            (r"^(20|19)[0-9]{2}年*[一二三四1-4]季度$", "Dt"),
+            (r"^(20|19)[0-9]{2}[ABCDE]$", "Dt"),
+            ("^[0-9.,+%/ -]+$", "Nu"),
+            (r"^[0-9A-Z/\._~-]+$", "Ca"),
+            (r"^[A-Z]*[a-z' -]+$", "En"),
+            (r"^[0-9.,+-]+[0-9A-Za-z/$￥%<>（）()' -]+$", "NE"),
+            (r"^.{1}$", "Sg")
+        ]
+        for p, n in patt:
+            if re.search(p, b["text"].strip()):
+                return n
+        tks = [t for t in huqie.qie(b["text"]).split(" ") if len(t) > 1]
+        if len(tks) > 3:
+            if len(tks) < 12:
+                return "Tx"
+            else:
+                return "Lx"
+
+        if len(tks) == 1 and huqie.tag(tks[0]) == "nr":
+            return "Nr"
+
+        return "Ot"
+
+    def __cal_spans(self, boxes, rows, cols, tbl, html=True):
+        # caculate span
+        clft = [np.mean([c.get("C_left", c["x0"]) for c in cln])
+                for cln in cols]
+        crgt = [np.mean([c.get("C_right", c["x1"]) for c in cln])
+                for cln in cols]
+        rtop = [np.mean([c.get("R_top", c["top"]) for c in row])
+                for row in rows]
+        rbtm = [np.mean([c.get("R_btm", c["bottom"])
+                         for c in row]) for row in rows]
+        for b in boxes:
+            if "SP" not in b:
+                continue
+            b["colspan"] = [b["cn"]]
+            b["rowspan"] = [b["rn"]]
+            # col span
+            for j in range(0, len(clft)):
+                if j == b["cn"]:
+                    continue
+                if clft[j] + (crgt[j] - clft[j]) / 2 < b["H_left"]:
+                    continue
+                if crgt[j] - (crgt[j] - clft[j]) / 2 > b["H_right"]:
+                    continue
+                b["colspan"].append(j)
+            # row span
+            for j in range(0, len(rtop)):
+                if j == b["rn"]:
+                    continue
+                if rtop[j] + (rbtm[j] - rtop[j]) / 2 < b["H_top"]:
+                    continue
+                if rbtm[j] - (rbtm[j] - rtop[j]) / 2 > b["H_bott"]:
+                    continue
+                b["rowspan"].append(j)
+
+        def join(arr):
+            if not arr:
+                return ""
+            return "".join([t["text"] for t in arr])
+
+        # rm the spaning cells
+        for i in range(len(tbl)):
+            for j, arr in enumerate(tbl[i]):
+                if not arr:
+                    continue
+                if all(["rowspan" not in a and "colspan" not in a for a in arr]):
+                    continue
+                rowspan, colspan = [], []
+                for a in arr:
+                    if isinstance(a.get("rowspan", 0), list):
+                        rowspan.extend(a["rowspan"])
+                    if isinstance(a.get("colspan", 0), list):
+                        colspan.extend(a["colspan"])
+                rowspan, colspan = set(rowspan), set(colspan)
+                if len(rowspan) < 2 and len(colspan) < 2:
+                    for a in arr:
+                        if "rowspan" in a:
+                            del a["rowspan"]
+                        if "colspan" in a:
+                            del a["colspan"]
+                    continue
+                rowspan, colspan = sorted(rowspan), sorted(colspan)
+                rowspan = list(range(rowspan[0], rowspan[-1] + 1))
+                colspan = list(range(colspan[0], colspan[-1] + 1))
+                assert i in rowspan, rowspan
+                assert j in colspan, colspan
+                arr = []
+                for r in rowspan:
+                    for c in colspan:
+                        arr_txt = join(arr)
+                        if tbl[r][c] and join(tbl[r][c]) != arr_txt:
+                            arr.extend(tbl[r][c])
+                        tbl[r][c] = None if html else arr
+                for a in arr:
+                    if len(rowspan) > 1:
+                        a["rowspan"] = len(rowspan)
+                    elif "rowspan" in a:
+                        del a["rowspan"]
+                    if len(colspan) > 1:
+                        a["colspan"] = len(colspan)
+                    elif "colspan" in a:
+                        del a["colspan"]
+                tbl[rowspan[0]][colspan[0]] = arr
+
+        return tbl
+
+    def __construct_table(self, boxes, html=False):
+        cap = ""
+        i = 0
+        while i < len(boxes):
+            if self.is_caption(boxes[i]):
+                cap += boxes[i]["text"]
+                boxes.pop(i)
+                i -= 1
+            i += 1
+
+        if not boxes:
+            return []
+        for b in boxes:
+            b["btype"] = self._blockType(b)
+        max_type = Counter([b["btype"] for b in boxes]).items()
+        max_type = max(max_type, key=lambda x: x[1])[0] if max_type else ""
+        logging.debug("MAXTYPE: " + max_type)
+
+        rowh = [b["R_bott"] - b["R_top"] for b in boxes if "R" in b]
+        rowh = np.min(rowh) if rowh else 0
+        # boxes = self.sort_Y_firstly(boxes, rowh/5)
+        boxes = self.sort_R_firstly(boxes, rowh / 2)
+        boxes[0]["rn"] = 0
+        rows = [[boxes[0]]]
+        btm = boxes[0]["bottom"]
+        for b in boxes[1:]:
+            b["rn"] = len(rows) - 1
+            lst_r = rows[-1]
+            if lst_r[-1].get("R", "") != b.get("R", "") \
+                    or (b["top"] >= btm - 3 and lst_r[-1].get("R", "-1") != b.get("R", "-2")
+            ):  # new row
+                btm = b["bottom"]
+                b["rn"] += 1
+                rows.append([b])
+                continue
+            btm = (btm + b["bottom"]) / 2.
+            rows[-1].append(b)
+
+        colwm = [b["C_right"] - b["C_left"] for b in boxes if "C" in b]
+        colwm = np.min(colwm) if colwm else 0
+        crosspage = len(set([b["page_number"] for b in boxes])) > 1
+        if crosspage:
+            boxes = self.sort_X_firstly(boxes, colwm / 2, False)
+        else:
+            boxes = self.sort_C_firstly(boxes, colwm / 2)
+        boxes[0]["cn"] = 0
+        cols = [[boxes[0]]]
+        right = boxes[0]["x1"]
+        for b in boxes[1:]:
+            b["cn"] = len(cols) - 1
+            lst_c = cols[-1]
+            if (int(b.get("C", "1")) - int(lst_c[-1].get("C", "1")) == 1 and b["page_number"] == lst_c[-1][
+                "page_number"]) \
+                    or (b["x0"] >= right and lst_c[-1].get("C", "-1") != b.get("C", "-2")):  # new col
+                right = b["x1"]
+                b["cn"] += 1
+                cols.append([b])
+                continue
+            right = (right + b["x1"]) / 2.
+            cols[-1].append(b)
+
+        tbl = [[[] for _ in range(len(cols))] for _ in range(len(rows))]
+        for b in boxes:
+            tbl[b["rn"]][b["cn"]].append(b)
+
+        if len(rows) >= 4:
+            # remove single in column
+            j = 0
+            while j < len(tbl[0]):
+                e, ii = 0, 0
+                for i in range(len(tbl)):
+                    if tbl[i][j]:
+                        e += 1
+                        ii = i
+                    if e > 1:
+                        break
+                if e > 1:
+                    j += 1
+                    continue
+                f = (j > 0 and tbl[ii][j - 1] and tbl[ii]
+                [j - 1][0].get("text")) or j == 0
+                ff = (j + 1 < len(tbl[ii]) and tbl[ii][j + 1] and tbl[ii]
+                [j + 1][0].get("text")) or j + 1 >= len(tbl[ii])
+                if f and ff:
+                    j += 1
+                    continue
+                bx = tbl[ii][j][0]
+                logging.debug("Relocate column single: " + bx["text"])
+                # j column only has one value
+                left, right = 100000, 100000
+                if j > 0 and not f:
+                    for i in range(len(tbl)):
+                        if tbl[i][j - 1]:
+                            left = min(left, np.min(
+                                [bx["x0"] - a["x1"] for a in tbl[i][j - 1]]))
+                if j + 1 < len(tbl[0]) and not ff:
+                    for i in range(len(tbl)):
+                        if tbl[i][j + 1]:
+                            right = min(right, np.min(
+                                [a["x0"] - bx["x1"] for a in tbl[i][j + 1]]))
+                assert left < 100000 or right < 100000
+                if left < right:
+                    for jj in range(j, len(tbl[0])):
+                        for i in range(len(tbl)):
+                            for a in tbl[i][jj]:
+                                a["cn"] -= 1
+                    if tbl[ii][j - 1]:
+                        tbl[ii][j - 1].extend(tbl[ii][j])
+                    else:
+                        tbl[ii][j - 1] = tbl[ii][j]
+                    for i in range(len(tbl)):
+                        tbl[i].pop(j)
+
+                else:
+                    for jj in range(j + 1, len(tbl[0])):
+                        for i in range(len(tbl)):
+                            for a in tbl[i][jj]:
+                                a["cn"] -= 1
+                    if tbl[ii][j + 1]:
+                        tbl[ii][j + 1].extend(tbl[ii][j])
+                    else:
+                        tbl[ii][j + 1] = tbl[ii][j]
+                    for i in range(len(tbl)):
+                        tbl[i].pop(j)
+                cols.pop(j)
+        assert len(cols) == len(tbl[0]), "Column NO. miss matched: %d vs %d" % (
+            len(cols), len(tbl[0]))
+
+        if len(cols) >= 4:
+            # remove single in row
+            i = 0
+            while i < len(tbl):
+                e, jj = 0, 0
+                for j in range(len(tbl[i])):
+                    if tbl[i][j]:
+                        e += 1
+                        jj = j
+                    if e > 1:
+                        break
+                if e > 1:
+                    i += 1
+                    continue
+                f = (i > 0 and tbl[i - 1][jj] and tbl[i - 1]
+                [jj][0].get("text")) or i == 0
+                ff = (i + 1 < len(tbl) and tbl[i + 1][jj] and tbl[i + 1]
+                [jj][0].get("text")) or i + 1 >= len(tbl)
+                if f and ff:
+                    i += 1
+                    continue
+
+                bx = tbl[i][jj][0]
+                logging.debug("Relocate row single: " + bx["text"])
+                # i row only has one value
+                up, down = 100000, 100000
+                if i > 0 and not f:
+                    for j in range(len(tbl[i - 1])):
+                        if tbl[i - 1][j]:
+                            up = min(up, np.min(
+                                [bx["top"] - a["bottom"] for a in tbl[i - 1][j]]))
+                if i + 1 < len(tbl) and not ff:
+                    for j in range(len(tbl[i + 1])):
+                        if tbl[i + 1][j]:
+                            down = min(down, np.min(
+                                [a["top"] - bx["bottom"] for a in tbl[i + 1][j]]))
+                assert up < 100000 or down < 100000
+                if up < down:
+                    for ii in range(i, len(tbl)):
+                        for j in range(len(tbl[ii])):
+                            for a in tbl[ii][j]:
+                                a["rn"] -= 1
+                    if tbl[i - 1][jj]:
+                        tbl[i - 1][jj].extend(tbl[i][jj])
+                    else:
+                        tbl[i - 1][jj] = tbl[i][jj]
+                    tbl.pop(i)
+
+                else:
+                    for ii in range(i + 1, len(tbl)):
+                        for j in range(len(tbl[ii])):
+                            for a in tbl[ii][j]:
+                                a["rn"] -= 1
+                    if tbl[i + 1][jj]:
+                        tbl[i + 1][jj].extend(tbl[i][jj])
+                    else:
+                        tbl[i + 1][jj] = tbl[i][jj]
+                    tbl.pop(i)
+                rows.pop(i)
+
+        # which rows are headers
+        hdset = set([])
+        for i in range(len(tbl)):
+            cnt, h = 0, 0
+            for j, arr in enumerate(tbl[i]):
+                if not arr:
+                    continue
+                cnt += 1
+                if max_type == "Nu" and arr[0]["btype"] == "Nu":
+                    continue
+                if any([a.get("H") for a in arr]) \
+                        or (max_type == "Nu" and arr[0]["btype"] != "Nu"):
+                    h += 1
+            if h / cnt > 0.5:
+                hdset.add(i)
+
+        if html:
+            return [self.__html_table(cap, hdset,
+                                      self.__cal_spans(boxes, rows,
+                                                       cols, tbl, True)
+                                      )]
+
+        return self.__desc_table(cap, hdset,
+                                 self.__cal_spans(boxes, rows, cols, tbl, False))
+
+    def __html_table(self, cap, hdset, tbl):
+        # constrcut HTML
+        html = "<table>"
+        if cap:
+            html += f"<caption>{cap}</caption>"
+        for i in range(len(tbl)):
+            row = "<tr>"
+            txts = []
+            for j, arr in enumerate(tbl[i]):
+                if arr is None:
+                    continue
+                if not arr:
+                    row += "<td></td>" if i not in hdset else "<th></th>"
+                    continue
+                txt = ""
+                if arr:
+                    h = min(np.min([c["bottom"] - c["top"] for c in arr]) / 2,
+                            self.mean_height[arr[0]["page_number"] - 1] / 2)
+                    txt = "".join([c["text"]
+                                   for c in self.sort_Y_firstly(arr, h)])
+                txts.append(txt)
+                sp = ""
+                if arr[0].get("colspan"):
+                    sp = "colspan={}".format(arr[0]["colspan"])
+                if arr[0].get("rowspan"):
+                    sp += " rowspan={}".format(arr[0]["rowspan"])
+                if i in hdset:
+                    row += f"<th {sp} >" + txt + "</th>"
+                else:
+                    row += f"<td {sp} >" + txt + "</td>"
+
+            if i in hdset:
+                if all([t in hdset for t in txts]):
+                    continue
+                for t in txts:
+                    hdset.add(t)
+
+            if row != "<tr>":
+                row += "</tr>"
+            else:
+                row = ""
+            html += "\n" + row
+        html += "\n</table>"
+        return html
+
+    def __desc_table(self, cap, hdr_rowno, tbl):
+        # get text of every colomn in header row to become header text
+        clmno = len(tbl[0])
+        rowno = len(tbl)
+        headers = {}
+        hdrset = set()
+        lst_hdr = []
+        for r in sorted(list(hdr_rowno)):
+            headers[r] = ["" for _ in range(clmno)]
+            for i in range(clmno):
+                if not tbl[r][i]:
+                    continue
+                txt = "".join([a["text"].strip() for a in tbl[r][i]])
+                headers[r][i] = txt
+                hdrset.add(txt)
+            if all([not t for t in headers[r]]):
+                del headers[r]
+                hdr_rowno.remove(r)
+                continue
+            for j in range(clmno):
+                if headers[r][j]:
+                    continue
+                if j >= len(lst_hdr):
+                    break
+                headers[r][j] = lst_hdr[j]
+            lst_hdr = headers[r]
+        for i in range(rowno):
+            if i not in hdr_rowno:
+                continue
+            for j in range(i + 1, rowno):
+                if j not in hdr_rowno:
+                    break
+                for k in range(clmno):
+                    if not headers[j - 1][k]:
+                        continue
+                    if headers[j][k].find(headers[j - 1][k]) >= 0:
+                        continue
+                    if len(headers[j][k]) > len(headers[j - 1][k]):
+                        headers[j][k] += ("的" if headers[j][k]
+                                          else "") + headers[j - 1][k]
+                    else:
+                        headers[j][k] = headers[j - 1][k] \
+                                        + ("的" if headers[j - 1][k] else "") \
+                                        + headers[j][k]
+
+        logging.debug(
+            f">>>>>>>>>>>>>>>>>{cap}：SIZE:{rowno}X{clmno} Header: {hdr_rowno}")
+        row_txt = []
+        for i in range(rowno):
+            if i in hdr_rowno:
+                continue
+            rtxt = []
+
+            def append(delimer):
+                nonlocal rtxt, row_txt
+                rtxt = delimer.join(rtxt)
+                if row_txt and len(row_txt[-1]) + len(rtxt) < 64:
+                    row_txt[-1] += "\n" + rtxt
+                else:
+                    row_txt.append(rtxt)
+
+            r = 0
+            if len(headers.items()):
+                _arr = [(i - r, r) for r, _ in headers.items() if r < i]
+                if _arr:
+                    _, r = min(_arr, key=lambda x: x[0])
+
+            if r not in headers and clmno <= 2:
+                for j in range(clmno):
+                    if not tbl[i][j]:
+                        continue
+                    txt = "".join([a["text"].strip() for a in tbl[i][j]])
+                    if txt:
+                        rtxt.append(txt)
+                if rtxt:
+                    append("：")
+                continue
+
+            for j in range(clmno):
+                if not tbl[i][j]:
+                    continue
+                txt = "".join([a["text"].strip() for a in tbl[i][j]])
+                if not txt:
+                    continue
+                ctt = headers[r][j] if r in headers else ""
+                if ctt:
+                    ctt += "："
+                ctt += txt
+                if ctt:
+                    rtxt.append(ctt)
+
+            if rtxt:
+                row_txt.append("; ".join(rtxt))
+
+        if cap:
+            row_txt = [t + f"\t——来自“{cap}”" for t in row_txt]
+        return row_txt
+
+    @staticmethod
+    def is_caption(bx):
+        patt = [
+            r"[图表]+[ 0-9:：]{2,}"
+        ]
+        if any([re.match(p, bx["text"].strip()) for p in patt]) \
+                or bx["layout_type"].find("caption") >= 0:
+            return True
+        return False
+
+    def __extract_table_figure(self, need_image, ZM, return_html):
+        tables = {}
+        figures = {}
+        # extract figure and table boxes
+        i = 0
+        lst_lout_no = ""
+        nomerge_lout_no = []
+        while i < len(self.boxes):
+            if "layoutno" not in self.boxes[i]:
+                i += 1
+                continue
+            lout_no = str(self.boxes[i]["page_number"]) + \
+                      "-" + str(self.boxes[i]["layoutno"])
+            if self.is_caption(self.boxes[i]) or self.boxes[i]["layout_type"] in ["table caption", "title",
+                                                                                  "figure caption", "reference"]:
+                nomerge_lout_no.append(lst_lout_no)
+            if self.boxes[i]["layout_type"] == "table":
+                if re.match(r"(数据|资料|图表)*来源[:： ]", self.boxes[i]["text"]):
+                    self.boxes.pop(i)
+                    continue
+                if lout_no not in tables:
+                    tables[lout_no] = []
+                tables[lout_no].append(self.boxes[i])
+                self.boxes.pop(i)
+                lst_lout_no = lout_no
+                continue
+            if need_image and self.boxes[i]["layout_type"] == "figure":
+                if re.match(r"(数据|资料|图表)*来源[:： ]", self.boxes[i]["text"]):
+                    self.boxes.pop(i)
+                    continue
+                if lout_no not in figures:
+                    figures[lout_no] = []
+                figures[lout_no].append(self.boxes[i])
+                self.boxes.pop(i)
+                lst_lout_no = lout_no
+                continue
+            i += 1
+
+        # merge table on different pages
+        nomerge_lout_no = set(nomerge_lout_no)
+        tbls = sorted([(k, bxs) for k, bxs in tables.items()],
+                      key=lambda x: (x[1][0]["top"], x[1][0]["x0"]))
+
+        i = len(tbls) - 1
+        while i - 1 >= 0:
+            k0, bxs0 = tbls[i - 1]
+            k, bxs = tbls[i]
+            i -= 1
+            if k0 in nomerge_lout_no:
+                continue
+            if bxs[0]["page_number"] == bxs0[0]["page_number"]:
+                continue
+            if bxs[0]["page_number"] - bxs0[0]["page_number"] > 1:
+                continue
+            mh = self.mean_height[bxs[0]["page_number"] - 1]
+            if self._y_dis(bxs0[-1], bxs[0]) > mh * 23:
+                continue
+            tables[k0].extend(tables[k])
+            del tables[k]
+
+        def x_overlapped(a, b):
+            return not any([a["x1"] < b["x0"], a["x0"] > b["x1"]])
+
+        # find captions and pop out
+        i = 0
+        while i < len(self.boxes):
+            c = self.boxes[i]
+            # mh = self.mean_height[c["page_number"]-1]
+            if not self.is_caption(c):
+                i += 1
+                continue
+
+            # find the nearest layouts
+            def nearest(tbls):
+                nonlocal c
+                mink = ""
+                minv = 1000000000
+                for k, bxs in tbls.items():
+                    for b in bxs[:10]:
+                        if b.get("layout_type", "").find("caption") >= 0:
+                            continue
+                        y_dis = self._y_dis(c, b)
+                        x_dis = self._x_dis(
+                            c, b) if not x_overlapped(
+                            c, b) else 0
+                        dis = y_dis * y_dis + x_dis * x_dis
+                        if dis < minv:
+                            mink = k
+                            minv = dis
+                return mink, minv
+
+            tk, tv = nearest(tables)
+            fk, fv = nearest(figures)
+            if min(tv, fv) > 2000:
+                i += 1
+                continue
+            if tv < fv:
+                tables[tk].insert(0, c)
+                logging.debug(
+                    "TABLE:" +
+                    self.boxes[i]["text"] +
+                    "; Cap: " +
+                    tk)
+            else:
+                figures[fk].insert(0, c)
+                logging.debug(
+                    "FIGURE:" +
+                    self.boxes[i]["text"] +
+                    "; Cap: " +
+                    tk)
+            self.boxes.pop(i)
+
+        res = []
+
+        def cropout(bxs, ltype):
+            nonlocal ZM
+            pn = set([b["page_number"] - 1 for b in bxs])
+            if len(pn) < 2:
+                pn = list(pn)[0]
+                ht = self.page_cum_height[pn]
+                b = {
+                    "x0": np.min([b["x0"] for b in bxs]),
+                    "top": np.min([b["top"] for b in bxs]) - ht,
+                    "x1": np.max([b["x1"] for b in bxs]),
+                    "bottom": np.max([b["bottom"] for b in bxs]) - ht
+                }
+                louts = [l for l in self.page_layout[pn] if l["type"] == ltype]
+                ii = self.__find_overlapped(b, louts, naive=True)
+                if ii is not None:
+                    b = louts[ii]
+                else:
+                    logging.warn(
+                        f"Missing layout match: {pn + 1},%s" %
+                        (bxs[0].get(
+                            "layoutno", "")))
+
+                left, top, right, bott = b["x0"], b["top"], b["x1"], b["bottom"]
+                return self.page_images[pn] \
+                    .crop((left * ZM, top * ZM,
+                           right * ZM, bott * ZM))
+            pn = {}
+            for b in bxs:
+                p = b["page_number"] - 1
+                if p not in pn:
+                    pn[p] = []
+                pn[p].append(b)
+            pn = sorted(pn.items(), key=lambda x: x[0])
+            imgs = [cropout(arr, ltype) for p, arr in pn]
+            pic = Image.new("RGB",
+                            (int(np.max([i.size[0] for i in imgs])),
+                             int(np.sum([m.size[1] for m in imgs]))),
+                            (245, 245, 245))
+            height = 0
+            for img in imgs:
+                pic.paste(img, (0, int(height)))
+                height += img.size[1]
+            return pic
+
+        # crop figure out and add caption
+        for k, bxs in figures.items():
+            txt = "\n".join(
+                [b["text"] for b in bxs
+                 if not re.match(r"[0-9a-z.\+%-]", b["text"].strip())
+                 and len(b["text"].strip()) >= 4
+                 ]
+            )
+            if not txt:
+                continue
+
+            res.append(
+                (cropout(
+                    bxs,
+                    "figure"),
+                 [txt] if not return_html else [f"<p>{txt}</p>"]))
+
+        for k, bxs in tables.items():
+            if not bxs:
+                continue
+            res.append((cropout(bxs, "table"),
+                        self.__construct_table(bxs, html=return_html)))
+
+        return res
+
+    def proj_match(self, line):
+        if len(line) <= 2:
+            return
+        if re.match(r"[0-9 ().,%%+/-]+$", line):
+            return False
+        for p, j in [
+            (r"第[零一二三四五六七八九十百]+章", 1),
+            (r"第[零一二三四五六七八九十百]+[条节]", 2),
+            (r"[零一二三四五六七八九十百]+[、 　]", 3),
+            (r"[\(（][零一二三四五六七八九十百]+[）\)]", 4),
+            (r"[0-9]+(、|\.[　 ]|\.[^0-9])", 5),
+            (r"[0-9]+\.[0-9]+(、|[. 　]|[^0-9])", 6),
+            (r"[0-9]+\.[0-9]+\.[0-9]+(、|[ 　]|[^0-9])", 7),
+            (r"[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(、|[ 　]|[^0-9])", 8),
+            (r".{,48}[：:?？]$", 9),
+            (r"[0-9]+）", 10),
+            (r"[\(（][0-9]+[）\)]", 11),
+            (r"[零一二三四五六七八九十百]+是", 12),
+            (r"[⚫•➢✓]", 12)
+        ]:
+            if re.match(p, line):
+                return j
+        return
+
+    def __filterout_scraps(self, boxes, ZM):
+        def line_tag(bx):
+            pn = [bx["page_number"]]
+            top = bx["top"] - self.page_cum_height[pn[0] - 1]
+            bott = bx["bottom"] - self.page_cum_height[pn[0] - 1]
+            while bott * ZM > self.page_images[pn[-1] - 1].size[1]:
+                bott -= self.page_images[pn[-1] - 1].size[1] / ZM
+                pn.append(pn[-1] + 1)
+
+            return "@@{}\t{:.1f}\t{:.1f}\t{:.1f}\t{:.1f}##" \
+                .format("-".join([str(p) for p in pn]),
+                        bx["x0"], bx["x1"], top, bott)
+
+        def width(b):
+            return b["x1"] - b["x0"]
+
+        def height(b):
+            return b["bottom"] - b["top"]
+
+        def usefull(b):
+            if b.get("layout_type"):
+                return True
+            if width(
+                    b) > self.page_images[b["page_number"] - 1].size[0] / ZM / 3:
+                return True
+            if b["bottom"] - b["top"] > self.mean_height[b["page_number"] - 1]:
+                return True
+            return False
+
+        res = []
+        while boxes:
+            lines = []
+            widths = []
+            pw = self.page_images[boxes[0]["page_number"] - 1].size[0] / ZM
+            mh = self.mean_height[boxes[0]["page_number"] - 1]
+            mj = self.proj_match(
+                boxes[0]["text"]) or boxes[0].get(
+                "layout_type",
+                "") == "title"
+
+            def dfs(line, st):
+                nonlocal mh, pw, lines, widths
+                lines.append(line)
+                widths.append(width(line))
+                width_mean = np.mean(widths)
+                mmj = self.proj_match(
+                    line["text"]) or line.get(
+                    "layout_type",
+                    "") == "title"
+                for i in range(st + 1, min(st + 20, len(boxes))):
+                    if (boxes[i]["page_number"] - line["page_number"]) > 0:
+                        break
+                    if not mmj and self._y_dis(
+                            line, boxes[i]) >= 3 * mh and height(line) < 1.5 * mh:
+                        break
+
+                    if not usefull(boxes[i]):
+                        continue
+                    if mmj or \
+                            (self._x_dis(boxes[i], line) < pw / 10): \
+                            # and abs(width(boxes[i])-width_mean)/max(width(boxes[i]),width_mean)<0.5):
+                        # concat following
+                        dfs(boxes[i], i)
+                        boxes.pop(i)
+                        break
+
+            try:
+                if usefull(boxes[0]):
+                    dfs(boxes[0], 0)
+                else:
+                    logging.debug("WASTE: " + boxes[0]["text"])
+            except Exception as e:
+                pass
+            boxes.pop(0)
+            mw = np.mean(widths)
+            if mj or mw / pw >= 0.35 or mw > 200:
+                res.append("\n".join([c["text"] + line_tag(c) for c in lines]))
+            else:
+                logging.debug("REMOVED: " +
+                              "<<".join([c["text"] for c in lines]))
+
+        return "\n\n".join(res)
+
+    def __call__(self, fnm, need_image=True, zoomin=3, return_html=False):
+        self.pdf = pdfplumber.open(fnm)
+        self.lefted_chars = []
+        self.mean_height = []
+        self.mean_width = []
+        self.boxes = []
+        self.garbages = {}
+        self.page_cum_height = [0]
+        self.page_layout = []
+        self.page_images = [p.to_image(
+            resolution=72 * zoomin).annotated for i, p in enumerate(self.pdf.pages[:299])]
+        logging.info("Images converted.")
+        logging.info("Table processed.")
+
+        for i, img in enumerate(self.page_images):
+            chars = [c for c in self.pdf.pages[i].chars if self._has_color(c)]
+            self.mean_height.append(
+                np.median(sorted([c["height"] for c in chars])) if chars else 0
+            )
+            self.mean_width.append(
+                np.median(sorted([c["width"] for c in chars])) if chars else 8
+            )
+            if i > 0:
+                if not chars:
+                    self.page_cum_height.append(img.size[1] / zoomin)
+                else:
+                    self.page_cum_height.append(
+                        np.max([c["bottom"] for c in chars]))
+            self.__ocr_paddle(i + 1, img, chars, zoomin)
+        self.__layouts_paddle(zoomin)
+
+        self.page_cum_height = np.cumsum(self.page_cum_height)
+        assert len(self.page_cum_height) == len(self.page_images)
+
+        garbage = set()
+        for k in self.garbages.keys():
+            self.garbages[k] = Counter(self.garbages[k])
+            for g, c in self.garbages[k].items():
+                if c > 1:
+                    garbage.add(g)
+
+        logging.debug("GARBAGE:" + ",".join(garbage))
+        self.boxes = [b for b in self.boxes if b["text"] not in garbage]
+
+        # cumlative Y
+        for i in range(len(self.boxes)):
+            self.boxes[i]["top"] += \
+                self.page_cum_height[self.boxes[i]["page_number"] - 1]
+            self.boxes[i]["bottom"] += \
+                self.page_cum_height[self.boxes[i]["page_number"] - 1]
+
+        self.__table_transformer_job(zoomin)
+        self.__text_merge(garbage)
+        self.__filter_forpages()
+        tbls = self.__extract_table_figure(need_image, zoomin, return_html)
+
+        return self.__filterout_scraps(deepcopy(self.boxes), zoomin), tbls
+
+    def remove_tag(self, txt):
+        return re.sub(r"@@[\t0-9.-]+?##", "", txt)
+
+    def crop(self, text, ZM=3):
+        imgs = []
+        for tag in re.findall(r"@@[0-9-]+\t[0-9.\t]+##", text):
+            pn, left, right, top, bottom = tag.strip(
+                "#").strip("@").split("\t")
+            left, right, top, bottom = float(left), float(
+                right), float(top), float(bottom)
+            bottom *= ZM
+            pns = [int(p) - 1 for p in pn.split("-")]
+            for pn in pns[1:]:
+                bottom += self.page_images[pn - 1].size[1]
+            imgs.append(
+                self.page_images[pns[0]].crop((left * ZM, top * ZM,
+                                               right *
+                                               ZM, min(
+                    bottom, self.page_images[pns[0]].size[1])
+                                               ))
+            )
+            bottom -= self.page_images[pns[0]].size[1]
+            for pn in pns[1:]:
+                imgs.append(
+                    self.page_images[pn].crop((left * ZM, 0,
+                                               right * ZM,
+                                               min(bottom,
+                                                   self.page_images[pn].size[1])
+                                               ))
+                )
+                bottom -= self.page_images[pn].size[1]
+
+        if not imgs:
+            return
+        GAP = 2
+        height = 0
+        for img in imgs:
+            height += img.size[1] + GAP
+        height = int(height)
+        pic = Image.new("RGB",
+                        (int(np.max([i.size[0] for i in imgs])), height),
+                        (245, 245, 245))
+        height = 0
+        for img in imgs:
+            pic.paste(img, (0, int(height)))
+            height += img.size[1] + GAP
+        return pic
+
+
+if __name__ == "__main__":
+    pass

python/util/config.py

@@ -0,0 +1,24 @@
+from configparser  import ConfigParser
+import os,inspect
+
+CF = ConfigParser()
+__fnm = os.path.join(os.path.dirname(__file__), '../conf/sys.cnf')
+if not os.path.exists(__fnm):__fnm = os.path.join(os.path.dirname(__file__), '../../conf/sys.cnf')
+assert os.path.exists(__fnm), f"【EXCEPTION】can't find {__fnm}." + os.path.dirname(__file__)
+if not os.path.exists(__fnm): __fnm = "./sys.cnf"
+
+CF.read(__fnm)
+
+
+class Config:
+    def __init__(self, env):
+        self.env = env
+        if env == "spark":CF.read("./cv.cnf")
+
+    def get(self, key):
+        global CF
+        return CF.get(self.env, key)
+
+def init(env):
+    return Config(env)
+

python/util/es_conn.py

@@ -0,0 +1,407 @@
+import re
+import logging
+import json
+import time
+import copy
+import elasticsearch
+from elasticsearch import Elasticsearch
+from elasticsearch_dsl import UpdateByQuery, Search, Index
+from util import config
+
+print("Elasticsearch version: ", elasticsearch.__version__)
+
+
+def instance(env):
+    CF = config.init(env)
+    ES_DRESS = CF.get("es").split(",")
+
+    ES = Elasticsearch(
+        ES_DRESS,
+        timeout=600
+    )
+
+    print("ES: ", ES_DRESS, ES.info())
+
+    return ES
+
+
+class HuEs:
+    def __init__(self, env):
+        self.env = env
+        self.info = {}
+        self.config = config.init(env)
+        self.conn()
+        self.idxnm = self.config.get("idx_nm")
+        if not self.es.ping():
+            raise Exception("Can't connect to ES cluster")
+
+    def conn(self):
+        for _ in range(10):
+            try:
+                c = instance(self.env)
+                if c:
+                    self.es = c
+                    self.info = c.info()
+                    logging.info("Connect to es.")
+                    break
+            except Exception as e:
+                logging.error("Fail to connect to es: " + str(e))
+
+    def version(self):
+        v = self.info.get("version", {"number": "5.6"})
+        v = v["number"].split(".")[0]
+        return int(v) >= 7
+
+    def upsert(self, df, idxnm=""):
+        res = []
+        for d in df:
+            id = d["id"]
+            del d["id"]
+            d = {"doc": d, "doc_as_upsert": "true"}
+            T = False
+            for _ in range(10):
+                try:
+                    if not self.version():
+                        r = self.es.update(
+                            index=(
+                                self.idxnm if not idxnm else idxnm),
+                            body=d,
+                            id=id,
+                            doc_type="doc",
+                            refresh=False,
+                            retry_on_conflict=100)
+                    else:
+                        r = self.es.update(
+                            index=(
+                                self.idxnm if not idxnm else idxnm),
+                            body=d,
+                            id=id,
+                            refresh=False,
+                            doc_type="_doc",
+                            retry_on_conflict=100)
+                    logging.info("Successfully upsert: %s" % id)
+                    T = True
+                    break
+                except Exception as e:
+                    logging.warning("Fail to index: " +
+                                    json.dumps(d, ensure_ascii=False) + str(e))
+                    if re.search(r"(Timeout|time out)", str(e), re.IGNORECASE):
+                        time.sleep(3)
+                        continue
+                    self.conn()
+                    T = False
+
+            if not T:
+                res.append(d)
+                logging.error(
+                    "Fail to index: " +
+                    re.sub(
+                        "[\r\n]",
+                        "",
+                        json.dumps(
+                            d,
+                            ensure_ascii=False)))
+                d["id"] = id
+                d["_index"] = self.idxnm
+
+        if not res:
+            return True
+        return False
+
+    def bulk(self, df, idx_nm=None):
+        ids, acts = {}, []
+        for d in df:
+            id = d["id"] if "id" in d else d["_id"]
+            ids[id] = copy.deepcopy(d)
+            ids[id]["_index"] = self.idxnm if not idx_nm else idx_nm
+            if "id" in d:
+                del d["id"]
+            if "_id" in d:
+                del d["_id"]
+            acts.append(
+                {"update": {"_id": id, "_index": ids[id]["_index"]}, "retry_on_conflict": 100})
+            acts.append({"doc": d, "doc_as_upsert": "true"})
+            logging.info("bulk upsert: %s" % id)
+
+        res = []
+        for _ in range(100):
+            try:
+                if elasticsearch.__version__[0] < 8:
+                    r = self.es.bulk(
+                        index=(
+                            self.idxnm if not idx_nm else idx_nm),
+                        body=acts,
+                        refresh=False,
+                        timeout="600s")
+                else:
+                    r = self.es.bulk(index=(self.idxnm if not idx_nm else
+                                            idx_nm), operations=acts,
+                                     refresh=False, timeout="600s")
+                if re.search(r"False", str(r["errors"]), re.IGNORECASE):
+                    return res
+
+                for it in r["items"]:
+                    if "error" in it["update"]:
+                        res.append(str(it["update"]["_id"]) +
+                                   ":" + str(it["update"]["error"]))
+
+                return res
+            except Exception as e:
+                logging.warn("Fail to bulk: " + str(e))
+                print(e)
+                if re.search(r"(Timeout|time out)", str(e), re.IGNORECASE):
+                    time.sleep(3)
+                    continue
+                self.conn()
+
+        return res
+
+    def bulk4script(self, df):
+        ids, acts = {}, []
+        for d in df:
+            id = d["id"]
+            ids[id] = copy.deepcopy(d["raw"])
+            acts.append({"update": {"_id": id, "_index": self.idxnm}})
+            acts.append(d["script"])
+            logging.info("bulk upsert: %s" % id)
+
+        res = []
+        for _ in range(10):
+            try:
+                if not self.version():
+                    r = self.es.bulk(
+                        index=self.idxnm,
+                        body=acts,
+                        refresh=False,
+                        timeout="600s",
+                        doc_type="doc")
+                else:
+                    r = self.es.bulk(
+                        index=self.idxnm,
+                        body=acts,
+                        refresh=False,
+                        timeout="600s")
+                if re.search(r"False", str(r["errors"]), re.IGNORECASE):
+                    return res
+
+                for it in r["items"]:
+                    if "error" in it["update"]:
+                        res.append(str(it["update"]["_id"]))
+
+                return res
+            except Exception as e:
+                logging.warning("Fail to bulk: " + str(e))
+                if re.search(r"(Timeout|time out)", str(e), re.IGNORECASE):
+                    time.sleep(3)
+                    continue
+                self.conn()
+
+        return res
+
+    def rm(self, d):
+        for _ in range(10):
+            try:
+                if not self.version():
+                    r = self.es.delete(
+                        index=self.idxnm,
+                        id=d["id"],
+                        doc_type="doc",
+                        refresh=True)
+                else:
+                    r = self.es.delete(
+                        index=self.idxnm,
+                        id=d["id"],
+                        refresh=True,
+                        doc_type="_doc")
+                logging.info("Remove %s" % d["id"])
+                return True
+            except Exception as e:
+                logging.warn("Fail to delete: " + str(d) + str(e))
+                if re.search(r"(Timeout|time out)", str(e), re.IGNORECASE):
+                    time.sleep(3)
+                    continue
+                if re.search(r"(not_found)", str(e), re.IGNORECASE):
+                    return True
+                self.conn()
+
+        logging.error("Fail to delete: " + str(d))
+
+        return False
+
+    def search(self, q, idxnm=None, src=False, timeout="2s"):
+        print(json.dumps(q, ensure_ascii=False))
+        for i in range(3):
+            try:
+                res = self.es.search(index=(self.idxnm if not idxnm else idxnm),
+                                     body=q,
+                                     timeout=timeout,
+                                     # search_type="dfs_query_then_fetch",
+                                     track_total_hits=True,
+                                     _source=src)
+                if str(res.get("timed_out", "")).lower() == "true":
+                    raise Exception("Es Timeout.")
+                return res
+            except Exception as e:
+                logging.error(
+                    "ES search exception: " +
+                    str(e) +
+                    "【Q】：" +
+                    str(q))
+                if str(e).find("Timeout") > 0:
+                    continue
+                raise e
+        logging.error("ES search timeout for 3 times!")
+        raise Exception("ES search timeout.")
+
+    def updateByQuery(self, q, d):
+        ubq = UpdateByQuery(index=self.idxnm).using(self.es).query(q)
+        scripts = ""
+        for k, v in d.items():
+            scripts += "ctx._source.%s = params.%s;" % (str(k), str(k))
+        ubq = ubq.script(source=scripts, params=d)
+        ubq = ubq.params(refresh=False)
+        ubq = ubq.params(slices=5)
+        ubq = ubq.params(conflicts="proceed")
+        for i in range(3):
+            try:
+                r = ubq.execute()
+                return True
+            except Exception as e:
+                logging.error("ES updateByQuery exception: " +
+                              str(e) + "【Q】：" + str(q.to_dict()))
+                if str(e).find("Timeout") > 0 or str(e).find("Conflict") > 0:
+                    continue
+
+        return False
+
+    def deleteByQuery(self, query, idxnm=""):
+        for i in range(3):
+            try:
+                r = self.es.delete_by_query(
+                    index=idxnm if idxnm else self.idxnm,
+                    body=Search().query(query).to_dict())
+                return True
+            except Exception as e:
+                logging.error("ES updateByQuery deleteByQuery: " +
+                              str(e) + "【Q】：" + str(query.to_dict()))
+                if str(e).find("Timeout") > 0 or str(e).find("Conflict") > 0:
+                    continue
+
+        return False
+
+    def update(self, id, script, routing=None):
+        for i in range(3):
+            try:
+                if not self.version():
+                    r = self.es.update(
+                        index=self.idxnm,
+                        id=id,
+                        body=json.dumps(
+                            script,
+                            ensure_ascii=False),
+                        doc_type="doc",
+                        routing=routing,
+                        refresh=False)
+                else:
+                    r = self.es.update(index=self.idxnm, id=id, body=json.dumps(script, ensure_ascii=False),
+                                       routing=routing, refresh=False)  # , doc_type="_doc")
+                return True
+            except Exception as e:
+                print(e)
+                logging.error("ES update exception: " + str(e) + " id：" + str(id) + ", version:" + str(self.version()) +
+                              json.dumps(script, ensure_ascii=False))
+                if str(e).find("Timeout") > 0:
+                    continue
+
+        return False
+
+    def indexExist(self, idxnm):
+        s = Index(idxnm if idxnm else self.idxnm, self.es)
+        for i in range(3):
+            try:
+                return s.exists()
+            except Exception as e:
+                logging.error("ES updateByQuery indexExist: " + str(e))
+                if str(e).find("Timeout") > 0 or str(e).find("Conflict") > 0:
+                    continue
+
+        return False
+
+    def docExist(self, docid, idxnm=None):
+        for i in range(3):
+            try:
+                return self.es.exists(index=(idxnm if idxnm else self.idxnm),
+                                      id=docid)
+            except Exception as e:
+                logging.error("ES Doc Exist: " + str(e))
+                if str(e).find("Timeout") > 0 or str(e).find("Conflict") > 0:
+                    continue
+        return False
+
+    def createIdx(self, idxnm, mapping):
+        try:
+            if elasticsearch.__version__[0] < 8:
+                return self.es.indices.create(idxnm, body=mapping)
+            from elasticsearch.client import IndicesClient
+            return IndicesClient(self.es).create(index=idxnm,
+                                                 settings=mapping["settings"],
+                                                 mappings=mapping["mappings"])
+        except Exception as e:
+            logging.error("ES create index error %s ----%s" % (idxnm, str(e)))
+
+    def deleteIdx(self, idxnm):
+        try:
+            return self.es.indices.delete(idxnm, allow_no_indices=True)
+        except Exception as e:
+            logging.error("ES delete index error %s ----%s" % (idxnm, str(e)))
+
+    def getTotal(self, res):
+        if isinstance(res["hits"]["total"], type({})):
+            return res["hits"]["total"]["value"]
+        return res["hits"]["total"]
+
+    def getDocIds(self, res):
+        return [d["_id"] for d in res["hits"]["hits"]]
+
+    def getSource(self, res):
+        rr = []
+        for d in res["hits"]["hits"]:
+            d["_source"]["id"] = d["_id"]
+            d["_source"]["_score"] = d["_score"]
+            rr.append(d["_source"])
+        return rr
+
+    def scrollIter(self, pagesize=100, scroll_time='2m', q={
+        "query": {"match_all": {}}, "sort": [{"updated_at": {"order": "desc"}}]}):
+        for _ in range(100):
+            try:
+                page = self.es.search(
+                    index=self.idxnm,
+                    scroll=scroll_time,
+                    size=pagesize,
+                    body=q,
+                    _source=None
+                )
+                break
+            except Exception as e:
+                logging.error("ES scrolling fail. " + str(e))
+                time.sleep(3)
+
+        sid = page['_scroll_id']
+        scroll_size = page['hits']['total']["value"]
+        logging.info("[TOTAL]%d" % scroll_size)
+        # Start scrolling
+        while scroll_size > 0:
+            yield page["hits"]["hits"]
+            for _ in range(100):
+                try:
+                    page = self.es.scroll(scroll_id=sid, scroll=scroll_time)
+                    break
+                except Exception as e:
+                    logging.error("ES scrolling fail. " + str(e))
+                    time.sleep(3)
+
+            # Update the scroll ID
+            sid = page['_scroll_id']
+            # Get the number of results that we returned in the last scroll
+            scroll_size = len(page['hits']['hits'])

python/util/setup_logging.py

@@ -0,0 +1,36 @@
+import json
+import logging.config
+import os
+
+
+def log_dir():
+    fnm = os.path.join(os.path.dirname(__file__), '../log/')
+    if not os.path.exists(fnm):
+        fnm = os.path.join(os.path.dirname(__file__), '../../log/')
+    assert os.path.exists(fnm), f"Can't locate log dir: {fnm}"
+    return fnm
+
+
+def setup_logging(default_path="conf/logging.json",
+                  default_level=logging.INFO,
+                  env_key="LOG_CFG"):
+    path = default_path
+    value = os.getenv(env_key, None)
+    if value:
+        path = value
+    if os.path.exists(path):
+        with open(path, "r") as f:
+            config = json.load(f)
+            fnm = log_dir()
+
+            config["handlers"]["info_file_handler"]["filename"] = fnm + "info.log"
+            config["handlers"]["error_file_handler"]["filename"] = fnm + "error.log"
+            logging.config.dictConfig(config)
+    else:
+        logging.basicConfig(level=default_level)
+
+
+__fnm = os.path.join(os.path.dirname(__file__), 'conf/logging.json')
+if not os.path.exists(__fnm):
+    __fnm = os.path.join(os.path.dirname(__file__), '../../conf/logging.json')
+setup_logging(__fnm)