update remove_* rules

Author: dongrixinyu

README.md

@@ -11,7 +11,7 @@
     <a alt="Downloads">
         <img src="https://img.shields.io/badge/downloads-6k-yellow" /></a>
     <a alt="Version">
-        <img src="https://img.shields.io/badge/version-1.3.50-green" /></a>
+        <img src="https://img.shields.io/badge/version-1.3.51-green" /></a>
     <a href="https://github.com/dongrixinyu/JioNLP/pulse" alt="Activity">
         <img src="https://img.shields.io/github/commit-activity/m/dongrixinyu/JioNLP?color=blue" /></a>
 </p>

jionlp/__init__.py

@@ -8,7 +8,7 @@
 # description: Preprocessing tool for Chinese NLP
 """
 
-__version__ = '1.3.50'
+__version__ = '1.3.51'
 
 
 import os
@@ -78,6 +78,7 @@
 │   | 2021-10-25 | update extract money and parse money                |   │
 │   | 2021-11-10 | add logger tuner                                    |   │
 │   | 2021-12-04 | add chinese word segmentor tools                    |   │
+│   | 2022-03-02 | update email & tel rules                            |   │
 │                                                                          │
 ╰──────────────────────────────────────────────────────────────────────────╯
 """

jionlp/rule/extractor.py

@@ -23,6 +23,7 @@ def __init__(self):
         self.money_pattern = None
         self.email_pattern = None
         self.email_domain_pattern = None
+        self.email_prefix_pattern = None
         self.url_pattern = None
         self.phone_number_pattern = None
         self.ip_address_pattern = None
@@ -32,6 +33,7 @@ def __init__(self):
         self.strict_qq_pattern = None
         self.cell_phone_pattern = None
         self.landline_phone_pattern = None
+        self.phone_prefix_pattern = None
         self.extract_parentheses_pattern = None
         self.remove_parentheses_pattern = None
         self.parentheses_pattern = PARENTHESES_PATTERN
@@ -47,6 +49,7 @@ def _extract_base(pattern, text, with_offset=False):
         """ 正则抽取器的基础函数
 
         Args:
+            pattern(re.compile): 正则表达式对象
             text(str): 字符串文本
             with_offset(bool): 是否携带 offset （抽取内容字段在文本中的位置信息）
 
@@ -55,15 +58,9 @@ def _extract_base(pattern, text, with_offset=False):
 
         """
         if with_offset:
-            '''
-            if pattern == self.strict_qq_pattern:
-                for item in pattern.finditer(text):
-                    pdb.set_trace()
-                pdb.set_trace()
-            #'''
             results = [{'text': item.group(1), 
                         'offset': (item.span()[0] - 1, item.span()[1] - 1)}
-                      for item in pattern.finditer(text)]
+                       for item in pattern.finditer(text)]
         else:
             results = [item.group(1) for item in pattern.finditer(text)]
 
@@ -95,8 +92,9 @@ def clean_text(self, text, remove_html_tag=True,
                    convert_full2half=True,
                    remove_exception_char=True, remove_url=True,
                    remove_redundant_char=True, remove_parentheses=True,
-                   remove_email=True, remove_phone_number=True):
-        """ 清洗文本
+                   remove_email=True, remove_phone_number=True,
+                   delete_prefix=False):
+        """ 清洗文本，关键字参数均默认为 True
 
         Args:
             text(str): 待清理文本
@@ -108,6 +106,7 @@ def clean_text(self, text, remove_html_tag=True,
             remove_url(bool): 是否删除 url 链接
             remove_email(bool): 是否删除 email
             remove_phone_number(bool): 是否删除电话号码
+            delete_prefix(bool): 是否删除 email 和 电话号码的前缀，如 `E-mail: [email protected]`
 
         Returns:
             str: 清理后的文本
@@ -127,9 +126,9 @@ def clean_text(self, text, remove_html_tag=True,
         if remove_url:
             text = self.remove_url(text)
         if remove_email:
-            text = self.remove_email(text)
+            text = self.remove_email(text, delete_prefix=delete_prefix)
         if remove_phone_number:
-            text = self.remove_phone_number(text)
+            text = self.remove_phone_number(text, delete_prefix=delete_prefix)
 
         return text
 
@@ -167,12 +166,11 @@ def extract_email(self, text, detail=False):
 
             detail_results = list()
             for item in results:
-                domain_name = self.email_domain_pattern.search(
-                    item['text']).group(1)
+                domain_name = self.email_domain_pattern.search(item['text']).group(1)
                 item.update({'domain_name': domain_name})
                 detail_results.append(item)
             return detail_results
-            
+
     def extract_id_card(self, text, detail=False):
         """ 提取文本中的 ID 身份证号
 
@@ -218,7 +216,7 @@ def extract_money(self, text, detail=False):
             detail(bool): 返回字符串的详细信息 offset，默认为 False
 
         Returns:
-            list: email列表
+            list: 货币金额列表
 
         Examples:
             >>> import jionlp as jio
@@ -285,7 +283,7 @@ def extract_qq(self, text, detail=False, strict=True):
             strict(bool): QQ号很容易和其他数字混淆，因此选择采用严格或宽松规则匹配
 
         Returns:
-            list: email列表
+            list: QQ 号列表
 
         """
         if self.qq_pattern is None:
@@ -400,21 +398,50 @@ def extract_parentheses(self, text, parentheses=PARENTHESES_PATTERN, detail=Fals
 
         return content_list
 
-    def remove_email(self, text):
+    def remove_email(self, text, delete_prefix=False):
         """ 删除文本中的 email
 
         Args:
             text(str): 字符串文本
+            delete_prefix(bool): 删除电子邮箱前的前缀符，如 `E-mail: [email protected]`
+                由于计算前缀符的匹配，该方法计算效率会慢。
 
         Returns:
             str: 删除 email 后的文本
 
         """
         if self.email_pattern is None:
             self.email_pattern = re.compile(EMAIL_PATTERN)
-            
+            self.email_prefix_pattern = re.compile(EMAIL_PREFIX_PATTERN)
+
         text = ''.join(['#', text, '#'])
-        return self.email_pattern.sub('', text)[1:-1]
+        if not delete_prefix:
+            text = self.email_pattern.sub('', text)
+            return text[1:-1]
+        else:
+
+            results = self._extract_base(self.email_pattern, text, with_offset=True)
+            prefix_results = self._extract_base(self.email_prefix_pattern, text, with_offset=True)
+
+            offset_list = [item['offset'][0] for item in results]
+
+            clean_prefix_offsets = [
+                item['offset'] for item in prefix_results if item['offset'][1] in offset_list]
+
+            final_text_list = list()
+            for idx, item in enumerate(clean_prefix_offsets):
+                if idx == 0:
+                    final_text_list.append(text[0: item[0]+1])
+
+                if idx == len(clean_prefix_offsets) - 1:
+                    final_text_list.append(text[item[1]+1:])
+                else:
+                    final_text_list.append(text[item[1]+1: clean_prefix_offsets[idx + 1][0]+1])
+
+            text = ''.join(final_text_list)
+            text = self.email_pattern.sub('', text)
+
+        return text[1:-1]
 
     def remove_exception_char(self, text):
         """ 删除文本中的异常字符
@@ -514,26 +541,57 @@ def remove_parentheses(self, text, parentheses=PARENTHESES_PATTERN):
                 return text
             length = len(text)
 
-    def remove_phone_number(self, text):
+    def remove_phone_number(self, text, delete_prefix=False):
         """ 删除文本中的电话号码
 
         Args:
             text(str): 字符串文本
+            delete_prefix(bool): 删除电话号码前缀，如 `电  话：198xxxxxxxx`
 
         Returns:
             str: 删除电话号码后的文本
 
         """
         if self.cell_phone_pattern is None:
             self.cell_phone_pattern = re.compile(CELL_PHONE_PATTERN)
-            
+            self.phone_prefix_pattern = re.compile(PHONE_PREFIX_PATTERN)
+
         if self.landline_phone_pattern is None:
             self.landline_phone_pattern = re.compile(LANDLINE_PHONE_PATTERN)
+            self.phone_prefix_pattern = re.compile(PHONE_PREFIX_PATTERN)
 
         text = ''.join(['#', text, '#'])
-        text = self.cell_phone_pattern.sub('', text)
-        text = self.landline_phone_pattern.sub('', text)
-        
+
+        if not delete_prefix:
+            text = self.cell_phone_pattern.sub('', text)
+            text = self.landline_phone_pattern.sub('', text)
+
+        else:
+            cell_results = self._extract_base(self.cell_phone_pattern, text, with_offset=True)
+            landline_results = self._extract_base(self.landline_phone_pattern, text, with_offset=True)
+            results = sorted(cell_results + landline_results, key=lambda i: i['offset'][0])
+
+            prefix_results = self._extract_base(self.phone_prefix_pattern, text, with_offset=True)
+
+            offset_list = [item['offset'][0] for item in results]
+
+            clean_prefix_offsets = [
+                item['offset'] for item in prefix_results if item['offset'][1] in offset_list]
+
+            final_text_list = list()
+            for idx, item in enumerate(clean_prefix_offsets):
+                if idx == 0:
+                    final_text_list.append(text[0: item[0]+1])
+
+                if idx == len(clean_prefix_offsets) - 1:
+                    final_text_list.append(text[item[1]+1:])
+                else:
+                    final_text_list.append(text[item[1]+1: clean_prefix_offsets[idx + 1][0]+1])
+
+            text = ''.join(final_text_list)
+            text = self.cell_phone_pattern.sub('', text)
+            text = self.landline_phone_pattern.sub('', text)
+
         return text[1:-1]
 
     def remove_qq(self, text, strict=True):

jionlp/rule/rule_pattern.py

@@ -16,6 +16,11 @@
 # 该规则用于抽取与判定手机号的归属地，即抽取前三位、中间4位
 CELL_PHONE_CHECK_PATTERN = r'((1[3-9][0-9]))([- ])?\d{4}([- ])?\d{4}'
 
+# 手机|电话 号码前缀，例如：`Tel: 18902437922`，用于删除前缀
+PHONE_PREFIX_PATTERN = r'(([tT](el(ephone)?|EL(EPHONE)?)|[cC](ell(phone)?|ELL(PHONE)?)|' \
+                       r'((联系)?电[ \t\u3000]*话|手[ \t\u3000]*机)(号(码)?)?)(:|：)?[\t \u3000]*)' \
+                       r'(?=[^:： \t\u3000])'
+
 # ---------------------------------------------------------------------
 # 中文字符正则
 ANCIENT_CHINESE_CHAR_PATTERN = '[一-龥㐀-䶵]'  # 在 gb13000.1 基础上扩展 6582 个古汉字，共 27484 个汉字
@@ -38,6 +43,10 @@
 # 抽取邮箱的域名
 EMAIL_DOMAIN_PATTERN = r'(?<=@)([0-9a-zA-Z]+)(?=\.)'
 
+# 抽取邮箱的前缀，一般为：`email: [email protected]`，用于删除前缀
+EMAIL_PREFIX_PATTERN = r'(([eE](\-|—)?(mail|MAIL)|(电子)?邮箱)(:|：)?[\t \u3000]*)' \
+                       r'(?=[^:： \t\u3000])'
+
 # ---------------------------------------------------------------------
 # 转义符号
 ESCAPE_CHAR_PATTERN = '\t\n\a\b\f\r\v'
@@ -291,7 +300,7 @@
 # URL
 URL_PATTERN = r'(?<=[^.])((?:(?:https?|ftp|file)://|(?<![a-zA-Z\-\.])www\.)' \
               r'[\-A-Za-z0-9\+&@\(\)#/%\?=\~_|!:\,\.\;]+[\-A-Za-z0-9\+&@#/%=\~_\|])' \
-              r'(?=[<\u4E00-\u9FA5￥，。；！？、“”‘’>（）—《》…● ])'
+              r'(?=[<\u4E00-\u9FA5￥，。；！？、“”‘’>（）—《》…● \t\n])'
 
 
 #######################################################################

test/test_main.py

@@ -8,7 +8,9 @@
 from test_money_parser import TestMoneyParser
 from test_time_extractor import TestTimeExtractor
 from test_money_extractor import TestMoneyExtractor
-from text_remove_url import TestRemoveUrl
+from test_remove_url import TestRemoveUrl
+from test_remove_email import TestRemoveEmail
+from test_remove_phone_number import TestRemovePhoneNumber
 
 
 if __name__ == '__main__':
@@ -23,8 +25,11 @@
         TestMoneyParser('test_money_parser'),  # 测试 金额抽取与规范化
         TestTimeExtractor('test_time_extractor'),  # 测试 时间实体抽取
         TestMoneyExtractor('test_money_extractor'),  # 测试 货币金额实体抽取
-        TestRemoveUrl('test_remove_url')  # 测试 清洗文本中的超链接
+        TestRemoveUrl('test_remove_url'),  # 测试 清洗文本中的超链接
+        TestRemoveEmail('test_remove_email'),  # 测试 清洗文本中的 email
+        TestRemovePhoneNumber('test_remove_phone_number')  # 测试 清洗文本中的电话号码
     ]
+
     suite.addTests(tests)
 
     runner = unittest.TextTestRunner(verbosity=1)

test/test_remove_email.py

@@ -0,0 +1,33 @@
+# -*- coding=utf-8 -*-
+
+import unittest
+
+import jionlp as jio
+
+
+class TestRemoveEmail(unittest.TestCase):
+    """ 测试清除 email 工具 """
+
+    def test_remove_email(self):
+        """ test func remove_email """
+
+        email_text_list = [
+            ['Beihang University E-mail 给她打电话啊 Email:  [email protected] ， 中国[email protected]。',
+             'Beihang University E-mail 给她打电话啊  ， 中国。'],
+        ]
+
+        for item in email_text_list:
+            clean_text = jio.remove_email(item[0], delete_prefix=True)
+            print(item[0])
+            self.assertEqual(clean_text, item[1])
+
+
+if __name__ == '__main__':
+
+    suite = unittest.TestSuite()
+    test_remove_email = [TestRemoveEmail('test_remove_email')]
+    suite.addTests(test_remove_email)
+
+    runner = unittest.TextTestRunner(verbosity=1)
+    runner.run(suite)
+

test/test_remove_phone_number.py

@@ -0,0 +1,33 @@
+# -*- coding=utf-8 -*-
+
+import unittest
+
+import jionlp as jio
+
+
+class TestRemovePhoneNumber(unittest.TestCase):
+    """ 测试清除 phone_number 工具 """
+
+    def test_remove_phone_number(self):
+        """ test func remove_phone_number """
+
+        phone_number_text_list = [
+            [' 电话：(010)37283893 他手机号多少？18702812943. 还有一个是17209374283    [email protected]联系电话： （0351）89082910',
+             '  他手机号多少？. 还有一个是    [email protected]'],
+        ]
+
+        for item in phone_number_text_list:
+            clean_text = jio.remove_phone_number(item[0], delete_prefix=True)
+            print(item[0])
+            self.assertEqual(clean_text, item[1])
+
+
+if __name__ == '__main__':
+
+    suite = unittest.TestSuite()
+    test_remove_phone_number = [TestRemovePhoneNumber('test_remove_phone_number')]
+    suite.addTests(test_remove_phone_number)
+
+    runner = unittest.TextTestRunner(verbosity=1)
+    runner.run(suite)
+