Updated library

Author: davidmezzetti

setup.py

@@ -28,7 +28,7 @@
           ],
       },
       install_requires=[
-          "html2text>=2020.1.16",
+          "html2markdown>=0.1.7",
           "mdv>=1.7.4",
           "tqdm==4.48.0",
           "txtai>=2.0.0"

src/python/codequestion/query.py

@@ -5,9 +5,10 @@
 import argparse
 import os
 import os.path
+import re
 import sqlite3
 
-import html2text
+import html2markdown
 import mdv
 
 from txtai.embeddings import Embeddings
@@ -20,6 +21,27 @@ class Query(object):
     Methods to query an embeddings index.
     """
 
+    @staticmethod
+    def markdown(text):
+        """
+        Converts html text to markdown.
+
+        Args:
+            text: html text
+
+        Returns:
+            text as markdown
+        """
+
+        # Remove rel attributes as they are not supported by html2markdown
+        text = re.sub(r' rel=".+?">', ">", text)
+
+        # Convert html to markdown
+        text = html2markdown.convert(text)
+
+        # Decode [<>&] characters
+        return text.replace("&lt;", "<").replace("&gt;", ">").replace("&amp;", "&")
+
     @staticmethod
     def escape(text):
         """
@@ -69,11 +91,8 @@ def render(text, html=True, tab_length=0):
         """
 
         if html:
-            # Convert HTML
-            parser = html2text.HTML2Text()
-            parser.body_width = 0
-            text = parser.handle(text)
-
+            # Convert html to markdown
+            text = Query.markdown(text)
             text = Query.escape(text)
 
         text = mdv.main(text, theme="592.2129", c_theme="953.3567", cols=180, tab_length=tab_length)