Improve test coverage

chatminer/chatparsers.py

@@ -120,7 +120,7 @@ def _parse_message(self, mess: Any) -> Optional[ParsedMessage]:
 class SignalParser(Parser):
     def _read_raw_messages_from_file(self):
         def _is_new_message(line: str):
-            regex = r"^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}\]"
+            regex = r"^\[\d{4}-\d{2}-\d{2}, \d{2}:\d{2}\]"
             return re.match(regex, line)
 
         with self._file.open(encoding="utf-8") as f:
@@ -136,7 +136,8 @@ def _is_new_message(line: str):
                 if buffer:
                     buffer.append(line)
                     buffer.reverse()
-                    self._raw_messages.append(" ".join(buffer))
+                    joined_buffer = " ".join(buffer)
+                    self._raw_messages.append("".join(joined_buffer.splitlines()))
                     buffer.clear()
                 else:
                     self._raw_messages.append(line)
@@ -221,7 +222,7 @@ def _parse_message(self, mess: Dict[str, Any]):
             self._logger.warning("Skipped message with unknown format: %s", mess)
             return None
 
-        time = dt.datetime.fromtimestamp(mess["timestamp_ms"] / 1000)
+        time = dt.datetime.utcfromtimestamp(mess["timestamp_ms"] / 1000)
         author = mess["sender_name"].encode("latin-1").decode("utf-8")
         body = body.encode("latin-1").decode("utf-8")
         return ParsedMessage(time, author, body)
@@ -263,7 +264,7 @@ def _parse_message(self, mess: Dict[str, Any]):
             self._logger.warning("Skipped message with unknown format: %s", mess)
             return None
 
-        time = dt.datetime.fromtimestamp(mess["timestamp_ms"] / 1000)
+        time = dt.datetime.utcfromtimestamp(mess["timestamp_ms"] / 1000)
         author = mess["sender_name"].encode("latin-1").decode("utf-8")
         body = body.encode("latin-1").decode("utf-8")
         return ParsedMessage(time, author, body)
@@ -313,7 +314,7 @@ def _parse_message(self, mess: Dict[str, Any]):
             else:
                 raise ValueError(f"Unable to parse type {type(mess['text'])} in {mess}")
 
-            time = dt.datetime.fromtimestamp(int(mess["date_unixtime"]))
+            time = dt.datetime.utcfromtimestamp(int(mess["date_unixtime"]))
             author = mess["from"]
             return ParsedMessage(time, author, body)
         return None
@@ -393,12 +394,8 @@ def _log_resulting_format(self):
         end = "]" if self.has_brackets else ""
         if self.is_yearfirst:
             date1 = "year"
-            if self.is_dayfirst:
-                date2 = "day"
-                date3 = "month"
-            else:
-                date2 = "month"
-                date3 = "day"
+            date2 = "month"
+            date3 = "day"
         elif self.is_dayfirst:
             date1 = "day"
             date2 = "month"

setup.cfg

@@ -45,6 +45,7 @@ NLP =
 
 [coverage:run]
 omit =
+    chatminer/cli.py
     chatminer/nlp.py
     chatminer/visualizations.py
     chatminer/__init__.py

test/instagram/target.json

@@ -0,0 +1,12 @@
+[
+    {
+        "timestamp": "2020-01-17T20:49:00",
+        "author": "John Doe",
+        "message": "Hello Jane!"
+    },
+    {
+        "timestamp": "2019-12-24T11:23:00",
+        "author": "Jane Doe",
+        "message": "Hello John!"
+    }
+]

test/instagram/test_export.json

@@ -0,0 +1,14 @@
+{
+  "messages": [
+    {
+      "sender_name": "John Doe",
+      "timestamp_ms": 1579294140000,
+      "content": "Hello Jane!"
+    },
+    {
+      "sender_name": "Jane Doe",
+      "timestamp_ms": 1577186580000,
+      "content": "Hello John!"
+    }
+  ]
+}

test/signal/target.json

@@ -0,0 +1,12 @@
+[
+    {
+        "timestamp": "2020-01-17T21:49:00",
+        "author": "John Doe",
+        "message": "Hello Jane!"
+    },
+    {
+        "timestamp": "2019-12-24T12:23:00",
+        "author": "Jane Doe",
+        "message": "Hello John!"
+    }
+]

test/signal/test_export.txt

@@ -0,0 +1,3 @@
+[2019-12-24, 12:23] Jane Doe: Hello John!
+[2020-01-17, 21:49] John Doe: Hello
+Jane!

test/telegram/target.json

@@ -0,0 +1,12 @@
+[
+    {
+        "timestamp": "2018-12-05T10:39:10",
+        "author": "Author 1",
+        "message": "Minimal Message"
+    },
+    {
+        "timestamp": "2018-12-05T10:49:24",
+        "author": "Author 2",
+        "message": "Link: https://telegram.org/"
+    }
+]

test/telegram/test_batch_export.json

@@ -0,0 +1,32 @@
+{
+  "chats": {
+    "list": [
+      {
+        "name": "Chatname",
+        "messages": [
+          {
+           "type": "message",
+           "date_unixtime": "1544006350",
+           "from": "Author 1",
+           "text": "Minimal Message"
+          },
+          {
+           "type": "message",
+           "date_unixtime": "1544006964",
+           "from": "Author 2",
+           "text": [
+             {
+              "type": "bold",
+              "text": "Link:"
+             },
+             {
+             "type": "link",
+             "text": "https://telegram.org/"
+             }
+            ]
+          }
+         ]
+      }
+    ]
+  }
+}

test/telegram/test_single_export.json

@@ -0,0 +1,25 @@
+{
+    "messages": [
+     {
+      "type": "message",
+      "date_unixtime": "1544006350",
+      "from": "Author 1",
+      "text": "Minimal Message"
+     },
+     {
+      "type": "message",
+      "date_unixtime": "1544006964",
+      "from": "Author 2",
+      "text": [
+        {
+         "type": "bold",
+         "text": "Link:"
+        },
+        {
+        "type": "link",
+        "text": "https://telegram.org/"
+        }
+       ]
+     }
+    ]
+}

test/test_instagram.py

@@ -1,18 +1,9 @@
-import pandas as pd
-from pandas.testing import assert_frame_equal
-
-from chatminer.chatparsers import InstagramJsonParser
+from chatminer.chatparsers import InstagramJsonParser, ParsedMessageCollection
 
 
 def test_instagram():
-    parser = InstagramJsonParser("test/instagram/testlog.json")
+    target = ParsedMessageCollection()
+    target.read_from_json("test/instagram/target.json")
+    parser = InstagramJsonParser("test/instagram/test_export.json")
     parser.parse_file()
-    df_res = parser.parsed_messages.get_df()
-    df_test = pd.read_csv(
-        "test/instagram/testlog_target.csv",
-        parse_dates=["timestamp"],
-    )
-    assert_frame_equal(
-        df_test[["author", "message", "words", "letters"]],
-        df_res[["author", "message", "words", "letters"]],
-    )
+    assert parser.parsed_messages == target

test/test_signal.py

@@ -0,0 +1,9 @@
+from chatminer.chatparsers import ParsedMessageCollection, SignalParser
+
+
+def test_signal():
+    target = ParsedMessageCollection()
+    target.read_from_json("test/signal/target.json")
+    parser = SignalParser("test/signal/test_export.txt")
+    parser.parse_file()
+    assert parser.parsed_messages == target