optimize parse data for lossy telemetry data

Author: MCLiii

Backend/core/comms.py

@@ -13,6 +13,7 @@
 from multiprocessing.managers import BaseManager
 from . import db
 from file_sync.file_sync_down.main import *
+import re
 
 format_string = '<' # little-endian
 byte_length = 0
@@ -241,35 +242,40 @@ async def remote_db_fetch(self, server_url: str):
 
     def parse_packets(self, new_data: bytes, tmp_source: str):
         """
-        Parse and check the length of each packet
-        :param new_data: Newly received bytes from the comm channel
-        :param tmp_source: Name of tmp data source, put comm channel name here e.g. tcp, lte
+        Parse and check the length of each packet.
+        
+        :param new_data: Newly received bytes from the comm channel.
+        :param tmp_source: Name of tmp data source, put comm channel name here e.g. tcp, lte.
         """
-        header = b'<bsr>'
-        footer = b'</bsr>'
+        header = b"<bsr>"
+        footer = b"<bsr"
+        if tmp_source not in self.__tmp_data:
+            self.__tmp_data[tmp_source] = b''
+
+        # Append new data to the temporary buffer
         self.__tmp_data[tmp_source] += new_data
+
+        # Regex pattern to match packets with <bsr> and </bsr> tags
+        pattern = re.compile(b'<bsr>(.*?)</bsr>', re.DOTALL)
+
         packets = []
         while True:
-            # Search for the next complete data packet
-            try:
-                start_index = self.__tmp_data[tmp_source].index(header)
-                end_index = self.__tmp_data[tmp_source].index(footer)
-            except ValueError:
+            match = pattern.search(self.__tmp_data[tmp_source])
+            if not match:
                 break
+            # Extract the packet data
+            packet = match.group(1)
+            #remove headers and footers
+            packets.append(packet)
 
-            # Extract a complete data packet
-            packets.append(self.__tmp_data[tmp_source][start_index + len(header):end_index])
-            # Update the remaining data to exclude the processed packet
-            self.__tmp_data[tmp_source] = self.__tmp_data[tmp_source][end_index + len(footer):]
-
-        # If the remaining data is longer than the expected packet length,
-        # there might be an incomplete packet, so log a warning.
-        if len(self.__tmp_data[tmp_source]) >= byte_length:
-            print(f"Source: {tmp_source}: Warning: Incomplete or malformed packet ------------------------------------")
-            self.__tmp_data[tmp_source] = b''
+            if match.start(0) != 0:
+                print(f"skipping {match.start(0)} bytes")
+            # Remove the processed packet from the temporary buffer
+            self.__tmp_data[tmp_source] = self.__tmp_data[tmp_source][match.end():]
 
         return packets
 
+
     def fs_down_callback(self, data):
         # copied from listen_upd()
         if not data:
@@ -301,11 +307,11 @@ def sigint_handler(signal, frame):
 
 def start_comms():
     # start file sync
-    p.start()
+    # p.start()
 
     # Start two live comm channels
-    vps_thread = threading.Thread(target=lambda : asyncio.run(telemetry.remote_db_fetch(config.VPS_URL)))
-    vps_thread.start()
+    #vps_thread = threading.Thread(target=lambda : asyncio.run(telemetry.remote_db_fetch(config.VPS_URL)))
+    #vps_thread.start()
     #socket_thread = threading.Thread(target=lambda: telemetry.listen_udp(config.UDP_PORT))
     #socket_thread.start()
     socket_thread = threading.Thread(target=lambda: telemetry.serial_read())