Merge pull request #62 from VIP-LES/59-science-driver-format

#59 science format first pass

CONTRIBUTORS.md

@@ -8,4 +8,6 @@ Panya Bhinder
 
 Ion Li
 
-Caroline Kerr
+Caroline Kerr
+
+Daryl Dohner

EosLib/format/__init__.py

@@ -1,4 +1,3 @@
-from EosLib.format.formats import telemetry_data, position, empty_format, cutdown, ping_format, valve, e_field
-
-
+from EosLib.format.formats import telemetry_data, position, empty_format, cutdown, ping_format, valve, e_field, \
+                                  science_data
 from EosLib.format.definitions import Type as Type

EosLib/format/base_format.py

@@ -9,9 +9,18 @@
 
 class BaseFormat(ABC):
     def __init_subclass__(cls, **kwargs):
-        if not inspect.isabstract(cls):
+        if not inspect.isabstract(cls) or cls._i_promise_all_abstract_methods_are_implemented():
             decode_factory.register_decoder(cls)
 
+    @staticmethod
+    def _i_promise_all_abstract_methods_are_implemented() -> bool:
+        """ This method exists because if you use a dataclass-based format ABC gets mad because it can't figure out
+            that the dataclass implements __init__, __eq__, etc.
+
+            :return: True if isabstract check should be bypassed, otherwise False
+        """
+        return False
+
     @classmethod
     def get_decoders(cls) -> dict[type, callable]:
         return {bytes: cls.decode}

EosLib/format/definitions.py

@@ -16,4 +16,5 @@ class Type(IntEnum):
     PING = 10
     VALVE = 11
     E_FIELD = 12
+    SCIENCE_DATA = 13
     ERROR = 255

EosLib/format/formats/science_data.py

@@ -0,0 +1,178 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+import csv
+import io
+import struct
+from typing_extensions import Self
+
+from EosLib.format.definitions import Type
+from EosLib.format.csv_format import CsvFormat
+
+
+@dataclass
+class ScienceData(CsvFormat):
+    """
+    This is the format for representing the output of the science sensors driver.
+    Turns out it's chonk, ~104 bytes.  If more fields are added, maybe consider breaking up the packets so we don't
+    hit radio max bytes
+    """
+
+    # note: all _count variables are unitless.  Some things can be calculated by referencing the data
+    #       sheets though.  But otherwise consider it a relative measure of comparison.
+    temperature_celsius: float         # from SHTC3 temperature-humidity sensor        (double)
+    relative_humidity_percent: float   # from SHTC3 temperature-humidity sensor        (double)
+    temperature_celsius_2: float       # from BMP388 temperature-pressure sensor       (double)
+    pressure_hpa: float                # from BMP388 temperature-pressure sensor       (double)
+    altitude_meters: float             # from BMP388 temperature-pressure sensor       (double)
+    ambient_light_count: int           # from LTR390 uv-light sensor                   (uint?)
+    ambient_light_lux: float           # from LTR390 uv-light sensor                   (double)
+    uv_count: int                      # from LTR390 uv-light sensor                   (uint?)
+    uv_index: float                    # from LTR390 uv-light sensor                   (double)
+    infrared_count: int                # from TSL2591 ir-light sensor                  (ushort)
+    visible_count: int                 # from TSL2591 ir-light sensor                  (uint)
+    full_spectrum_count: int           # from TSL2591 ir-light sensor                  (uint)
+    ir_visible_lux: float              # from TSL2591 ir-light sensor                  (double)
+    pm10_standard_ug_m3: int           # from PMSA003I particulate sensor              (ushort)
+    pm25_standard_ug_m3: int           # from PMSA003I particulate sensor              (ushort)
+    pm100_standard_ug_m3: int          # from PMSA003I particulate sensor              (ushort)
+    pm10_environmental_ug_m3: int      # from PMSA003I particulate sensor              (ushort)
+    pm25_environmental_ug_m3: int      # from PMSA003I particulate sensor              (ushort)
+    pm100_environmental_ug_m3: int     # from PMSA003I particulate sensor              (ushort)
+    particulate_03um_per_01L: int      # from PMSA003I particulate sensor              (ushort)
+    particulate_05um_per_01L: int      # from PMSA003I particulate sensor              (ushort)
+    particulate_10um_per_01L: int      # from PMSA003I particulate sensor              (ushort)
+    particulate_25um_per_01L: int      # from PMSA003I particulate sensor              (ushort)
+    particulate_50um_per_01L: int      # from PMSA003I particulate sensor              (ushort)
+    particulate_100um_per_01L: int     # from PMSA003I particulate sensor              (ushort)
+
+    # useful for csv, not included in binary encode/decode, use data header instead.  Not compared in __eq__
+    timestamp: datetime | None = field(default=None, compare=False)
+
+    @staticmethod
+    def _i_promise_all_abstract_methods_are_implemented() -> bool:
+        return True
+
+    @staticmethod
+    def get_format_type() -> Type:
+        return Type.SCIENCE_DATA
+
+    @staticmethod
+    def get_format_string() -> str:
+        #         SHTC3 BMP388 LTR390 TSL2591 PMSA003I = 104 bytes
+        return "!  2d     3d    IdId   HIId     12H   "
+
+    def get_csv_headers(self):
+        return [
+            'timestamp', 'temperature_celsius', 'relative_humidity_percent', 'temperature_celsius_2', 'pressure_hpa',
+            'altitude_meters', 'ambient_light_count', 'ambient_light_lux', 'uv_count', 'uv_index', 'infrared_count',
+            'visible_count', 'full_spectrum_count', 'ir_visible_lux', 'pm10_standard_ug_m3', 'pm25_standard_ug_m3',
+            'pm100_standard_ug_m3', 'pm10_environmental_ug_m3', 'pm25_environmental_ug_m3', 'pm100_environmental_ug_m3',
+            'particulate_03um_per_01L', 'particulate_05um_per_01L', 'particulate_10um_per_01L',
+            'particulate_25um_per_01L', 'particulate_50um_per_01L', 'particulate_100um_per_01L'
+        ]
+
+    def encode(self) -> bytes:
+        return struct.pack(
+            self.get_format_string(),
+            self.temperature_celsius,
+            self.relative_humidity_percent,
+            self.temperature_celsius_2,
+            self.pressure_hpa,
+            self.altitude_meters,
+            self.ambient_light_count,
+            self.ambient_light_lux,
+            self.uv_count,
+            self.uv_index,
+            self.infrared_count,
+            self.visible_count,
+            self.full_spectrum_count,
+            self.ir_visible_lux,
+            self.pm10_standard_ug_m3,
+            self.pm25_standard_ug_m3,
+            self.pm100_standard_ug_m3,
+            self.pm10_environmental_ug_m3,
+            self.pm25_environmental_ug_m3,
+            self.pm100_environmental_ug_m3,
+            self.particulate_03um_per_01L,
+            self.particulate_05um_per_01L,
+            self.particulate_10um_per_01L,
+            self.particulate_25um_per_01L,
+            self.particulate_50um_per_01L,
+            self.particulate_100um_per_01L,
+        )
+
+    @classmethod
+    def decode(cls, data: bytes) -> Self:
+        unpacked_data = struct.unpack(cls.get_format_string(), data)
+        return ScienceData(*unpacked_data)
+
+    def encode_to_csv(self) -> str:
+        output = io.StringIO()
+        writer = csv.writer(output)
+        writer.writerow([
+            self.timestamp.isoformat() if self.timestamp is not None else None,
+            str(self.temperature_celsius),
+            str(self.relative_humidity_percent),
+            str(self.temperature_celsius_2),
+            str(self.pressure_hpa),
+            str(self.altitude_meters),
+            str(self.ambient_light_count),
+            str(self.ambient_light_lux),
+            str(self.uv_count),
+            str(self.uv_index),
+            str(self.infrared_count),
+            str(self.visible_count),
+            str(self.full_spectrum_count),
+            str(self.ir_visible_lux),
+            str(self.pm10_standard_ug_m3),
+            str(self.pm25_standard_ug_m3),
+            str(self.pm100_standard_ug_m3),
+            str(self.pm10_environmental_ug_m3),
+            str(self.pm25_environmental_ug_m3),
+            str(self.pm100_environmental_ug_m3),
+            str(self.particulate_03um_per_01L),
+            str(self.particulate_05um_per_01L),
+            str(self.particulate_10um_per_01L),
+            str(self.particulate_25um_per_01L),
+            str(self.particulate_50um_per_01L),
+            str(self.particulate_100um_per_01L),
+        ])
+
+        return output.getvalue()
+
+    @classmethod
+    def decode_from_csv(cls, csv_string: str) -> Self:
+        reader = csv.reader([csv_string])
+        csv_list = list(reader)[0]
+        ret = ScienceData(
+            float(csv_list[1]),
+            float(csv_list[2]),
+            float(csv_list[3]),
+            float(csv_list[4]),
+            float(csv_list[5]),
+            int(csv_list[6]),
+            float(csv_list[7]),
+            int(csv_list[8]),
+            float(csv_list[9]),
+            int(csv_list[10]),
+            int(csv_list[11]),
+            int(csv_list[12]),
+            float(csv_list[13]),
+            int(csv_list[14]),
+            int(csv_list[15]),
+            int(csv_list[16]),
+            int(csv_list[17]),
+            int(csv_list[18]),
+            int(csv_list[19]),
+            int(csv_list[20]),
+            int(csv_list[21]),
+            int(csv_list[22]),
+            int(csv_list[23]),
+            int(csv_list[24]),
+            int(csv_list[25]),
+            datetime.fromisoformat(csv_list[0]) if csv_list[0] is not None else None
+        )
+        return ret
+
+    def get_validity(self) -> bool:
+        return True  # i think this is a case where if one parameter is invalid

setup.py

@@ -4,7 +4,7 @@
 from setuptools import find_packages
 
 setup(name='EosLib',
-      version='4.3.5',
+      version='4.4.0',
       description='Library of shared code between EosPayload and EosGround',
       author='Lightning From The Edge of Space',
       author_email='[email protected]',

tests/format/formats/format_test.py

@@ -1,27 +1,29 @@
+from typing import Type
 import abc
 import copy
 import datetime
 
+from EosLib.format.base_format import BaseFormat
 from EosLib.format.decode_factory import decode_factory
 
 
 # Name can't contain the word Test, hence Check
 class CheckFormat(abc.ABC):
 
     @abc.abstractmethod
-    def get_format(self):
+    def get_format_class(self) -> Type[BaseFormat]:
         raise NotImplementedError
 
     @abc.abstractmethod
-    def get_good_format_params(self):
+    def get_good_format_params(self) -> list:
         """Provides a list of parameters that can be used by get_format_from_list to generate a valid instance of
         the format. Used to automate validating __eq__()"""
         raise NotImplementedError
 
-    def get_good_format(self):
+    def get_good_format(self) -> BaseFormat:
         """Returns a valid instance of the format being tested. This is a helper function combining get_good_format_list
         and get_format_from_list."""
-        return self.get_format()(*self.get_good_format_params())
+        return self.get_format_class()(*self.get_good_format_params())
 
     def test_is_eq(self):
         data_1 = self.get_good_format()
@@ -30,8 +32,6 @@ def test_is_eq(self):
         assert data_1 == data_2
 
     def test_not_eq(self):
-        test_passed = True
-
         data_1 = self.get_good_format()
 
         # Iterates over each parameter given in get_good_format_list, creating a new instance of the format with that
@@ -45,12 +45,9 @@ def test_not_eq(self):
             elif isinstance(new_data_list[i], datetime.datetime):
                 new_data_list[i] += datetime.timedelta(1)
 
-            data_2 = self.get_format()(*new_data_list)
-
-            if data_1 == data_2:
-                test_passed = False
+            data_2 = self.get_format_class()(*new_data_list)
 
-        assert test_passed
+            assert data_1 != data_2
 
     def test_encode_decode_bytes(self):
         base_format = self.get_good_format()

tests/format/formats/test_cutdown.py

@@ -38,7 +38,7 @@ def test_terminal_output_string():
 
 class TestCutDown(CheckFormat):
 
-    def get_format(self):
+    def get_format_class(self):
         return CutDown
 
     def get_good_format_params(self):

tests/format/formats/test_e_field.py

@@ -31,7 +31,7 @@ def test_get_validity_bad_voltages(bad_voltage):
 
 class TestEField(CheckCsvFormat):
 
-    def get_format(self):
+    def get_format_class(self):
         return EField
 
     def get_good_format_params(self):

tests/format/formats/test_empty_format.py

@@ -12,7 +12,7 @@ def test_empty_format_length(test_size):
 
 
 class TestEmptyFormat(CheckFormat):
-    def get_format(self):
+    def get_format_class(self):
         return EmptyFormat
 
     def get_good_format_params(self):

tests/format/formats/test_ping_format.py

@@ -46,7 +46,7 @@ def test_terminal_output_string_ack():
 
 class TestPing(CheckFormat):
 
-    def get_format(self):
+    def get_format_class(self):
         return Ping
 
     def get_good_format_params(self):

tests/format/formats/test_position.py

@@ -41,7 +41,7 @@ def test_get_validity_bad_satellites():
 
 class TestPosition(CheckCsvFormat):
 
-    def get_format(self):
+    def get_format_class(self):
         return Position
 
     def get_good_format_params(self):

tests/format/formats/test_science_format.py

@@ -0,0 +1,65 @@
+from datetime import datetime, timedelta
+from typing import Type
+import copy
+
+from EosLib.format.base_format import BaseFormat
+from EosLib.format.formats.science_data import ScienceData
+from tests.format.formats.csv_format_test import CheckCsvFormat
+
+
+good_format_params = [
+    26.38,              # temperature
+    39.72,              # relative_humidity
+    26.99,              # temperature_2
+    981.3804861416284,  # pressure
+    268.7146768726267,  # altitude
+    579,                # light
+    463.2,              # uv lux
+    0,                  # uvs
+    0.0,                # uvi
+    376,                # infrared
+    24643214,           # visible
+    24643590,           # full_spectrum
+    463.9,              # ir lux
+    11,                 # pm10_standard
+    24,                 # pm25_standard
+    26,                 # pm100_standard
+    11,                 # pm10_env
+    24,                 # pm25_env
+    26,                 # pm100_env
+    1815,               # part_03
+    567,                # part_05
+    191,                # part_10
+    13,                 # part_25
+    3,                  # part_50
+    3,                  # part_100
+    datetime.fromisoformat("2023-06-29T04:02:34.294887"),  # timestamp, clearly
+]
+
+
+class TestScienceFormat(CheckCsvFormat):
+
+    def get_format_class(self) -> Type[BaseFormat]:
+        return ScienceData
+
+    def get_good_format_params(self) -> list:
+        return good_format_params
+
+    # overriding to exclude timestamp from comparison, it is not used in the binary encoding
+    def test_not_eq(self):
+        data_1 = self.get_good_format()
+
+        # Iterates over each parameter given in get_good_format_list, creating a new instance of the format with that
+        # parameter modified. It then verifies that this modification causes the instances to evaluate as not equal.
+        for i in range(len(self.get_good_format_params()) - 1):  # exclude timestamp
+            new_data_list = copy.deepcopy(self.get_good_format_params())
+
+            # Modifies current parameter
+            if isinstance(new_data_list[i], (int, float)):
+                new_data_list[i] += 1
+            elif isinstance(new_data_list[i], datetime):
+                new_data_list[i] += timedelta(1)
+
+            data_2 = self.get_format_class()(*new_data_list)
+
+            assert data_1 != data_2