diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index cd895d5b3..40923dd62 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -45,10 +45,12 @@ jobs: - name: Install Python Dependencies run: | python -m pip install --upgrade pip - python -m pip install -r requirements_develop.txt + python -m pip install build + python -m pip install -r requirements_tests.txt - name: Check black format - run: black --check --diff . + run: | + black --check --diff . - name: Install Doxygen run: sudo apt-get install doxygen graphviz @@ -88,10 +90,14 @@ jobs: - name: Prepare C++ Build run: mkdir build - # Versioning - - name: Get versioning + - name: Add Development Version Suffix + if: ${{ !startsWith(github.ref, 'refs/tags') }} + run: | + echo "VERSION_SUFFIX = .dev`date -u '+%Y%m%d%H%M%S'`" >> VERSION + + - name: Get git Version id: get_version - run: echo "VERSION=$(git describe --always)" >> $GITHUB_OUTPUT + run: echo "VERSION=$(git describe --tags --always)" >> $GITHUB_OUTPUT - name: Prepare Documentation Build run: | @@ -108,7 +114,7 @@ jobs: run: cmake --build . --config Release -j 4 - name: Build Python - run: python setup.py build && python setup.py sdist + run: python -m build - name: Install Python run: python -m pip install . @@ -124,7 +130,14 @@ jobs: path: doc/html if-no-files-found: error - - name: deploy to gh-pages if push to master branch + - name: Upload Python Distribution + if: ${{ github.event_name == 'pull_request' }} + uses: actions/upload-artifact@v4 + with: + name: python-dist + path: dist/ + + - name: Deploy to gh-pages if push to master branch if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: peaceiris/actions-gh-pages@v3 with: @@ -148,7 +161,10 @@ jobs: python-version: '3.8' - name: Install Python Dependencies - run: python -m pip install --upgrade pip setuptools wheel pyyaml + run: | + python -m pip install --upgrade pip + python -m pip install build + python -m pip install -r requirements_tests.txt - name: Cache Dependencies id: cache-depends @@ -187,6 +203,11 @@ jobs: bash convert-to-proto3.sh rm *.pb2 + - name: Add Development Version Suffix + if: ${{ !startsWith(github.ref, 'refs/tags') }} + run: | + echo "VERSION_SUFFIX = .dev`date -u '+%Y%m%d%H%M%S'`" >> VERSION + - name: Configure C++ Build working-directory: build run: cmake ${{ env.PROTOBUF_VARIANT =='' && '-DCMAKE_CXX_STANDARD=17' }} .. @@ -196,7 +217,7 @@ jobs: run: cmake --build . --config Release -j 4 - name: Build Python - run: python setup.py build && python setup.py sdist + run: python -m build - name: Install Python run: python -m pip install . diff --git a/.gitignore b/.gitignore index 2076b7c5d..e29f907d0 100644 --- a/.gitignore +++ b/.gitignore @@ -20,8 +20,6 @@ build cmake_install.cmake install_manifest.txt osi_version.proto -version.py -pyproject.toml compile_commands.json diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..ceeea233f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include VERSION diff --git a/format/OSITrace.py b/format/OSITrace.py deleted file mode 100644 index cc92f06c7..000000000 --- a/format/OSITrace.py +++ /dev/null @@ -1,174 +0,0 @@ -""" -Module to handle and manage OSI scenarios. -""" -from collections import deque -import time -import lzma -import struct - -from osi3.osi_sensorview_pb2 import SensorView -from osi3.osi_groundtruth_pb2 import GroundTruth -from osi3.osi_sensordata_pb2 import SensorData -import warnings - -warnings.simplefilter("default") - -BUFFER_SIZE = 1000000 - - -def get_size_from_file_stream(file_object): - """ - Return a file size from a file stream given in parameters - """ - current_position = file_object.tell() - file_object.seek(0, 2) - size = file_object.tell() - file_object.seek(current_position) - return size - - -MESSAGES_TYPE = { - "SensorView": SensorView, - "GroundTruth": GroundTruth, - "SensorData": SensorData, -} - - -class OSITrace: - """This class wrap OSI data. It can import and decode OSI scenarios.""" - - def __init__(self, path=None, type_name="SensorView"): - self.scenario_file = None - self.message_offsets = None - self.type_name = type_name - self.timestep_count = 0 - self.retrieved_scenario_size = 0 - self._int_length = len(struct.pack(" 1000000000: - # Throw a warning if trace file is bigger than 1GB - gb_size_input = round(message_length / 1000000000, 2) - gb_size_output = round(3.307692308 * message_length / 1000000000, 2) - warnings.warn( - f"The trace file you are trying to make readable has the size {gb_size_input}GB. This will generate a readable file with the size {gb_size_output}GB. Make sure you have enough disc space and memory to read the file with your text editor.", - ResourceWarning, - ) - - with open(name, "a") as f: - if interval is None and index is None: - for i in self.get_messages(): - f.write(str(i)) - - if interval is not None and index is None: - if ( - type(interval) == tuple - and len(interval) == 2 - and interval[0] < interval[1] - ): - for i in self.get_messages_in_index_range(interval[0], interval[1]): - f.write(str(i)) - else: - raise Exception( - "Argument 'interval' needs to be a tuple of length 2! The first number must be smaller then the second." - ) - - if interval is None and index is not None: - if type(index) == int: - f.write(str(self.get_message_by_index(0))) - else: - raise Exception("Argument 'index' needs to be of type 'int'") - - if interval is not None and index is not None: - raise Exception("Arguments 'index' and 'interval' can not be set both") diff --git a/format/osi2read.py b/osi3trace/osi2read.py similarity index 63% rename from format/osi2read.py rename to osi3trace/osi2read.py index 240507177..c6bee861e 100644 --- a/format/osi2read.py +++ b/osi3trace/osi2read.py @@ -5,24 +5,24 @@ python3 osi2read.py -d trace.osi -o myreadableosifile """ -from OSITrace import OSITrace -import struct -import lzma +from osi3trace.osi_trace import OSITrace import argparse -import os +import pathlib def command_line_arguments(): """Define and handle command line interface""" - dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - parser = argparse.ArgumentParser( description="Convert a serialized osi trace file to a readable txth output.", - prog="osi2read converter", + prog="osi2read", ) parser.add_argument( - "--data", "-d", help="Path to the file with serialized data.", type=str + "--data", + "-d", + help="Path to the file with serialized data.", + type=str, + required=True, ) parser.add_argument( "--type", @@ -37,7 +37,6 @@ def command_line_arguments(): "--output", "-o", help="Output name of the file.", - default="converted.txth", type=str, required=False, ) @@ -50,16 +49,17 @@ def main(): args = command_line_arguments() # Initialize the OSI trace class - trace = OSITrace() - trace.from_file(path=args.data, type_name=args.type) + trace = OSITrace(args.data, args.type) - args.output = args.output.split(".", 1)[0] + ".txth" + if not args.output: + path = pathlib.Path(args.data).with_suffix(".txth") + args.output = str(path) - if args.output == "converted.txth": - args.output = args.data.split(".", 1)[0] + ".txth" + with open(args.output, "wt") as f: + for message in trace: + f.write(str(message)) - trace.make_readable(args.output) - trace.scenario_file.close() + trace.close() if __name__ == "__main__": diff --git a/osi3trace/osi_trace.py b/osi3trace/osi_trace.py new file mode 100644 index 000000000..49453c660 --- /dev/null +++ b/osi3trace/osi_trace.py @@ -0,0 +1,168 @@ +""" +Module to handle and manage OSI trace files. +""" +import lzma +import struct + +from osi3.osi_sensorview_pb2 import SensorView +from osi3.osi_groundtruth_pb2 import GroundTruth +from osi3.osi_sensordata_pb2 import SensorData + + +MESSAGES_TYPE = { + "SensorView": SensorView, + "GroundTruth": GroundTruth, + "SensorData": SensorData, +} + + +class OSITrace: + """This class can import and decode OSI trace files.""" + + @staticmethod + def map_message_type(type_name): + """Map the type name to the protobuf message type.""" + return MESSAGES_TYPE[type_name] + + def __init__(self, path=None, type_name="SensorView", cache_messages=False): + self.type = self.map_message_type(type_name) + self.file = None + self.current_index = None + self.message_offsets = None + self.read_complete = False + self.message_cache = {} if cache_messages else None + self._header_length = 4 + if path: + self.from_file(path, type_name, cache_messages) + + def from_file(self, path, type_name="SensorView", cache_messages=False): + """Import a trace from a file""" + self.type = self.map_message_type(type_name) + + if path.lower().endswith((".lzma", ".xz")): + self.file = lzma.open(path, "rb") + else: + self.file = open(path, "rb") + + self.read_complete = False + self.current_index = 0 + self.message_offsets = [0] + self.message_cache = {} if cache_messages else None + + def retrieve_offsets(self, limit=None): + """Retrieve the offsets of the messages from the file.""" + if not self.read_complete: + self.current_index = len(self.message_offsets) - 1 + self.file.seek(self.message_offsets[-1], 0) + while ( + not self.read_complete and not limit or len(self.message_offsets) <= limit + ): + self.retrieve_message(skip=True) + return self.message_offsets + + def retrieve_message(self, index=None, skip=False): + """Retrieve the next message from the file at the current position or given index, or skip it if skip is true.""" + if index is not None: + self.current_index = index + self.file.seek(self.message_offsets[index], 0) + if self.message_cache is not None and self.current_index in self.message_cache: + message = self.message_cache[self.current_index] + self.current_index += 1 + if self.current_index == len(self.message_offsets): + self.file.seek(0, 2) + else: + self.file.seek(self.message_offsets[self.current_index], 0) + if skip: + return self.message_offsets[self.current_index] + else: + return message + start = self.file.tell() + header = self.file.read(self._header_length) + if len(header) < self._header_length: + if start == self.message_offsets[-1]: + self.message_offsets.pop() + self.read_complete = True + self.file.seek(start, 0) + return None + message_length = struct.unpack("= len(self.message_offsets): + self.retrieve_offsets(index) + if self.message_cache is not None and index in self.message_cache: + return self.message_cache[index] + return self.retrieve_message(index=index) + + def get_messages(self): + """ + Yield an iterator over all messages in the file. + """ + return self.get_messages_in_index_range(0, None) + + def get_messages_in_index_range(self, begin, end): + """ + Yield an iterator over messages of indexes between begin and end included. + """ + if begin >= len(self.message_offsets): + self.retrieve_offsets(begin) + self.restart(begin) + current = begin + while end is None or current < end: + if self.message_cache is not None and current in self.message_cache: + yield self.message_cache[current] + else: + message = self.retrieve_message() + if message is None: + break + yield message + current += 1 + + def close(self): + if self.file: + self.file.close() + self.file = None + self.current_index = None + self.message_cache = None + self.message_offsets = None + self.read_complete = False + self.read_limit = None + self.type = None diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..6627fa881 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = [ + "setuptools", + "wheel", + "protoc-wheel-0==24.4", +] +build-backend = "setuptools.build_meta" + +[project] +name = "open-simulation-interface" +description = "ASAM Open Simulation Interface Python Bindings." +authors = [ + {name = "ASAM Open Simulation Interface Project", email = "osi@asam.net"}, +] +maintainers = [ + {name = "ASAM Open Simulation Interface Project", email = "osi@asam.net"}, +] +dependencies = [ + "protobuf>=4.24.4", +] + +license = {file = "LICENSE"} +readme = "README.md" +classifiers = [ + "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", +] +dynamic = ["version"] + +[project.urls] +Homepage = "https://github.com/OpenSimulationInterface/open-simulation-interface" +Repository = "https://github.com/OpenSimulationInterface/open-simulation-interface.git" +"Bug Tracker" = "https://github.com/OpenSimulationInterface/open-simulation-interface/issues" + +[project.scripts] +osi2read = "osi3trace.osi2read:main" diff --git a/requirements_develop.txt b/requirements_tests.txt similarity index 56% rename from requirements_develop.txt rename to requirements_tests.txt index 5777ec27c..43e06de0a 100644 --- a/requirements_develop.txt +++ b/requirements_tests.txt @@ -1,4 +1,2 @@ -setuptools -wheel pyyaml black==23.12.1 diff --git a/setup.py b/setup.py index cad557993..5a310a8b3 100644 --- a/setup.py +++ b/setup.py @@ -7,38 +7,41 @@ import re from distutils.spawn import find_executable -# setuptool is dependend on wheel package from setuptools import setup +from setuptools.command.sdist import sdist from setuptools.command.build_py import build_py -# configure the version number -from shutil import copyfile - -copyfile("VERSION", "version.py") -from version import * +# protoc +from protoc import PROTOC_EXE -with open("osi_version.proto.in", "rt") as fin: - with open("osi_version.proto", "wt") as fout: - for line in fin: - lineConfigured = line.replace("@VERSION_MAJOR@", str(VERSION_MAJOR)) - lineConfigured = lineConfigured.replace( - "@VERSION_MINOR@", str(VERSION_MINOR) - ) - lineConfigured = lineConfigured.replace( - "@VERSION_PATCH@", str(VERSION_PATCH) - ) - fout.write(lineConfigured) +# configure the version number +VERSION_MAJOR = None +VERSION_MINOR = None +VERSION_PATCH = None +VERSION_SUFFIX = None +with open("VERSION", "rt") as versionin: + for line in versionin: + if line.startswith("VERSION_MAJOR"): + VERSION_MAJOR = int(line.split("=")[1].strip()) + if line.startswith("VERSION_MINOR"): + VERSION_MINOR = int(line.split("=")[1].strip()) + if line.startswith("VERSION_PATCH"): + VERSION_PATCH = int(line.split("=")[1].strip()) + if line.startswith("VERSION_SUFFIX"): + VERSION_SUFFIX = line.split("=")[1].strip() package_name = "osi3" package_path = os.path.join(os.getcwd(), package_name) -class GenerateProtobufCommand(build_py): +class ProtobufGenerator: @staticmethod def find_protoc(): """Locates protoc executable""" - if "PROTOC" in os.environ and os.path.exists(os.environ["PROTOC"]): + if os.path.exists(PROTOC_EXE): + protoc = PROTOC_EXE + elif "PROTOC" in os.environ and os.path.exists(os.environ["PROTOC"]): protoc = os.environ["PROTOC"] else: protoc = find_executable("protoc") @@ -89,7 +92,19 @@ def find_protoc(): """ Generate Protobuf Messages """ - def run(self): + def generate(self): + sys.stdout.write("Generating Protobuf Version Message\n") + with open("osi_version.proto.in", "rt") as fin: + with open("osi_version.proto", "wt") as fout: + for line in fin: + lineConfigured = line.replace("@VERSION_MAJOR@", str(VERSION_MAJOR)) + lineConfigured = lineConfigured.replace( + "@VERSION_MINOR@", str(VERSION_MINOR) + ) + lineConfigured = lineConfigured.replace( + "@VERSION_PATCH@", str(VERSION_PATCH) + ) + fout.write(lineConfigured) pattern = re.compile('^import "osi_') for source in self.osi_files: with open(source) as src_file: @@ -103,38 +118,46 @@ def run(self): source_path = os.path.join(package_name, source) subprocess.check_call([self.find_protoc(), "--python_out=.", source_path]) + def maybe_generate(self): + if os.path.exists("osi_version.proto.in"): + self.generate() + + +class CustomBuildPyCommand(build_py): + def run(self): + ProtobufGenerator().maybe_generate() build_py.run(self) +class CustomSDistCommand(sdist): + def run(self): + ProtobufGenerator().generate() + sdist.run(self) + + try: os.mkdir(package_path) except Exception: pass try: - open(os.path.join(package_path, "__init__.py"), "a").close() + with open(os.path.join(package_path, "__init__.py"), "wt") as init_file: + init_file.write( + f"__version__ = '{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}{VERSION_SUFFIX or ''}'\n" + ) except Exception: pass setup( - name="open-simulation-interface", - version=str(VERSION_MAJOR) + "." + str(VERSION_MINOR) + "." + str(VERSION_PATCH), - description="A generic interface for the environmental perception of" - "automated driving functions in virtual scenarios.", - author="Carlo van Driesten, Timo Hanke, Nils Hirsenkorn," - "Pilar Garcia-Ramos, Mark Schiementz, Sebastian Schneider", - author_email="Carlo.van-Driesten@bmw.de, Timo.Hanke@bmw.de," - "Nils.Hirsenkorn@tum.de, Pilar.Garcia-Ramos@bmw.de," - "Mark.Schiementz@bmw.de, Sebastian.SB.Schneider@bmw.de", - packages=[package_name], - install_requires=["protobuf"], + version=str(VERSION_MAJOR) + + "." + + str(VERSION_MINOR) + + "." + + str(VERSION_PATCH) + + (VERSION_SUFFIX or ""), + packages=[package_name, "osi3trace"], cmdclass={ - "build_py": GenerateProtobufCommand, + "sdist": CustomSDistCommand, + "build_py": CustomBuildPyCommand, }, - url="https://github.com/OpenSimulationInterface/open-simulation-interface", - license="MPL 2.0", - classifiers=[ - "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", - ], - data_files=[("", ["LICENSE"])], ) diff --git a/tests/test_osi_trace.py b/tests/test_osi_trace.py index c832b63d7..8a9a430ec 100644 --- a/tests/test_osi_trace.py +++ b/tests/test_osi_trace.py @@ -2,7 +2,7 @@ import tempfile import unittest -from format.OSITrace import OSITrace +from osi3trace.osi_trace import OSITrace from osi3.osi_sensorview_pb2 import SensorView import struct @@ -14,10 +14,11 @@ def test_osi_trace(self): path_input = os.path.join(tmpdirname, "input.osi") create_sample(path_input) - trace = OSITrace() - trace.from_file(path=path_input) - trace.make_readable(path_output, index=1) - trace.scenario_file.close() + trace = OSITrace(path_input) + with open(path_output, "wt") as f: + for message in trace: + f.write(str(message)) + trace.close() self.assertTrue(os.path.exists(path_output))