From 6f17f9f2db3c42b3d32161172b9ec012f5a1ca73 Mon Sep 17 00:00:00 2001 From: ruttendx Date: Fri, 5 Dec 2025 15:06:45 -0500 Subject: [PATCH 1/4] jsonl to mdf conversion script --- scripts/convertmf4/.gitignore | 1 + scripts/convertmf4/README.md | 9 ++++++++ scripts/convertmf4/jsonl_to_mf4.py | 33 +++++++++++++++++++++++++++++ scripts/convertmf4/requirements.txt | 1 + 4 files changed, 44 insertions(+) create mode 100644 scripts/convertmf4/.gitignore create mode 100644 scripts/convertmf4/README.md create mode 100644 scripts/convertmf4/jsonl_to_mf4.py create mode 100644 scripts/convertmf4/requirements.txt diff --git a/scripts/convertmf4/.gitignore b/scripts/convertmf4/.gitignore new file mode 100644 index 000000000..f76cdf78f --- /dev/null +++ b/scripts/convertmf4/.gitignore @@ -0,0 +1 @@ +./mf4 diff --git a/scripts/convertmf4/README.md b/scripts/convertmf4/README.md new file mode 100644 index 000000000..0b3b8a73f --- /dev/null +++ b/scripts/convertmf4/README.md @@ -0,0 +1,9 @@ +# Jsonl to MDF + +Script to convert json lines output from tracer to mdf file; used for visualization with Asammdf gui (https://canlogger.csselectronics.com/canedge-getting-started/ce2/log-file-tools/asammdf-gui/). Recommended to use uv python package manager. + +# Usage + +```bash +uv run ./jsonl_to_mdf.py ./trace.jsonl +``` diff --git a/scripts/convertmf4/jsonl_to_mf4.py b/scripts/convertmf4/jsonl_to_mf4.py new file mode 100644 index 000000000..520b9b423 --- /dev/null +++ b/scripts/convertmf4/jsonl_to_mf4.py @@ -0,0 +1,33 @@ +from asammdf import MDF, Signal +import sys +import numpy as np +import pandas as pd + +input = sys.argv[1] # .jsonl +output = input + ".mdf" # .mf4 +mdf = MDF(version="4.10") + +# read jsonl file into pandas dataframe +dataframe = pd.read_json(input, lines=True) + +# convert each can id to mdf signal +for can_id, group in dataframe.groupby("id"): + # convert time to seconds from start + times = pd.to_datetime(group["time"], format="%H:%M:%S.%f") + t0 = times.iloc[0] + timestamps = (times - t0).dt.total_seconds().to_numpy(dtype=np.float64) + + # bytes is tuple, make 2D uint8 array [n samples, n bytes] + samples = np.array(group["bytes"].tolist(), dtype=np.uint8) + + # generate signal & append mdf file + sig = Signal( + name = str(can_id), + samples = samples, + timestamps = timestamps, + ) + mdf.append(sig) + +# save mdf file (.mf4) +mdf.save(output) +mdf.close() diff --git a/scripts/convertmf4/requirements.txt b/scripts/convertmf4/requirements.txt new file mode 100644 index 000000000..2ff743560 --- /dev/null +++ b/scripts/convertmf4/requirements.txt @@ -0,0 +1 @@ +asammdf From fd89c2b70abeb792952e648ed64354ba31573ec2 Mon Sep 17 00:00:00 2001 From: ruttendx Date: Fri, 5 Dec 2025 15:11:07 -0500 Subject: [PATCH 2/4] jsonl to mf4 conversion script for asammdf metrics visualization --- scripts/{convertmf4 => convert_trace}/.gitignore | 0 scripts/{convertmf4 => convert_trace}/README.md | 0 scripts/{convertmf4 => convert_trace}/jsonl_to_mf4.py | 0 scripts/{convertmf4 => convert_trace}/requirements.txt | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename scripts/{convertmf4 => convert_trace}/.gitignore (100%) rename scripts/{convertmf4 => convert_trace}/README.md (100%) rename scripts/{convertmf4 => convert_trace}/jsonl_to_mf4.py (100%) rename scripts/{convertmf4 => convert_trace}/requirements.txt (100%) diff --git a/scripts/convertmf4/.gitignore b/scripts/convert_trace/.gitignore similarity index 100% rename from scripts/convertmf4/.gitignore rename to scripts/convert_trace/.gitignore diff --git a/scripts/convertmf4/README.md b/scripts/convert_trace/README.md similarity index 100% rename from scripts/convertmf4/README.md rename to scripts/convert_trace/README.md diff --git a/scripts/convertmf4/jsonl_to_mf4.py b/scripts/convert_trace/jsonl_to_mf4.py similarity index 100% rename from scripts/convertmf4/jsonl_to_mf4.py rename to scripts/convert_trace/jsonl_to_mf4.py diff --git a/scripts/convertmf4/requirements.txt b/scripts/convert_trace/requirements.txt similarity index 100% rename from scripts/convertmf4/requirements.txt rename to scripts/convert_trace/requirements.txt From db838ea6e84b38ec2108d0b0d49fc4fa72885848 Mon Sep 17 00:00:00 2001 From: ruttendx Date: Fri, 5 Dec 2025 15:13:43 -0500 Subject: [PATCH 3/4] updated readme --- scripts/convert_trace/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/convert_trace/README.md b/scripts/convert_trace/README.md index 0b3b8a73f..ae7443c08 100644 --- a/scripts/convert_trace/README.md +++ b/scripts/convert_trace/README.md @@ -5,5 +5,5 @@ Script to convert json lines output from tracer to mdf file; used for visualizat # Usage ```bash -uv run ./jsonl_to_mdf.py ./trace.jsonl +python ./jsonl_to_mdf.py ./trace.jsonl ``` From 5953a7dbbd6d159f48b33ba9285997b2f942a3f5 Mon Sep 17 00:00:00 2001 From: ruttendx Date: Wed, 10 Dec 2025 21:47:13 -0500 Subject: [PATCH 4/4] updated readme --- scripts/convert_trace/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/convert_trace/README.md b/scripts/convert_trace/README.md index ae7443c08..4c2a4973e 100644 --- a/scripts/convert_trace/README.md +++ b/scripts/convert_trace/README.md @@ -1,9 +1,13 @@ # Jsonl to MDF -Script to convert json lines output from tracer to mdf file; used for visualization with Asammdf gui (https://canlogger.csselectronics.com/canedge-getting-started/ce2/log-file-tools/asammdf-gui/). Recommended to use uv python package manager. +Convert a [JSON lines](https://github.com/macformula/hil/blob/main/canlink/jsonl.go) log file from our [CAN tracer](https://github.com/macformula/hil/blob/main/canlink/tracer.go) to an MDF file. + +For developers to convert JSON lines log format to MDF for integration with [`asammdf`](https://asammdf.readthedocs.io/en/latest/). With this format, visualization tools such as [Asammdf GUI](https://canlogger.csselectronics.com/canedge-getting-started/ce2/log-file-tools/asammdf-gui/) can be used to view and debug traces. # Usage +It is recommended to use uv python package manager. + ```bash python ./jsonl_to_mdf.py ./trace.jsonl ```