Skip to content

Commit 47db48d

Browse files
andreadesimandreadesim
committed
refactored test environment
1 parent f5e8dfd commit 47db48d

File tree

6 files changed

+173
-117
lines changed

6 files changed

+173
-117
lines changed

‎.gitignore

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
1+
# build folder for C code
12
build/
23

34
.vscode/
45
__pycache__/
56
.idea/
67

8+
# private path of bpatch exectable
79
path_bpatch.py
810

9-
/test/*
10-
!/test/*.py
11+
# test files
12+
/test/testin/*
13+
/test/testout/*

‎bpatch.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,15 @@
1+
#!/usr/bin/python3
2+
3+
"""bpatch.py - Bit Patch Firmware Tool"""
4+
5+
__author__ = "Andrea De Simone and Fabrizio Riente"
6+
__copyright__ = "Copyright 2025, Politecnico di Torino"
7+
__license__ = "Apache-2.0"
8+
__version__ = "1.0.1"
9+
__maintainer__ = "Andrea De Simone"
10+
__email__ = "[email protected], [email protected]"
11+
12+
113
import binascii
214
import re
315
from math import log, ceil

‎c/main.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
#include "bpatch.h"
77

88
// Define the size of the firmware slots in Kilobytes
9-
#define FW_SLOT_SIZE 2000
9+
#define FW_SLOT_SIZE 3000
1010
// Define the size of the patch slot in Kilobytes
11-
#define PATCH_SLOT_SIZE 1000
11+
#define PATCH_SLOT_SIZE 2000
1212

1313
#endif

‎test/README.md

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Test Environment
2+
3+
This section describes the environment used to test patch generation and application.
4+
It includes two scripts:
5+
6+
- `test.py`: Automatically generates and verifies binary patches.
7+
- `plot.py`: Visualizes patch sizes over firmware versions.
8+
9+
## How to Use
10+
11+
### 1. Patch Generation and Verification (`test.py`)
12+
13+
Place one or more folders inside the `testin` directory. Each folder should contain the binaries to be tested.
14+
Binaries must include the firmware version in their filename, e.g., `f_v1.bin`, `f_v2.bin`, etc.
15+
The script extracts the version number from the last numeric part of the filename.
16+
17+
For each folder in `testin`:
18+
19+
- Patches will be generated and verified.
20+
- Output will be saved in a corresponding folder in `testout`.
21+
- Patches will be named `p_n.bin`, where `n` indicates the version number of the new firmware.
22+
23+
### 2. Patch Size Plotting (`plot.py`)
24+
25+
This script generates a plot showing the size of each patch relative to its firmware version.
26+
27+
**Requirements**:
28+
29+
- Patches must be generated using `test.py`.
30+
- The `matplotlib` Python library is required.
31+
32+
For each folder in `testout`, a separate plot will be generated.

‎test/plot.py

Lines changed: 41 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -1,87 +1,44 @@
11
import matplotlib.pyplot as plt
2-
import numpy as np
3-
from math import ceil
42
import os
53

6-
from test import extract_last_decimal
7-
8-
for d in os.listdir():
9-
if d.startswith('f'):
10-
r = {}
11-
12-
fw_size = np.array([])
13-
patch_size = np.array([])
14-
n_frag_patch = np.array([])
15-
n_frag_fw = np.array([])
16-
ovh_nbd = np.array([])
17-
ovh_nbc = np.array([])
18-
ovh_nba = np.array([])
19-
ovh_tot = np.array([])
20-
n_nba = np.array([])
21-
22-
if os.path.exists(d + '/testout'):
23-
for report in os.listdir(d + '/testout'):
24-
if report.startswith('report'):
25-
# extract version
26-
version = extract_last_decimal(report)
27-
# add to dictionary
28-
r[version] = report
29-
30-
# check if there are report in the folder
31-
if r:
32-
fw_versions = sorted(r.keys())
33-
for v in fw_versions:
34-
with open(d + '/testout/' + r[v], 'r') as f:
35-
f.readline()
36-
fields = f.readline().split(',')
37-
38-
fw_size = np.append(fw_size, int(fields[0]))
39-
patch_size = np.append(patch_size, int(fields[1]))
40-
n_frag_patch = np.append(n_frag_patch, ceil(int(fields[1])/112))
41-
n_frag_fw = np.append(n_frag_fw, ceil(int(fields[0])/112))
42-
n_nba = np.append(n_nba, int(fields[4]))
43-
ovh_nbd = np.append(ovh_nbd, float(fields[5]))
44-
ovh_nbc = np.append(ovh_nbc, float(fields[6]))
45-
ovh_nba = np.append(ovh_nba, float(fields[7]))
46-
ovh_tot = np.append(ovh_tot, float(fields[8]))
47-
48-
# fig, axs = plt.subplots(2, 2, figsize=(15, 10))
49-
fig, axs = plt.subplots(2, 2)
50-
fig.suptitle(f'Statistics folder: {d}')
51-
52-
axs[0, 0].set_title('Compression')
53-
axs[0, 0].plot(fw_versions, patch_size / fw_size * 100)
54-
axs[0, 0].set_xlabel('FW version')
55-
axs[0, 0].set_ylabel('Patch compression [%]')
56-
axs[0, 0].grid()
57-
58-
axs[0, 1].set_title('New Bytes')
59-
axs[0, 1].plot(fw_versions, np.ceil(n_nba / 1024))
60-
axs[0, 1].set_xlabel('FW version')
61-
axs[0, 1].set_ylabel('kB')
62-
axs[0, 1].grid()
63-
64-
axs[1, 0].set_title('Overhead')
65-
axs[1, 0].plot(fw_versions, ovh_nbd, fw_versions, ovh_nbc, fw_versions, ovh_nba, fw_versions, ovh_tot)
66-
axs[1, 0].set_xlabel('FW version')
67-
axs[1, 0].set_ylabel('Overhead respect new bytes [%]')
68-
axs[1, 0].legend(['nbd', 'nbc', 'nba', 'total'])
69-
axs[1, 0].grid()
70-
71-
width = 0.3 # the width of the bars
72-
species = np.arange(len(fw_versions))
73-
74-
p1 = axs[1, 1].bar(species - width / 2, n_frag_fw, width, label='FW')
75-
axs[1, 1].bar_label(p1, label_type='edge')
76-
p2 = axs[1, 1].bar(species + width / 2, n_frag_patch, width, label='Patch')
77-
axs[1, 1].bar_label(p2, label_type='edge')
78-
79-
axs[1, 1].set_title('Number of fragment comparison')
80-
axs[1, 1].set_xticks(species)
81-
axs[1, 1].set_xticklabels(fw_versions)
82-
axs[1, 1].legend()
83-
84-
plt.tight_layout()
85-
plt.show()
86-
87-
exit()
4+
from test import extract_last_decimal, TESTOUT_FOLDER, TESTIN_FOLDER
5+
6+
# exit if TESTOUT_FOLDER does not exist
7+
if not os.path.exists(TESTOUT_FOLDER):
8+
exit(0)
9+
10+
# iterate over all directories in TESTOUT_FOLDER
11+
for d in os.listdir(TESTOUT_FOLDER):
12+
# create patch dict with patch name and version
13+
p = {}
14+
15+
for patch in os.listdir(f"{TESTOUT_FOLDER}/{d}"):
16+
# extract the version from the firmware name
17+
version = extract_last_decimal(patch)
18+
# add patch in dict
19+
p[version] = os.path.getsize(f"{TESTOUT_FOLDER}/{d}/{patch}") / 1024
20+
21+
# create firmware dict with fw name and version
22+
fw = {}
23+
24+
for f in os.listdir(f"{TESTIN_FOLDER}/{d}"):
25+
# extract the version from the firmware name
26+
version = extract_last_decimal(f)
27+
# add patch in dict
28+
fw[version] = os.path.getsize(f"{TESTIN_FOLDER}/{d}/{f}") / 1024
29+
30+
# sort the patch and firmware dicts by version
31+
versions = sorted(p.keys())
32+
p_sizes = [p[v] for v in versions]
33+
f_sizes = [fw[v] for v in versions]
34+
35+
# plot the firmwares and patch sizes
36+
plt.figure()
37+
plt.title(f"{d}")
38+
plt.xlabel('Firmware Version')
39+
plt.ylabel('KBytes')
40+
plt.grid()
41+
plt.plot(versions, p_sizes, 'o-', label='Patch Size')
42+
plt.plot(versions, f_sizes, 'o-', label='Firmware Size')
43+
plt.legend()
44+
plt.show()

‎test/test.py

Lines changed: 81 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,50 +1,102 @@
11
import os
2-
import subprocess
32
import re
43
import sys
54

5+
6+
# path to testin and testout folders
7+
TESTIN_FOLDER = "testin"
8+
TESTOUT_FOLDER = "testout"
9+
10+
# set verbosity level
11+
"""
12+
with VERBOSE = 0, only errors and final result is printed
13+
with VERBOSE = 1, also test start and end for each folder is printed
14+
with VERBOSE = 2, also a message for each patch is printed
15+
"""
16+
VERBOSE = 1
17+
# set option for bpatch
18+
OPTIONS = "-d"
19+
20+
621
def extract_last_decimal(s):
22+
"""
23+
Extract the last decimal number from a string.
24+
25+
:param s: string to extract the last decimal number from
26+
:type s: str
27+
:return: the last decimal number found in the string, or None if no number is found
28+
:rtype: int
29+
"""
30+
731
# find all numbers in the string
832
numbers = re.findall(r'\d+', s)
933
# restur last number if exists
1034
return numbers[-1]if numbers else None
1135

1236
def test_bpatch():
37+
"""
38+
run the bpatch test on all directories in TESTIN_FOLDER.
39+
40+
:return: error count
41+
:rtype: int
42+
"""
43+
1344
error_count = 0
45+
tested_patches = 0
46+
47+
# check if testin and testout folders exist, otherwise create them
48+
if not os.path.exists(TESTIN_FOLDER):
49+
os.makedirs(TESTIN_FOLDER)
50+
if not os.path.exists(TESTOUT_FOLDER):
51+
os.makedirs(TESTOUT_FOLDER)
1452

15-
for d in os.listdir():
16-
if d.startswith('f'):
53+
# iterate over all directories in TESTIN_FOLDER
54+
for d in os.listdir(TESTIN_FOLDER):
55+
if VERBOSE >= 1:
1756
print(f'### Start test folder {d} ###')
18-
# create output directory if not exists
19-
if not os.path.exists(d + '/testout'):
20-
os.makedirs(d + '/testout')
21-
# create firmware dict with fw name and version
22-
f = {}
23-
for el in os.listdir(d):
24-
# check if the file is a binary file
25-
if el.endswith('.bin'):
26-
version = extract_last_decimal(el)
27-
# check if the version is not None and not in the dict
28-
if version is None:
29-
print(f'Error: cannot find version in {el}', file=sys.stderr)
30-
exit(-1)
31-
if version in f:
32-
print(f'Error: duplicate version {version} in {el}', file=sys.stderr)
33-
exit(-1)
34-
# add fw in dict
35-
f[version] = el
36-
37-
for version in sorted(f.keys())[:-1]:
38-
print(f'Start test bpatch for {d}/{f[version]} and {d}/{f[str(int(version) + 1)]}')
39-
# run bpatch
40-
if os.system(f'python3 ../bpatch.py encode {d}/{f[version]} {d}/{f[str(int(version) + 1)]} {d}/testout/patch_{int(version) + 1}.bin -v -R {d}/testout/report_{int(version) + 1}.csv> /dev/null') != 0:
41-
print(f'Error while running bpatch for {d}/{f[version]} and {d}/{f[str(int(version) + 1)]}', file=sys.stderr)
42-
error_count += 1
43-
else:
57+
58+
# create output directory if not exists or clean previous results
59+
if not os.path.exists(f"{TESTOUT_FOLDER}/{d}"):
60+
os.makedirs(f"{TESTOUT_FOLDER}/{d}")
61+
else:
62+
for file in os.listdir(f"{TESTOUT_FOLDER}/{d}"):
63+
os.remove(os.path.join(f"{TESTOUT_FOLDER}/{d}", file))
64+
65+
# create firmware dict with fw name and version
66+
f = {}
67+
for fw in os.listdir(f"{TESTIN_FOLDER}/{d}"):
68+
# extract the version from the firmware name
69+
version = extract_last_decimal(fw)
70+
# check if the version is not None and not in the dict
71+
if version is None:
72+
print(f'Error: cannot find version in {fw}', file=sys.stderr)
73+
exit(-1)
74+
if version in f:
75+
print(f'Error: duplicate version {version} in {fw}', file=sys.stderr)
76+
exit(-1)
77+
# add fw in dict
78+
f[version] = fw
79+
80+
# sort the versions
81+
versions = sorted(f.keys())
82+
83+
for i in range(len(versions) - 1):
84+
if VERBOSE >= 2:
85+
print(f'Start test bpatch for {d}/{f[versions[i]]} and {d}/{f[versions[i+1]]}')
86+
# run bpatch
87+
if os.system(f'python3 ../bpatch.py encode {TESTIN_FOLDER}/{d}/{f[versions[i]]} {TESTIN_FOLDER}/{d}/{f[versions[i+1]]} {TESTOUT_FOLDER}/{d}/p_{versions[i+1]}.bin {OPTIONS} -v > /dev/null') != 0:
88+
print(f'Error while running bpatch for {d}/{f[versions[i]]} and {d}/{f[versions[i+1]]}', file=sys.stderr)
89+
error_count += 1
90+
else:
91+
if VERBOSE >= 2:
4492
print('Test OK')
93+
tested_patches += 1
4594

95+
if VERBOSE >= 1:
4696
print(f'### End test folder {d} ###\n\n')
4797

98+
print(f"Test finished\nTotal patches tested: {tested_patches}\nErrors found: {error_count}\n")
99+
48100
return error_count
49101

50102

0 commit comments

Comments
 (0)