Skip to content

Commit 9cc9e15

Browse files
minkyu-choi07minkyu-choi07
authored
Merge pull request #25 from UTAustin-SwarmLab/test-ltl-scripts
For release 0.0.4
2 parents 22a7905 + 35d6843 commit 9cc9e15

38 files changed

+1937
-357
lines changed

‎.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
#ns_vfs_custom
2+
test_scripts/runs/
3+
artifacts/
4+
15
# Byte-compiled / optimized / DLL files
26
__pycache__/
37
*.py[cod]

‎LICENSE

100644100755
File mode changed.

‎Makefile

100644100755
+4-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ SHELL := /bin/bash
55
BASE_IMG=nvidia/cuda:11.8.0-devel-ubuntu20.04
66

77
# USER INPUT (TODO: PLEASE MODIFY)
8-
CODE_PATH := /home/mc76728/repos/Video-to-Automaton/
8+
CODE_PATH := /home/ss96869/Neuro-Symbolic-Video-Frame-Search
9+
DS_PATH := /store/datasets/
910

1011
# Custom Image
1112
DOCKER_IMG := ns_vfs
@@ -26,6 +27,7 @@ run_docker_container:
2627
--cap-add=SYS_PTRACE \
2728
--ulimit core=0:0 \
2829
--volume ${CODE_PATH}:/opt/Neuro-Symbolic-Video-Frame-Search \
30+
--volume ${DS_PATH}:/store/datasets \
2931
${DOCKER_IMG}:${TAG} \
3032
/bin/bash
3133

@@ -39,6 +41,7 @@ run_docker_container_gpu:
3941
--cap-add=SYS_PTRACE \
4042
--ulimit core=0:0 \
4143
--volume ${CODE_PATH}:/opt/Neuro-Symbolic-Video-Frame-Search \
44+
--volume ${DS_PATH}:/store/datasets \
4245
${DOCKER_IMG}:${TAG} \
4346
/bin/bash
4447

‎README.md

100644100755
File mode changed.

‎experiments/common.py

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from pathlib import Path
2+
3+
4+
def get_available_benchmark_video(path_to_directory: str):
5+
if isinstance(path_to_directory, str):
6+
directory_path = Path(path_to_directory)
7+
return list(directory_path.glob("*.pkl"))
8+
else:
9+
directory_path = path_to_directory
10+
return list(directory_path.rglob("*.pkl"))

‎experiments/evaluate_benchmark_frame_video_with_manual_confidence.py

+164
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
from __future__ import annotations
2+
3+
import csv
4+
from pathlib import Path
5+
6+
from ns_vfs.common.utility import save_frames
7+
from ns_vfs.config.loader import load_config
8+
from ns_vfs.data.frame import BenchmarkLTLFrame, FramesofInterest
9+
from ns_vfs.frame_searcher import FrameSearcher
10+
from ns_vfs.model.vision.grounding_dino import GroundingDino
11+
from ns_vfs.processor.benchmark_video_processor import BenchmarkVideoFrameProcessor
12+
from ns_vfs.video_to_automaton import VideotoAutomaton
13+
14+
15+
def get_frames(frames_of_interest: list, benchmark_video: BenchmarkLTLFrame):
16+
output = []
17+
for frame_set in frames_of_interest:
18+
if len(frame_set) == 1:
19+
idx = frame_set[0]
20+
output.append([benchmark_video.images_of_frames[idx]])
21+
else:
22+
idx_first, idx_last = frame_set[0], frame_set[-1]
23+
tmp_list = []
24+
for idx in range(idx_first, idx_last + 1):
25+
tmp_list.append(benchmark_video.images_of_frames[idx])
26+
output.append(tmp_list)
27+
tmp_list = []
28+
return output
29+
30+
31+
def evaluate_frame_of_interest(
32+
benchmark_video_file: str,
33+
benchmark_video: BenchmarkLTLFrame,
34+
frame_of_interest: FramesofInterest,
35+
directory_path: str,
36+
):
37+
result = dict()
38+
dir_path = Path(directory_path) / benchmark_video_file.name.split(".pkl")[0]
39+
dir_path.mkdir(parents=True, exist_ok=True)
40+
41+
true_foi_list = benchmark_video.frames_of_interest
42+
total_num_true_foi = len(true_foi_list)
43+
44+
num_of_mathching_frame_set = sum(1 for a, b in zip(true_foi_list, frame_of_interest.foi_list) if a == b)
45+
frame_set_accuracy = num_of_mathching_frame_set / total_num_true_foi
46+
47+
# matching_accuracy
48+
flattened_true_foi = set([item for sublist in true_foi_list for item in sublist])
49+
flattened_predicted_foi = set([item for sublist in frame_of_interest.foi_list for item in sublist])
50+
true_positive_set = flattened_true_foi.intersection(flattened_predicted_foi)
51+
false_positive_set = flattened_predicted_foi.difference(flattened_true_foi)
52+
false_negatives = flattened_true_foi.difference(flattened_predicted_foi)
53+
54+
# filename = benchmark_video_file.name.split("_ltl_")[-1].split("_")[0]
55+
dir_path / benchmark_video_file.name.split(".pkl")[0] / ".json"
56+
57+
result["ltl_formula"] = benchmark_video.ltl_formula
58+
result["total_number_of_frame"] = len(benchmark_video.labels_of_frames)
59+
result["exact_frame_accuracy"] = frame_set_accuracy
60+
result["num_true_positive"] = len(true_positive_set)
61+
result["num_false_positive"] = len(false_positive_set)
62+
result["precision"] = len(true_positive_set) / len(true_positive_set) + len(false_positive_set)
63+
result["recall"] = len(true_positive_set) / len(true_positive_set) + len(false_negatives)
64+
65+
result["groud_truth_frame"] = benchmark_video.frames_of_interest
66+
result["predicted_frame"] = frame_of_interest.foi_list
67+
68+
result["total_number_of_framer_of_interest"] = len(benchmark_video.frames_of_interest)
69+
result["total_number_of_frame"] = len(benchmark_video.labels_of_frames)
70+
71+
i = 0
72+
for frame_image_set in get_frames(frame_of_interest.foi_list, benchmark_video):
73+
path = Path(directory_path) / benchmark_video_file.name.split(".pkl")[0] / f"video_frame_{i}"
74+
save_frames(frames=frame_image_set, path=path, file_label="predicted_frame")
75+
i += 1
76+
77+
# save_dict_to_pickle(
78+
# path=Path(directory_path) / benchmark_video_file.name.split(".pkl")[0],
79+
# dict_obj=result,
80+
# file_name="result.pkl",
81+
# )
82+
# Specifying the file name
83+
csv_file_name = Path(directory_path) / "data.csv"
84+
85+
with open(csv_file_name, mode="a", newline="") as file:
86+
writer = csv.DictWriter(file, fieldnames=result.keys())
87+
88+
if not csv_file_name.exists():
89+
# If file does not exist, write header
90+
writer.writeheader()
91+
writer.writerow(result)
92+
93+
acc_file = Path(directory_path) / "accuracy.txt"
94+
with acc_file.open("a") as f:
95+
f.write(
96+
f"""{result["ltl_formula"]} - total num frame: {result["total_number_of_frame"]} - exact_frame_accuracy: {result["exact_frame_accuracy"]}
97+
num_true_positive: {result["num_true_positive"]}, num_false_positive: {result["num_false_positive"]} ,
98+
precision: {result["precision"]} recall: {result["recall"]}\n"""
99+
)
100+
101+
102+
def get_available_benchmark_video(path_to_directory: str):
103+
if isinstance(path_to_directory, str):
104+
directory_path = Path(path_to_directory)
105+
return list(directory_path.glob("*.pkl"))
106+
else:
107+
directory_path = path_to_directory
108+
return list(directory_path.rglob("*.pkl"))
109+
110+
111+
if __name__ == "__main__":
112+
config = load_config()
113+
benchmark_frame_video_root_dir = Path(
114+
"/opt/Neuro-Symbolic-Video-Frame-Search/artifacts/benchmark_frame_video/"
115+
)
116+
117+
benchmark_image_set_dir = [x for x in benchmark_frame_video_root_dir.iterdir() if x.is_dir()]
118+
119+
for benchmark_name_dir in benchmark_image_set_dir:
120+
ltl_video_dir_set = [x for x in benchmark_name_dir.iterdir() if x.is_dir()]
121+
if len(ltl_video_dir_set) > 0:
122+
print(f"--processing {benchmark_name_dir.name}--")
123+
print(f"number of ltl rule: {len(ltl_video_dir_set)}")
124+
for ltl_video_dir in ltl_video_dir_set:
125+
benchmark_video_file_list = get_available_benchmark_video(ltl_video_dir)
126+
print(f"number of examples of {ltl_video_dir.name}: {len(benchmark_video_file_list)}")
127+
128+
for benchmark_video_file in benchmark_video_file_list:
129+
benchmark_video_processor = BenchmarkVideoFrameProcessor(
130+
video_path=benchmark_video_file,
131+
artifact_dir=config.VERSION_AND_PATH.ARTIFACTS_PATH,
132+
manual_confidence_probability=1.0,
133+
)
134+
135+
benchmark_img_frame: BenchmarkLTLFrame = benchmark_video_processor.benchmark_image_frames
136+
137+
video_automata_builder = VideotoAutomaton(
138+
detector=GroundingDino(
139+
config=config.GROUNDING_DINO,
140+
weight_path=config.GROUNDING_DINO.GROUNDING_DINO_CHECKPOINT_PATH,
141+
config_path=config.GROUNDING_DINO.GROUNDING_DINO_CONFIG_PATH,
142+
),
143+
video_processor=benchmark_video_processor,
144+
artifact_dir=config.VERSION_AND_PATH.ARTIFACTS_PATH,
145+
proposition_set=benchmark_img_frame.proposition,
146+
save_annotation=False, # TODO: Debug only
147+
save_image=False, # TODO: Debug only
148+
ltl_formula=f"P>=0.80 [{benchmark_img_frame.ltl_formula}]",
149+
verbose=False,
150+
manual_confidence_probability=1.0,
151+
)
152+
frame_sercher = FrameSearcher(
153+
video_automata_builder=video_automata_builder,
154+
video_processor=benchmark_video_processor,
155+
)
156+
157+
frame_of_interest = frame_sercher.search()
158+
159+
evaluate_frame_of_interest(
160+
benchmark_video_file=benchmark_video_file,
161+
benchmark_video=benchmark_img_frame,
162+
frame_of_interest=frame_of_interest,
163+
directory_path="/opt/Neuro-Symbolic-Video-Frame-Search/artifacts/benchmark_eval_results",
164+
)

‎experiments/experiment_1_frame_search_with_benchmark_video.ipynb

+129
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 5,
6+
"metadata": {},
7+
"outputs": [],
8+
"source": [
9+
"import csv\n",
10+
"from pathlib import Path\n",
11+
"\n",
12+
"from ns_vfs.common.utility import save_frames\n",
13+
"from ns_vfs.config.loader import load_config\n",
14+
"from ns_vfs.data.frame import BenchmarkLTLFrame, FramesofInterest\n",
15+
"from ns_vfs.frame_searcher import FrameSearcher\n",
16+
"from ns_vfs.model.vision.grounding_dino import GroundingDino\n",
17+
"from ns_vfs.processor.benchmark_video_processor import BenchmarkVideoFrameProcessor\n",
18+
"from ns_vfs.video_to_automaton import VideotoAutomaton\n",
19+
"from common import get_available_benchmark_video\n",
20+
"from ns_vfs.model.vision.yolo import Yolo"
21+
]
22+
},
23+
{
24+
"cell_type": "markdown",
25+
"metadata": {},
26+
"source": [
27+
"**Global Variable**"
28+
]
29+
},
30+
{
31+
"cell_type": "code",
32+
"execution_count": 3,
33+
"metadata": {},
34+
"outputs": [],
35+
"source": [
36+
"config = load_config()\n",
37+
"benchmark_frame_video_root_dir = Path(\n",
38+
" \"/opt/Neuro-Symbolic-Video-Frame-Search/artifacts/benchmark_frame_video/\"\n",
39+
")\n",
40+
"benchmark_image_set_dir = [x for x in benchmark_frame_video_root_dir.iterdir() if x.is_dir()]\n",
41+
"cv_model_list = [\"grounding_dino\", \"yolo\"]"
42+
]
43+
},
44+
{
45+
"cell_type": "markdown",
46+
"metadata": {},
47+
"source": [
48+
"**Local Variable for the experiment**"
49+
]
50+
},
51+
{
52+
"cell_type": "code",
53+
"execution_count": null,
54+
"metadata": {},
55+
"outputs": [],
56+
"source": [
57+
"\"\"\"\n",
58+
"It will go over all available benchmark video and search for frame of interest for each cv detection model.\n",
59+
"\"\"\"\n",
60+
"for benchmark_name_dir in benchmark_image_set_dir:\n",
61+
" ltl_video_dir_set = [x for x in benchmark_name_dir.iterdir() if x.is_dir()]\n",
62+
" if len(ltl_video_dir_set) > 0:\n",
63+
" print(f\"--processing {benchmark_name_dir.name}--\")\n",
64+
" print(f\"number of ltl rule: {len(ltl_video_dir_set)}\")\n",
65+
" for ltl_video_dir in ltl_video_dir_set:\n",
66+
" benchmark_video_file_list = get_available_benchmark_video(ltl_video_dir)\n",
67+
" print(f\"number of examples of {ltl_video_dir.name}: {len(benchmark_video_file_list)}\")\n",
68+
"\n",
69+
" for benchmark_video_file in benchmark_video_file_list:\n",
70+
" for cv_model in cv_model_list:\n",
71+
" if cv_model == \"yolo\":\n",
72+
" cv_detection_model = Yolo(config=config.YOLO,\n",
73+
" weight_path=config.YOLO.YOLO_CHECKPOINT_PATH)\n",
74+
" elif cv_model == \"grounding_dino\":\n",
75+
" cv_detection_model = GroundingDino(\n",
76+
" config=config.GROUNDING_DINO,\n",
77+
" weight_path=config.GROUNDING_DINO.GROUNDING_DINO_CHECKPOINT_PATH,\n",
78+
" config_path=config.GROUNDING_DINO.GROUNDING_DINO_CONFIG_PATH,\n",
79+
" )\n",
80+
" benchmark_video_processor = BenchmarkVideoFrameProcessor(\n",
81+
" video_path=benchmark_video_file,\n",
82+
" artifact_dir=config.VERSION_AND_PATH.ARTIFACTS_PATH,\n",
83+
" manual_confidence_probability=1.0,\n",
84+
" )\n",
85+
"\n",
86+
" benchmark_img_frame: BenchmarkLTLFrame = benchmark_video_processor.benchmark_image_frames\n",
87+
"\n",
88+
" video_automata_builder = VideotoAutomaton(\n",
89+
" detector=cv_detection_model,\n",
90+
" video_processor=benchmark_video_processor,\n",
91+
" artifact_dir=config.VERSION_AND_PATH.ARTIFACTS_PATH,\n",
92+
" proposition_set=benchmark_img_frame.proposition,\n",
93+
" save_annotation=False, # TODO: Debug only\n",
94+
" save_image=False, # TODO: Debug only\n",
95+
" ltl_formula=f\"P>=0.80 [{benchmark_img_frame.ltl_formula}]\",\n",
96+
" verbose=False,\n",
97+
" )\n",
98+
" frame_sercher = FrameSearcher(\n",
99+
" video_automata_builder=video_automata_builder,\n",
100+
" video_processor=benchmark_video_processor,\n",
101+
" )\n",
102+
"\n",
103+
" frame_of_interest = frame_sercher.search()"
104+
]
105+
}
106+
],
107+
"metadata": {
108+
"kernelspec": {
109+
"display_name": "Python 3",
110+
"language": "python",
111+
"name": "python3"
112+
},
113+
"language_info": {
114+
"codemirror_mode": {
115+
"name": "ipython",
116+
"version": 3
117+
},
118+
"file_extension": ".py",
119+
"mimetype": "text/x-python",
120+
"name": "python",
121+
"nbconvert_exporter": "python",
122+
"pygments_lexer": "ipython3",
123+
"version": "3.8.10"
124+
},
125+
"orig_nbformat": 4
126+
},
127+
"nbformat": 4,
128+
"nbformat_minor": 2
129+
}

‎install.sh

100644100755
+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ if [[ ! -e $WEIGHT_DIR ]]; then
1111
cd $WEIGHT_DIR
1212
wget -q https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth
1313
wget -q https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth
14+
wget -q https://github.com/ultralytics/assets/releases/download/v0.0.0/yolov8n.pt
1415
elif [[ ! -d $WEIGHT_DIR ]]; then
1516
echo "$WEIGHT_DIR already exists but is not a directory" 1>&2
1617
fi

‎ns_vfs/common/frame_grouping.py

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
def combine_consecutive_lists(data):
2+
if len(data) > 0:
3+
# Normalize data to ensure all elements are lists
4+
data = [[x] if not isinstance(x, list) else x for x in data]
5+
6+
# Sort the data based on the first element of each sublist
7+
data.sort(key=lambda x: x[0])
8+
9+
combined_lists = [data[0]]
10+
11+
for sublist in data[1:]:
12+
# Check if the last number of the previous sublist is consecutive to the first number of the current sublist
13+
if sublist[0] - combined_lists[-1][-1] == 1:
14+
# If the current sublist is single-item and the previous sublist is also single-item, combine them
15+
if len(sublist) == len(combined_lists[-1]) == 1:
16+
combined_lists[-1].extend(sublist)
17+
# If the current sublist is single-item but the previous sublist is multi-item, append it
18+
elif len(sublist) == 1 and len(combined_lists[-1]) > 1:
19+
combined_lists[-1].append(sublist[0])
20+
# Otherwise, start a new group
21+
else:
22+
combined_lists.append(sublist)
23+
else:
24+
combined_lists.append(sublist)
25+
26+
return combined_lists
27+
else:
28+
return []
29+
30+
31+
if __name__ == "__main__":
32+
data = [1, 2, [3], [4, 5], 9, 21]
33+
# data = [[2, 4, 6], [9, 21]]
34+
# data = [[1], [2], [3], [5], [7], [9], [10], [21]]
35+
print(combine_consecutive_lists(data))

0 commit comments

Comments
 (0)