various improvements

.dockerignore

@@ -0,0 +1 @@
+venv

Dockerfile

@@ -8,8 +8,9 @@ RUN apt-get update && apt-get install -y git ninja-build libglib2.0-0 libsm6 lib
 RUN apt-get update && apt-get install -y ffmpeg libsm6 libxext6
 RUN pip install cython --no-cache-dir
 RUN mkdir /cocodemo
-ADD . /cocodemo
+ADD environment.yml requirements.txt coco_explorer.py cocoinspector.py config.py pycoco.py vis.py /cocodemo/
 WORKDIR /cocodemo
 RUN conda install -y pytorch torchvision torchaudio cpuonly -c pytorch
 RUN pip install -r requirements.txt
-RUN git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI & pip install easyimages
+EXPOSE 8501
+ENTRYPOINT ["/opt/conda/bin/streamlit", "run", "coco_explorer.py", "--"]

README.MD

@@ -1,36 +1,44 @@
 ### What is this
 
-This tool given a coco annotations file and coco predictions file will let you explore your dataset, visualize results
+This tool given a COCO annotations file and COCO predictions file will let you explore your dataset, visualize results
 and calculate important metrics.
 
 
 ### Running the explorer on example data 
 
-You can use the predictions i prepared and explore the results on the coco validation dataset
-the predictions are coming from a Mask RCNN model trained with mmdetection.
+You can use the predictions I prepared and explore the results on the COCO validation dataset.
+The predictions are coming from a Mask R-CNN model trained with mmdetection.
 
-1 - Download (and extract in project directory) the labels, annotations and images:
+1. Download (and extract in project directory) the labels, annotations and images:
 
 https://drive.google.com/open?id=1wxIagenNdCt_qphEe8gZYK7H2_to9QXl
 
+2. Setup using docker
 
-2 -  Setup using docker
-
-```bash
-sudo docker run -p 8501:8501 -it -v "$(pwd)"/coco_data:/coco_data i008/coco_explorer  \
-    streamlit run  coco_explorer.py -- \
+```sh
+sudo docker run -p 8501:8501 -it -v "$PWD"/coco_data:/coco_data i008/coco_explorer  \
     --coco_train /coco_data/ground_truth_annotations.json \
     --coco_predictions /coco_data/predictions.json  \
     --images_path /coco_data/images/
 ```
 
-2 - Setup using conda
-```bash
+2. Setup using conda
+```sh
 conda env update
 conda activate cocoexplorer
 streamlit run coco_explorer.py -- --coco_train ./coco_data/ground_truth_annotations.json --coco_predictions ./coco_data/predictions.json  --images_path ./coco_data/val2017/
 ```
-3 - go to localhost:8501
+
+2. Setup using pip
+
+```sh
+python3 -m venv .venv
+. .venv/bin/activate
+pip install -r requirements.txt
+streamlit run coco_explorer.py -- --coco_train ./coco_data/ground_truth_annotations.json --coco_predictions ./coco_data/predictions.json  --images_path ./coco_data/val2017/
+```
+
+3. go to http://localhost:8501
 
 
 ### Running on your own data

coco_explorer.py

@@ -1,4 +1,5 @@
 import argparse
+import os
 
 import pandas as pd
 import plotly.express as px
@@ -19,6 +20,7 @@ def get_inspector(coco_train, coco_predictions, images_path):
 
 
 def app(args):
+    st.set_page_config(layout='wide')
     st.title('COCO Explorer')
     topbox = st.sidebar.selectbox("Choose what to do ", ['inspect predictions visually',
                                                          'inspect image statistics',
@@ -32,46 +34,64 @@ def app(args):
 
         vis_options = {'true positives': 'tp',
                        'ground truth': 'gt',
+                       'false negatives': 'fn',
                        'false positives': 'fp',
                        }
 
         st.sidebar.text("""
         What to show on image
-        TP - boxes matched with GT (orange)
-        FP - boxes that did not match with GT (teal)
-        GT - ground truth annotations (green)
+        TP - results matching GT (orange)
+        FP - results not matching GT (teal)
+        FN - GT not matching results (red)
+        GT - all ground truth (green)
         """)
         ms = st.sidebar.multiselect("",
                                     list(vis_options.keys()),
                                     default=list(vis_options.keys())
                                     )
 
         st.sidebar.subheader('Visual settings')
+        size = st.sidebar.slider('plot resolution', min_value=1, max_value=50, value=15)
         score = st.sidebar.slider('score threshold', min_value=0.0, max_value=1.0, value=0.5)
 
         draw_pred_mask = st.sidebar.checkbox("Draw predictions masks (red)")
         draw_gt_mask = st.sidebar.checkbox("Draw ground truth masks (green)")
+        adjust_labels = st.sidebar.checkbox("Optimize label placement")
 
         r = st.sidebar.radio('Inspect by', options=['image_id', 'category', 'precision'])
 
         if r == 'image_id':
-            r = st.slider('slider trough all images', min_value=0, max_value=len(inspector.image_ids))
-            st.text(inspector._imageid2path(inspector.image_ids[r]))
+            path = st.text_input('select image by path:',)
+            if path:
+                r = inspector._path2imageid(path)
+                if r < 0:
+                    st.error('No such image file_name')
+                    r = 0
+                else:
+                    r = inspector.image_ids.index(r)
+            else:
+                r = 0
+            r = st.slider('slider trough all images', value=r, min_value=0, max_value=len(inspector.image_ids))
+            path = inspector._imageid2path(inspector.image_ids[r])
+            st.text(path)
+            print(path)
             f, fn = inspector.visualize_image(inspector.image_ids[r],
                                               draw_gt_mask=draw_gt_mask,
                                               draw_pred_mask=draw_pred_mask,
+                                              adjust_labels=adjust_labels,
                                               score_threshold=score,
-                                              fontsize=33,
+                                              fontsize=size,
                                               show_only=[vis_options[o] for o in ms],
-                                              figsize=(15, 15))
-
-            st.image(fn, use_column_width=True)
+                                              figsize=(size, size))
+            st.pyplot(f[0])
             imscores = inspector.image_scores_agg
-            st.dataframe(imscores.loc[inspector.image_ids[r]])
+            if inspector.image_ids[r] in imscores.index:
+                st.dataframe(imscores.loc[inspector.image_ids[r]])
 
         if r == 'category':
             category = st.sidebar.selectbox(label='select by category',
                                             options=[c['name'] for c in inspector.categories])
+            exclusive = st.sidebar.checkbox(label='Show only this category')
             print(category)
             if category:
                 random_ids = inspector.get_random_images_with_category(category)
@@ -80,10 +100,11 @@ def app(args):
                     f, fn = inspector.visualize_image(id,
                                                       draw_gt_mask=draw_gt_mask,
                                                       draw_pred_mask=draw_pred_mask,
+                                                      only_categories=[category] if exclusive else [],
                                                       score_threshold=score,
                                                       show_only=[vis_options[o] for o in ms],
-                                                      fontsize=30,
-                                                      figsize=(20, 20))
+                                                      fontsize=size,
+                                                      figsize=(size, size))
 
                     st.pyplot(f[0])
 
@@ -102,8 +123,8 @@ def app(args):
                                                   draw_pred_mask=draw_pred_mask,
                                                   score_threshold=score,
                                                   show_only=[vis_options[o] for o in ms],
-                                                  fontsize=30,
-                                                  figsize=(20, 20))
+                                                  fontsize=size,
+                                                  figsize=(size, size))
 
                 st.pyplot(f[0])
 
@@ -147,8 +168,13 @@ def app(args):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument("--coco_train", type=str, default=None)
-    parser.add_argument("--coco_predictions", type=str, default=None)
-    parser.add_argument("--images_path", type=str, default=None)
+    parser.add_argument("--coco_train", type=str, required=True, metavar="PATH/TO/COCO.json",
+                        help="COCO dataset to inspect")
+    parser.add_argument("--coco_predictions", type=str, required=True, metavar="PATH/TO/COCO.json",
+                        help="COCO annotations to compare to")
+    parser.add_argument("--images_path", type=str, default=os.getcwd(), metavar="PATH/TO/IMAGES/",
+                        help="Directory path to prepend to file_name paths in COCO")
     args = parser.parse_args()
+    if args.images_path[-1] != '/':
+        args.images_path += '/'
     app(args)

cocoinspector.py

@@ -108,10 +108,16 @@ def categories(self):
         return sorted(categories, key=lambda x: x['id'])
 
     @staticmethod
-    def _get_detections(coco, image_id):
-        annotations = coco.loadAnns(coco.getAnnIds(imgIds=image_id))
+    def _get_detections(coco, image_id, cat_ids=[]):
+        annotations = coco.loadAnns(coco.getAnnIds(imgIds=image_id, catIds=cat_ids))
         return annotations
 
+    def _path2imageid(self, path):
+        if path.startswith(self.base_path):
+            path = path[len(self.base_path):]
+        return next((image_id for image_id, img in self.coco_gt.imgs.items()
+                     if img['file_name'] == path), -1)
+
     def _imageid2name(self, image_id):
         return self.coco_gt.loadImgs(ids=[image_id])[0]['file_name']
 
@@ -133,18 +139,21 @@ def get_detection_matches(self, image_id):
             dtmatches = []
         return list(set(gtmatches)), list(set(dtmatches))
 
-    def organize_annotations(self, all_annotations, gtmatches):
+    def organize_annotations(self, all_annotations, gtmatches, dtmatches):
         collect = []
         for a in all_annotations:
             a['label'] = self.coco_gt.cats[a['category_id']]['name']
-            if 'score' not in a and a['id']:
-                a['type'] = 'gt'
+            if 'score' not in a:
+                if a['id'] in dtmatches:
+                    a['type'] = 'gt'
+                else:
+                    a['type'] = 'fn'
                 collect.append(a)
                 continue
-            if 'score' in a: a['type'] = 'pred'
-            if a['id'] in gtmatches: a['type'] = 'tp'
-            if a['id'] not in gtmatches: a['type'] = 'fp'
-
+            if a['id'] in gtmatches:
+                a['type'] = 'tp'
+            else:
+                a['type'] = 'fp'
             collect.append(a)
         return collect
 
@@ -153,22 +162,30 @@ def visualize_image(self, image_id,
                         score_threshold=0.1,
                         draw_gt_mask=True,
                         draw_pred_mask=True,
-                        fontsize=20,
+                        only_categories=None,
+                        adjust_labels=False,
+                        fontsize=12,
                         figsize=(10, 10),
                         dpi=200,
                         ):
-        annotations = self._get_detections(self.coco_gt, image_id)
+        annotations = self._get_detections(self.coco_gt, image_id,
+                                           cat_ids=[self.cat2id[cat] for cat in only_categories or []])
         if self.coco_dt:
-            dt_annotations = self._get_detections(self.coco_dt, image_id)
+            dt_annotations = self._get_detections(self.coco_dt, image_id,
+                                                  cat_ids=[self.cat2id[cat] for cat in only_categories or []])
             gtmatches, dtmatches = self.get_detection_matches(image_id)
             annotations = annotations + dt_annotations
-            annotations = self.organize_annotations(annotations, gtmatches)
+            annotations = self.organize_annotations(annotations, gtmatches, dtmatches)
 
         image = Image.open(self._imageid2path(image_id))
+        # cannot work with 16/32 bit or float images due to Pillow#3011 Pillow#3159 Pillow#3838
+        assert not image.mode.startswith(('I', 'F')), "image %d has unsupported color mode" % image_i
+        image = image.convert('RGB')
         f = vis_image(image, annotations,
                       show_only=show_only,
                       score_threshold=score_threshold,
                       draw_gt_mask=draw_gt_mask,
+                      adjust_labels=adjust_labels,
                       coco=self.coco_gt,
                       axis_off=False,
                       figsize=figsize,
@@ -179,6 +196,8 @@ def visualize_image(self, image_id,
         # plt.show()
         plt.draw()
         fn = 'tmpfile.png'
-        fig1.savefig(fn, dpi=dpi)
+        # savefig + st.image can be faster than st.pyplot,
+        # but only when using small dpi/resolution
+        #fig1.savefig(fn, dpi=dpi)
 
         return f, fn

requirements.txt

@@ -1,8 +1,12 @@
 albumentations
-opencv_python
+opencv-python-headless
 plotly
 numpy
 pandas
-streamlit==0.57.0
+streamlit>=0.57.0
 matplotlib
 Pillow
+easyimages
+pycocotools
+torchvision
+adjustText