Updated dockerfile, geo_coords

Author: avanetten

README.md

@@ -127,7 +127,7 @@ During the test phase, input images of arbitrary size are processed.
 			--label_map_path class_labels_car.pbtxt \
 			--train_model_path [ssd_train_path] \
 			--tf_cfg_train_file ssd_inception_v2_simrdwn.config \
-			--use_tfrecords=1 \
+			--use_tfrecords=0 \
 			--testims_dir cowc/Utah_AGRC  \
 			--keep_test_slices 0 \
 			--test_slice_sep __ \
@@ -142,7 +142,7 @@ During the test phase, input images of arbitrary size are processed.
 				
 		# YOLT vehicle search
 		python /raid/local/src/simrdwn/core/simrdwn.py \
-			--framework yolt \
+			--framework yolt2 \
 			--mode test \
 			--outname dense_cowc \
 			--label_map_path class_labels_car.pbtxt \

docker/Dockerfile

@@ -2,11 +2,6 @@
 FROM nvidia/cuda:9.0-devel-ubuntu16.04
 MAINTAINER avanetten
 
-# nvidia-docker build -t simrdwn2.1 .  # build (use existing packages)
-# nvidia-docker build --no-cache -t simrdwn2.2 .  # rebuild from scratch
-# NV_GPU=0 nvidia-docker run -it -v /local_data:/local_data --name simrdwn2.2_gpu0 simrdwn2.2
-# NV_GPU=0 nvidia-docker run -it -v /local_data:/local_data -v /cosmiq:/cosmiq --name simrdwn2.1_gpu0 simrdwn2.1
-
 # IF YOU WANT PROGRESS PRINTED TO TERMINAL
 # Update model_main to log to screen...
 # https://stackoverflow.com/questions/52016255/tensorflow-object-detection-api-not-displaying-global-steps
@@ -73,78 +68,84 @@ SHELL ["/bin/bash", "-c"]
 ENV PATH /opt/conda/bin:$PATH
 
 # install anaconda
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O ~/miniconda.sh && \
+# https://repo.continuum.io/miniconda/
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O ~/miniconda.sh && \
     /bin/bash ~/miniconda.sh -b -p /opt/conda && \
     rm ~/miniconda.sh && \
     /opt/conda/bin/conda clean -tipsy && \
     ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
     echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
     echo "conda activate base" >> ~/.bashrc
-ENV TINI_VERSION v0.16.1
+# ENV TINI_VERSION v0.16.1
+ENV TINI_VERSION v0.18.0
 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
 RUN chmod +x /usr/bin/tini
 
 # use conda-forge instead of default channel
-RUN conda update conda && \
-    conda config --remove channels defaults && \
-    conda config --add channels conda-forge
+RUN conda config --add channels conda-forge 
+	# && \
+	# conda update conda
+    # conda config --remove channels defaults && \
 
 # set up conda environment and add to $PATH
-RUN conda create -n simrdwn2 python=3.6 \
-                    && echo "source activate simrdwn2" > ~/.bashrc
-ENV PATH /opt/conda/envs/simrdwn2/bin:$PATH
+RUN conda create -n simrdwn python=3.6 \
+                    && echo "source activate simrdwn" > ~/.bashrc
+ENV PATH /opt/conda/envs/simrdwn/bin:$PATH
 
 # install GPU version of tensorflow
-RUN source activate simrdwn2 && \
-    conda install -n simrdwn2 -c defaults tensorflow-gpu=1.13.1
+RUN source activate simrdwn && \
+    conda install -n simrdwn -c defaults tensorflow-gpu=1.13.1
 
 # install keras with tf backend
 ENV KERAS_BACKEND=tensorflow
-RUN source activate simrdwn2 \
-  && conda install -n simrdwn2 keras
-
-RUN conda install -n simrdwn2 \
-	              #awscli \
-	              affine \
-	              pyproj \
+RUN source activate simrdwn \
+  && conda install -n simrdwn keras=2.2.4
+
+RUN source activate simrdwn \
+	&& conda install -n simrdwn \
+				  libgdal=2.4.1 \
+    			  gdal=2.4.1 \
+	              geopandas=0.5.0 \
+				  opencv=4.1.0 \
+	              affine=2.2.2 \
+	              pyproj=2.1.3 \
 	              pyhamcrest=1.9.0 \
-	              cython \
-				  contextlib2 \
-	              fiona \
-	              h5py \
-	              ncurses \
-	              jupyter \
-	              jupyterlab \
-	              ipykernel \
-	              libgdal \
-	              matplotlib \
-				  ncurses \
-	              numpy \
-	              #opencv=3.4.1 \
-	              #py-opencv \
-	              pandas \
-	              pillow \
-	              pip \
-	              scipy \
-	              scikit-image \
-	              scikit-learn \
-	              shapely \
-	              gdal \
-	              rtree \
-	              testpath \
-	              tqdm \
-	              pandas \
-	              geopandas \
-	              rasterio \
-				  opencv=4.0.0 \
+	              cython=0.29.10 \
+				  contextlib2=0.5.5 \
+	              h5py=2.9.0 \
+	              ncurses=6.1 \
+	              jupyter=1.0.0 \
+	              jupyterlab=0.35.6 \
+	              ipykernel=5.1.1 \
+	              matplotlib=3.1.0 \
+	              numpy=1.16.4 \
+	              pandas=0.24.2 \
+	              pillow=6.0.0 \
+	              pip=19.1.1 \
+	              scipy=1.3.0 \
+	              scikit-image=0.15.0 \
+	              scikit-learn=0.21.2 \
+	              shapely=1.6.4 \
+	              rtree=0.8.3 \
+	              testpath=0.4.2 \
+	              tqdm=4.32.1 \
+				  statsmodels=0.10.1 \
 	&& conda clean -p \
 	&& conda clean -t \
 	&& conda clean --yes --all 
 
-RUN pip install statsmodels				 
+# Below is a hack to get around conda issues (2019-07-19)
+RUN source activate simrdwn \
+    && conda install -n simrdwn \
+				-c conda-forge rasterio=1.0.24 \
+	&& conda clean -p \
+	&& conda clean -t \
+	&& conda clean --yes --all 
 
 # tf object detection api
 RUN git clone https://github.com/tensorflow/models.git /tensorflow/models
+WORKDIR /tensorflow/models/research/
+RUN git checkout r1.13.0
 #WORKDIR /tensorflow/models/research/
 #RUN protoc object_detection/protos/*.proto --python_out=.
 # WORKDIR /tensorflow/models/research/
@@ -167,8 +168,28 @@ RUN export PYTHONPATH=$PYTHONPATH:`pwd`:`pwd`/slim
 ENV PYTHONPATH $PYTHONPATH:/tensorflow/models/:/tensorflow/models/slim
 ENV PYTHONPATH $PYTHONPATH:/tensorflow/models/:/tensorflow/models/research/slim
 
+# # this always seems to fail, so do it manually (as above)
+# RUN git clone https://github.com/cocodataset/cocoapi.git /cocoapi
+# WORKDIR /cocoapi/PythonAPI
+# RUN make
+# RUN cp -r pycocotools /tensorflow/models/research/
+# # From tensorflow/models/research/
+# WORKDIR /tensorflow/models/research/
+# RUN protoc object_detection/protos/*.proto --python_out=.
+
+# # test
+# WORKDIR /tensorflow/models/research/
+# RUN PYTHONPATH=$PYTHONPATH:`pwd`:`pwd`/slim
+# RUN python /tensorflow/models/research/object_detection/builders/model_builder_test.py
+#
+# # RUN source activate simrdwn && \
+# #      python /tensorflow/models/research/object_detection/builders/model_builder_test.py
+# # # ENV MODELS_DIR /tensorflow/models/research/
+# # # PYTHONPATH=${MODELS_DIR}/research:$MODELS_DIR/research/slim \
+# # #PYTHON $MODELS_DIR/research/object_detection/builders/model_builder_test.py
+
 # # add a jupyter kernel for the conda environment in case it's wanted
-RUN source activate simrdwn2 && python -m ipykernel.kernelspec
+RUN source activate simrdwn && python -m ipykernel.kernelspec
 RUN python -m ipykernel.kernelspec
 
 

simrdwn/core/add_geo_coords.py

@@ -185,9 +185,12 @@ def add_geo_coords_to_df(df_, inProj_str='epsg:4326', outProj_str='epsg:3857',
         # raster geo transform
         gdal_geomTransform = gdal.Open(raster_loc).GetGeoTransform()
         affineObject = rio.open(raster_loc).transform
+
         if verbose:
             print("raster_loc:", raster_loc)
             print("gdal_geomTransform:", gdal_geomTransform)
+            print("gdal_geomTransform == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0))",
+                     gdal_geomTransform == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0))
             print("affineObject:", affineObject)
         geo_dict[raster_loc] = [gdal_geomTransform, affineObject]
 
@@ -204,14 +207,21 @@ def add_geo_coords_to_df(df_, inProj_str='epsg:4326', outProj_str='epsg:3857',
                   row['Xmin_Glob'], row['Ymin_Glob'],
                   row['Xmax_Glob'], row['Ymax_Glob'])
 
-        out_arr_row, poly_geo = get_row_geo_coords(
-            row, affineObject=affineObject,
-            gdal_geomTransform=gdal_geomTransform,
-            inProj_str=inProj_str, outProj_str=outProj_str,
-            verbose=verbose)
-        out_arr.append(out_arr_row)
-        if create_geojson:
-            out_arr_json.append(poly_geo)
+        # check if the the image as geographic metadata
+        if gdal_geomTransform == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0):
+            out_arr_row = 8 * [0]
+            out_arr.append(out_arr_row)
+
+        # else, get geo locations
+        else:
+            out_arr_row, poly_geo = get_row_geo_coords(
+                row, affineObject=affineObject,
+                gdal_geomTransform=gdal_geomTransform,
+                inProj_str=inProj_str, outProj_str=outProj_str,
+                verbose=verbose)
+            out_arr.append(out_arr_row)
+            if create_geojson:
+                out_arr_json.append(poly_geo)
 
     # update dataframe
     # [lon0, lat0, lon1, lat1, x0_wmp, y0_wmp, x1_wmp, y1_wmp]
@@ -227,7 +237,7 @@ def add_geo_coords_to_df(df_, inProj_str='epsg:4326', outProj_str='epsg:3857',
 
     # geodataframe if desired
     #   https://gis.stackexchange.com/questions/174159/convert-a-pandas-dataframe-to-a-geodataframe
-    if create_geojson:
+    if create_geojson and (len(out_arr_json) > 0):
         crs_init = {'init': inProj_str}
         df_json = pd.DataFrame(out_arr_json, columns=['geometry'])
         # add important columns to df_json

simrdwn/core/infer_detections.py

@@ -68,8 +68,7 @@
 tf.flags.DEFINE_string('inference_graph', None,
                        'Path to the inference graph with embedded weights.')
 tf.flags.DEFINE_boolean('verbose', False, 'Lots o print statements')
-tf.flags.DEFINE_boolean('use_tfrecords', True, 'Switch to use tfrecords')
-
+tf.flags.DEFINE_boolean('use_tfrecords', False, 'Switch to use tfrecords')
 
 # tfrecords
 tf.flags.DEFINE_string('input_tfrecord_paths', None,
@@ -97,8 +96,8 @@
 tf.flags.DEFINE_integer('BGR2RGB', 0,
                         'Sometimes we need to change cv2 images to BGR')
 
-
 FLAGS = tf.flags.FLAGS
+# print("FLAGS:", FLAGS)
 
 
 ###############################################################################
@@ -166,7 +165,7 @@ def main(_):
                                     str(t1 - t0) + ' seconds')
 
     else:
-
+        # infer on csv
         if FLAGS.verbose:
             print("min_thresh:", FLAGS.min_thresh)
         t0 = time.time()
@@ -180,7 +179,7 @@ def main(_):
                 tf.import_graph_def(graph_def, name='')
         print("Time to load graph:", time.time() - t0, "seconds")
 
-        with open(FLAGS.output_csv_path, 'wb') as csvfile:
+        with open(FLAGS.output_csv_path, 'w') as csvfile:
             csvwriter = csv.writer(csvfile, delimiter=',')
             output_columns = ['Loc_Tmp', u'Prob', u'Xmin',
                               u'Ymin', u'Xmax', u'Ymax', u'Category']
@@ -196,9 +195,19 @@ def main(_):
 
                     line_count = 0
                     for i, image_path in enumerate(image_paths):
-
                         image_root = os.path.basename(image_path).strip()
-                        image_bgr = cv2.imread(image_path.strip(), 1)
+                        # print("image_path:", image_path)
+                        # print("str image_path.strip", str(image_path.strip()))
+                        # print("image_path.strip", image_path.strip().decode('utf-8'))
+                         #print("str image_path.strip", str(image_path.strip().decode('utf-8')))
+                        # print("os.path.exists image_path.strip", os.path.exists(image_path.strip()))
+                        # print("os.path.exists str image_path.strip", os.path.exists(str(image_path.strip())))
+                        # print("os.path.exists str image_path.strip", os.path.exists(str(image_path.strip()).strip()))
+                        # print("os.path.exists str image_path.strip", os.path.exists(str(image_path.strip().decode('utf-8'))))
+
+                        # image_bgr = cv2.imread(image_path, cv2.IMREAD_COLOR)
+                        # image_bgr = cv2.imread(str(image_path.strip()), 1)
+                        image_bgr = cv2.imread(str(image_path.strip().decode('utf-8')), 1)
                         # invert colors, if required
                         if FLAGS.BGR2RGB == 1:
                             image = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)

simrdwn/core/post_process.py

@@ -548,6 +548,8 @@ def refine_df(df, groupby='Image_Path',
                     if verbose:
                         print("num boxes_all:", len(xmins))
                         print("num good_idxs:", len(good_idxs))
+                    if len(boxes) == 0:
+                        print("Error, No boxes detected!")
                     boxes = boxes[good_idxs]
                     scores = scores[good_idxs]
                     df_idxs = df_idxs[good_idxs]