Add files via upload

Author: ShanL03

gcn_layers.py

@@ -0,0 +1,249 @@
+import os
+import cv2 
+import numpy as np
+# import random
+import imutils
+import copy
+# import tensorflow.compat.v1 as tf
+# np.random.seed(0)
+
+# #########
+# def _bytes_feature(value):
+#     return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+# def gen_tf_ex(imgs,segs,ex_name,gt_valid_id,writer,size,img_num):
+#     W,H = size
+
+#     concat_view = np.concatenate(imgs,axis=1)
+#     if segs is not None:
+#         concat_seg = np.concatenate(segs,axis=1)
+#     else:
+#         concat_seg = np.zeros((H,W*img_num,1))
+    
+#     # gt_valid_id = 0
+#     good_seg = np.zeros((H,W,1))
+    
+#     h, w, c = concat_view.shape
+#     if not (h==H and w==W*img_num and c==3): 
+#         print('STOP:',concat_view.shape,concat_depth.shape)
+#         return writer
+    
+#     # CONVERT THE VARIABLES TO THE TARGET TYPE: IMPORTANT!!!
+#     concat_view = concat_view.astype(np.uint8)
+#     good_seg = good_seg.astype(np.float32)
+#     concat_seg = concat_seg.astype(np.float32)
+#     gt_valid_id = np.array([gt_valid_id]).astype(np.int32)
+
+#     bbox_seq = np.zeros(img_num*4)
+#     # print(concat_seg.shape,concat_view.shape)
+    
+#     example = tf.train.Example(features=tf.train.Features(feature={
+#                 'image_seq': _bytes_feature(concat_view.tostring()),
+#                 'good_seg': _bytes_feature(good_seg.tostring()),
+#                 'seg_seq': _bytes_feature(concat_seg.tostring()),
+#                 'gt_valid_id': _bytes_feature(gt_valid_id.tostring()),
+#                 'bbox_seq': _bytes_feature(bbox_seq.tostring()),
+#                 # 'bbox_segs': _bytes_feature(bbox_segs.tostring()),
+#                 # 'edge_seq': _bytes_feature(edge_seq.tostring()),
+#                 'seq_name': _bytes_feature(str.encode(ex_name)),
+#                 }))
+    
+#     writer.write(example.SerializeToString())
+#     return writer
+# #########
+
+def distance(p1,p2): 
+    p1 = np.squeeze(p1)
+    p2 = np.squeeze(p2)
+    
+    diff = p1-p2
+    if len(diff.shape) == 2:
+        return np.linalg.norm(diff,axis=1)
+    else:
+        return np.linalg.norm(diff)
+
+def ext_endo_pos(img):
+    img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
+    
+    endo_pos = cv2.HoughCircles(img,cv2.HOUGH_GRADIENT,1,10,param1=50,param2=30,minRadius=100,maxRadius=150)
+    if endo_pos is None:
+        return (np.nan,np.nan,np.nan)
+    else:
+        endo_pos = endo_pos[0][0]
+        x,y,r = endo_pos
+        return (x,y,r)
+
+def genMask(endo_pos,img):
+    cx, cy, r = endo_pos
+    mask = np.zeros_like(img)
+    if np.isnan(cx):
+        return 1-mask
+    cv2.circle(mask,(int(cx),int(cy)), int(r), (1,1,1), -1)
+    return mask
+
+def shift_image(img,dx,dy):
+    rows, cols, _ = img.shape
+    M = np.float32([[1,0,dx],[0,1,dy]])
+    dst = cv2.warpAffine(img,M,(cols,rows))
+    return dst
+
+#####
+
+def postproc_image(image,resize_wh=None, nearest_interpolate=True):
+  if resize_wh is not None:
+      if nearest_interpolate:
+          image = cv2.resize(image,resize_wh,interpolation=cv2.INTER_NEAREST)
+      else:
+          image = cv2.resize(image,resize_wh)
+  return image
+
+# def inpaint_image(imagefile, labelfile, inpaint_dir, seg_length, resize_wh=None, angle_range=(30,60),shift_range=(10,30,30,80),bg_max_shift=40):
+def inpaint_image(imagefile, labelfile, inpaint_dir, seg_length, resize_wh=None, angle_range=(30,40),shift_range=(10,30,20,60),bg_max_shift=40):
+    if 'EP' in os.path.basename(imagefile):
+        angle_range=(30,40)
+        shift_range=(10,30,20,60)
+        bg_max_shift=40
+
+    sample_num = seg_length//2
+    # sample_num = 0 # $$$$$$$$$
+    # shuffle_idx = np.random.choice(seg_length,seg_length,replace=False)
+
+    image = cv2.imread(imagefile)
+
+    basename = os.path.basename(imagefile)[:-4]
+    inpaint_img = cv2.imread(os.path.join(inpaint_dir,basename+"_bg.jpg"))
+    height,width,_ = inpaint_img.shape
+    width = int(width/2)
+    inpaint_img,edge_mask = inpaint_img[:,:width],inpaint_img[:,width:]
+
+    # shift bg
+    _bg_seq = []
+    # generate bw and fw translation (max: 0~20 pixels)
+    max_shift = np.random.random()*bg_max_shift
+    dx,dy = np.random.random(2)*max_shift-max_shift/2
+    dx_list = np.linspace(-dx,dx,seg_length)
+    dy_list = np.linspace(-dy,dy,seg_length)
+    for dx,dy in zip(dx_list,dy_list):
+        shift_bg = shift_image(inpaint_img,dx,dy)
+        _bg_seq.append(shift_bg*edge_mask)
+    # # randomly shuffle bg seq
+    # shuffle_idx = np.random.choice(seg_length,seg_length,replace=False)
+    # _bg_seq = [_bg_seq[i] for i in shuffle_idx]
+
+    # cv2.imwrite(str(file_id)+"_bg_seq.jpg",np.concatenate(_bg_seq,axis=1))
+        
+    new_label = cv2.imread(os.path.join(inpaint_dir,basename+"_label.png"))
+    inpaint_inst = cv2.imread(os.path.join(inpaint_dir,basename+"_inst.jpg"))
+
+    if new_label is None:
+        # read the label image
+        label = cv2.imread(labelfile)
+        if np.sum(label)>0:
+            # print("## bad image ##",os.path.basename(imagefile))
+            return None, None, None
+
+        _bg_seq[sample_num] = image*edge_mask
+        
+        _label_seq = [np.zeros((height,width,1))]*seg_length 
+        #[np.zeros((height,width,1)) for i in range(seg_length)]
+        _new_label = label*edge_mask
+        _label_seq[sample_num] = _new_label[:,:,0][...,np.newaxis]
+        
+        _bg_seq = [postproc_image(_bg_seq[i],resize_wh=resize_wh) for i in range(seg_length)]
+        _label_seq = [postproc_image(_label_seq[i],resize_wh=resize_wh) for i in range(seg_length)]
+        # _bg_seq = [postproc_image(_bg_seq[i],resize_wh=resize_wh) for i in shuffle_idx]
+        # _label_seq = [postproc_image(_label_seq[i],resize_wh=resize_wh) for i in shuffle_idx]
+        # gt_valid_id = np.squeeze(np.argwhere(np.array(shuffle_idx)==sample_num))
+        gt_valid_id = sample_num
+        return _bg_seq, _label_seq, gt_valid_id
+
+    cv2.imwrite(os.path.join("./samples",basename+".png"),new_label/np.max(new_label)*255)
+
+    if 'EP' in os.path.basename(imagefile):
+        new_width = width*1.2
+        new_height = height*1.2
+    else:
+        new_width = width*2
+        new_height = height*2
+    # randomly rotate and shift instrument
+    if np.random.random() < 0.0:
+        _angle_range = angle_range[0]
+        _shift_min, _shift_max = shift_range[0], shift_range[1]
+    else:
+        _angle_range = angle_range[1]
+        _shift_min, _shift_max = shift_range[2], shift_range[3]
+
+    # randomly shift every instruments
+    new_label_all = copy.deepcopy(new_label)
+    inst_ids = np.unique(new_label_all[new_label_all>0])
+    _img_seq = copy.deepcopy(_bg_seq)
+    _label_seq = [np.zeros((height,width))]*seg_length
+    for inst_id in inst_ids:
+        new_label = np.zeros_like(new_label_all)
+        new_label[new_label_all==inst_id] = 1
+
+        # generate bw and fw angle (max: ranges -40~40 degree)
+        max_degree = np.random.random()*_angle_range-_angle_range/2
+        ang_bw = np.random.random()*max_degree-max_degree 
+        ang_fw = np.random.random()*max_degree
+        ang_list = list(np.linspace(ang_bw,0,sample_num+1)[:-1])+list(np.linspace(0,ang_fw,sample_num+1))
+        # generate bw and fw translation (max: 10~50 pixels)
+        max_shift = np.random.random()*(_shift_max-_shift_min)+_shift_min
+        dx,dy = np.random.random(2)*max_shift-max_shift/2
+        dx_list = np.linspace(-dx,dx,seg_length)
+        dy_list = np.linspace(-dy,dy,seg_length)
+    #
+    
+        _frame_id = 0
+        for angle,dx,dy,bg_img in zip(ang_list,dx_list,dy_list,_img_seq):
+
+            _inpaint_inst = imutils.rotate(inpaint_inst, angle)
+            _inpaint_inst = shift_image(_inpaint_inst,dx,dy)
+            # cv2.imwrite("inpaint_inst.jpg",_inpaint_inst)
+
+            _new_label = imutils.rotate(new_label, angle)
+            _new_label = shift_image(_new_label,dx,dy)
+            label_dx,label_dy = int((new_width-width)/2),int((new_height-height)/2)
+            _new_label = _new_label[label_dy:label_dy+height,label_dx:label_dx+width]
+            _new_label = _new_label*edge_mask
+            # cv2.imwrite("new_label.jpg",_new_label*255)
+
+            # if np.sum(_new_label[:,:,0]) < 8000 and np.random.random() < 0.4 and not _frame_id == sample_num:
+            #     _new_label = np.zeros_like(_new_label)
+            #     _inpaint_img = bg_img
+            # else:
+            #     _inpaint_inst = _inpaint_inst[label_dy:label_dy+height,label_dx:label_dx+width]
+            #     _smooth_new_label = cv2.GaussianBlur(_new_label.astype(np.float32),(3,3),3)
+            #     _inpaint_img = bg_img * (1-_smooth_new_label) + _inpaint_inst * _smooth_new_label
+            _inpaint_inst = _inpaint_inst[label_dy:label_dy+height,label_dx:label_dx+width]
+            _smooth_new_label = cv2.GaussianBlur(_new_label.astype(np.float32),(3,3),3)
+            _inpaint_img = bg_img * (1-_smooth_new_label) + _inpaint_inst * _smooth_new_label
+            # sample = np.concatenate([image,inpaint_img],axis=1)
+            # cv2.imwrite("inpaint_results.jpg",sample)
+
+            if _frame_id == sample_num:
+                _inpaint_img = image*edge_mask
+                _new_label = cv2.imread(labelfile)*edge_mask
+                # _new_label = _new_label*edge_mask
+            
+            if np.random.random() < 0.6:
+                _inpaint_img = np.clip(_inpaint_img + np.random.randint(10,60)*1.0,0.,255.)
+                _inpaint_img = _inpaint_img*edge_mask
+            _img_seq[_frame_id] = _inpaint_img#*final_edge_mask
+
+            _new_label = _new_label[:,:,0] 
+            _new_label = _new_label+_label_seq[_frame_id]
+            _new_label[_new_label>0]=1
+            _label_seq[_frame_id] = _new_label#*final_edge_mask[:,:,0][...,np.newaxis]
+
+            # if np.sum((np.mean(_inpaint_img,axis=2)*_new_label[:,:,0])>240)/(np.sum(_new_label[:,:,0])+0.00000001) >= 0.6:
+            #     save_seq += 1
+            _frame_id+=1
+
+    # randomly shuffle img and seg seq
+    _img_seq = [postproc_image(_img_seq[i],resize_wh=resize_wh) for i in range(seg_length)]
+    _label_seq = [postproc_image(_label_seq[i][...,np.newaxis],resize_wh=resize_wh) for i in range(seg_length)]
+    # _img_seq = [postproc_image(_img_seq[i],resize_wh=resize_wh) for i in shuffle_idx]
+    # _label_seq = [postproc_image(_label_seq[i],resize_wh=resize_wh) for i in shuffle_idx]
+    # gt_valid_id = np.squeeze(np.argwhere(np.array(shuffle_idx)==sample_num))
+    gt_valid_id = sample_num
+    return _img_seq, _label_seq, gt_valid_id

image_inpaint.py

@@ -0,0 +1,19 @@
+[{
+    "name": "background-tissue",
+    "color": [
+      0,
+      0,
+      0
+    ],
+    "classid": 0
+  },
+  {
+    "name": "instrument",
+    "color": [
+      0,
+      255,
+      0
+    ],
+    "classid": 1
+  }
+]

labels.json

@@ -0,0 +1,170 @@
+import math
+import numpy as np 
+import tensorflow as tf
+
+from tensorflow.python.framework import ops
+
+from utils import *
+
+
+image_summary = tf.summary.image
+scalar_summary = tf.summary.scalar
+histogram_summary = tf.summary.histogram
+merge_summary = tf.summary.merge
+SummaryWriter = tf.summary.FileWriter
+seed = 23
+
+def batchnorm(input_,is_train=False,name="batchnorm"):
+    with tf.variable_scope(name):
+        normalized = tf.layers.batch_normalization(input_, training=is_train)
+        return normalized
+
+def conv2d(input_, output_dim, ksize=3, stride=2, stddev=0.02,name="conv2d"):
+  with tf.variable_scope(name):
+    w = tf.get_variable('w', [ksize, ksize, input_.get_shape()[-1], output_dim],
+              initializer=tf.truncated_normal_initializer(stddev=stddev, seed=seed))
+              
+    conv = tf.nn.conv2d(input_, w, strides=[1, stride, stride, 1], padding='SAME')
+
+    biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0))
+#    conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape())
+    conv = tf.nn.bias_add(conv, biases)
+
+    return conv
+
+def conv2d_dilated(input_, output_dim, ksize=3, rate=2, stddev=0.02,name="conv2d_dilated"):
+  with tf.variable_scope(name):
+    w = tf.get_variable('w', [ksize, ksize, input_.get_shape()[-1], output_dim],
+              initializer=tf.truncated_normal_initializer(stddev=stddev, seed=seed))
+              
+    conv = tf.nn.atrous_conv2d(input_,w,rate=rate,padding="SAME")
+
+
+    biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0))
+#    conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape())
+    conv = tf.nn.bias_add(conv, biases)
+
+    return conv
+
+def deconv2d(input_, output_shape,
+       ksize=5, stride=2, stddev=0.02,
+       name="deconv2d", with_w=False):
+  with tf.variable_scope(name):
+    # filter : [height, width, output_channels, in_channels]
+    w = tf.get_variable('w', [ksize, ksize, output_shape[-1], input_.get_shape()[-1]],
+              initializer=tf.truncated_normal_initializer(stddev=stddev, seed=seed))
+    
+    try:
+      deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape,
+                strides=[1, stride, stride, 1])
+
+    # Support for verisons of TensorFlow before 0.7.0
+    except AttributeError:
+      deconv = tf.nn.deconv2d(input_, w, output_shape=output_shape,
+                strides=[1, stride, stride, 1])
+
+    biases = tf.get_variable('biases', [output_shape[-1]], initializer=tf.constant_initializer(0.0))
+    deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape())
+
+    if with_w:
+      return deconv, w, biases
+    else:
+      return deconv
+     
+def lrelu(x, leak=0.2, name="lrelu"):
+  return tf.maximum(x, leak*x)
+  
+def prelu(x, name="prelu"):
+  with tf.variable_scope(name):
+    alpha = tf.get_variable("prelu", shape=x.get_shape()[-1], initializer=tf.constant_initializer(0.0))
+    return tf.maximum(0.0, x) + alpha * tf.minimum(0.0, x)
+    
+    
+def relu(x, name="relu"):
+  return tf.maximum(x, 0)
+  
+def separable_conv2d(input_, output_dim, ksize=3, stride=1,rate=1, stddev=0.02,name=''):
+    with tf.variable_scope(name+"_separable_conv2d"):
+        in_chns = input_.get_shape()[3].value
+        w_depth = tf.get_variable('w_depth', [ksize,ksize,in_chns,1],initializer=tf.truncated_normal_initializer(stddev=stddev, seed=seed))
+        w_point = tf.get_variable('w_point', [1,1,in_chns,output_dim],initializer=tf.truncated_normal_initializer(stddev=stddev, seed=seed))
+        conv = tf.nn.separable_conv2d( input_,
+                        depthwise_filter = w_depth,
+                        pointwise_filter = w_point,
+                        strides = [1,stride,stride,1],
+                        padding="SAME",
+                        rate=[rate,rate],
+                        name="sep_conv")
+        biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0))
+        output = tf.nn.bias_add(conv, biases)
+
+    return output
+
+# do_batchnorm=True
+def atrous_spatial_pyramid_pooling(input_, output_stride=16, depth=256,is_train=False,dropout=False,keep_prob=1.0):
+  """Atrous Spatial Pyramid Pooling.
+  Args:
+    inputs: A tensor of size [batch, height, width, channels].
+    output_stride: The ResNet unit's stride. Determines the rates for atrous convolution.
+      the rates are (6, 12, 18) when the stride is 16, and doubled when 8.
+
+    depth: The depth of the ResNet unit output.
+  Returns:
+    The atrous spatial pyramid pooling output.
+  """
+  with tf.variable_scope("aspp"):
+
+    atrous_rates = [2,4]#[6, 12, 18]
+    if output_stride == 8:
+      atrous_rates = [2*rate for rate in atrous_rates]
+
+    # (a) one 1x1 convolution and three 3x3 convolutions with rates = (6, 12, 18) when output stride = 16.
+    # the rates are doubled when output stride = 8.
+    h1 = conv2d(input_, depth, ksize=1, stride=1, name="conv1")
+    h1 = tf.nn.relu(batchnorm(h1,is_train,'bn1'))
+    # if do_batchnorm:
+    #   h1 = tf.nn.relu(batchnorm(h1,is_train,'bn1'))
+    # else:
+    #   h1 = tf.nn.relu(h1)
+    
+    h2 = conv2d_dilated(input_, depth, ksize=3,rate=atrous_rates[0], name="conv3_1")
+    h2 = tf.nn.relu(batchnorm(h2,is_train,'bn2'))
+    # if do_batchnorm:
+    #   h2 = tf.nn.relu(batchnorm(h2,is_train,'bn2'))
+    # else:
+    #   h2 = tf.nn.relu(h2)
+    
+    h3 = conv2d_dilated(input_, depth, ksize=3,rate=atrous_rates[1], name="conv3_2")
+    h3 = tf.nn.relu(batchnorm(h3,is_train,'bn3'))
+    # if do_batchnorm:
+    #   h3 = tf.nn.relu(batchnorm(h3,is_train,'bn3'))
+    # else:
+    #   h3 = tf.nn.relu(h3)
+    
+    # (b) the image-level features
+    input_size = tf.shape(input_)[1:3]
+    h0 = tf.reduce_mean(input_, [1, 2], name='global_average_pooling', keepdims=True)
+    h0 = conv2d(h0, depth, ksize=1, stride=1, name="conv1_pool")
+    h0 = tf.nn.relu(batchnorm(h0,is_train,'bn_gap'))
+    # if do_batchnorm:
+    #   h0 = tf.nn.relu(batchnorm(h0,is_train,'bn_gap'))
+    # else:
+    #   h0 = tf.nn.relu(h0)
+    h0 = tf.image.resize_bilinear(h0, input_size, name='upsample')
+    
+    
+    h = tf.concat([h0,h1,h2,h3],axis=3)
+
+    h = conv2d(h, depth, ksize=1, stride=1, name="conv1_out")
+    h = tf.nn.relu(batchnorm(h,is_train,'bn_out'))
+    # if do_batchnorm:
+    #   h = tf.nn.relu(batchnorm(h,is_train,'bn_out'))
+    # else:
+    #   h = tf.nn.relu(h)
+    
+    if dropout:
+        h = tf.nn.dropout(h,keep_prob,seed=seed) 
+    
+    return h
+    
+

mobilenetv1.py

@@ -0,0 +1,10 @@
+# run: conda env create -f resources/env.yml
+# to update existing environment: conda env update -f resources/env.yml
+name: mffa
+channels:
+  - conda-forge
+dependencies:
+  - tensorflow-gpu=1.13
+  - cudatoolkit
+  - opencv
+  - imutils

model.py

@@ -0,0 +1,135 @@
+import tensorflow as tf
+
+class ConvLSTMCell(tf.nn.rnn_cell.RNNCell):
+  """A LSTM cell with convolutions instead of multiplications.
+
+  Reference:
+    Xingjian, S. H. I., et al. "Convolutional LSTM network: A machine learning approach for precipitation nowcasting." Advances in Neural Information Processing Systems. 2015.
+  """
+
+  def __init__(self, shape, filters, kernel, forget_bias=1.0, activation=tf.tanh, normalize=True, peephole=True, data_format='channels_last', reuse=None):
+    super(ConvLSTMCell, self).__init__(_reuse=reuse)
+    self._kernel = kernel
+    self._filters = filters
+    self._forget_bias = forget_bias
+    self._activation = activation
+    self._normalize = normalize
+    self._peephole = peephole
+    if data_format == 'channels_last':
+        self._size = tf.TensorShape(shape + [self._filters])
+        self._feature_axis = self._size.ndims
+        self._data_format = None
+    elif data_format == 'channels_first':
+        self._size = tf.TensorShape([self._filters] + shape)
+        self._feature_axis = 0
+        self._data_format = 'NC'
+    else:
+        raise ValueError('Unknown data_format')
+
+  @property
+  def state_size(self):
+    return tf.nn.rnn_cell.LSTMStateTuple(self._size, self._size)
+
+  @property
+  def output_size(self):
+    return self._size
+
+  def call(self, x, state):
+    c, h = state
+
+    x = tf.concat([x, h], axis=self._feature_axis)
+    n = x.shape[-1].value
+    m = 4 * self._filters if self._filters > 1 else 4
+    W = tf.get_variable('kernel', self._kernel + [n, m])
+    y = tf.nn.convolution(x, W, 'SAME', data_format=self._data_format)
+    if not self._normalize:
+      y += tf.get_variable('bias', [m], initializer=tf.zeros_initializer())
+    j, i, f, o = tf.split(y, 4, axis=self._feature_axis)
+
+    if self._peephole:
+      i += tf.get_variable('W_ci', c.shape[1:]) * c
+      f += tf.get_variable('W_cf', c.shape[1:]) * c
+
+    if self._normalize:
+      j = tf.contrib.layers.layer_norm(j)
+      i = tf.contrib.layers.layer_norm(i)
+      f = tf.contrib.layers.layer_norm(f)
+
+    f = tf.sigmoid(f + self._forget_bias)
+    i = tf.sigmoid(i)
+    c = c * f + i * self._activation(j)
+
+    if self._peephole:
+      o += tf.get_variable('W_co', c.shape[1:]) * c
+
+    if self._normalize:
+      o = tf.contrib.layers.layer_norm(o)
+      c = tf.contrib.layers.layer_norm(c)
+
+    o = tf.sigmoid(o)
+    h = o * self._activation(c)
+
+    state = tf.nn.rnn_cell.LSTMStateTuple(c, h)
+
+    return h, state
+
+
+class ConvGRUCell(tf.nn.rnn_cell.RNNCell):
+  """A GRU cell with convolutions instead of multiplications."""
+
+  def __init__(self, shape, filters, kernel, activation=tf.tanh, normalize=True, data_format='channels_last', reuse=None):
+    super(ConvGRUCell, self).__init__(_reuse=reuse)
+    self._filters = filters
+    self._kernel = kernel
+    self._activation = activation
+    self._normalize = normalize
+    if data_format == 'channels_last':
+        self._size = tf.TensorShape(shape + [self._filters])
+        self._feature_axis = self._size.ndims
+        self._data_format = None
+    elif data_format == 'channels_first':
+        self._size = tf.TensorShape([self._filters] + shape)
+        self._feature_axis = 0
+        self._data_format = 'NC'
+    else:
+        raise ValueError('Unknown data_format')
+
+  @property
+  def state_size(self):
+    return self._size
+
+  @property
+  def output_size(self):
+    return self._size
+
+  def call(self, x, h):
+    channels = x.shape[self._feature_axis].value
+
+    with tf.variable_scope('gates'):
+      inputs = tf.concat([x, h], axis=self._feature_axis)
+      n = channels + self._filters
+      m = 2 * self._filters if self._filters > 1 else 2
+      W = tf.get_variable('kernel', self._kernel + [n, m])
+      y = tf.nn.convolution(inputs, W, 'SAME', data_format=self._data_format)
+      if self._normalize:
+        r, u = tf.split(y, 2, axis=self._feature_axis)
+        r = tf.contrib.layers.layer_norm(r)
+        u = tf.contrib.layers.layer_norm(u)
+      else:
+        y += tf.get_variable('bias', [m], initializer=tf.ones_initializer())
+        r, u = tf.split(y, 2, axis=self._feature_axis)
+      r, u = tf.sigmoid(r), tf.sigmoid(u)
+
+    with tf.variable_scope('candidate'):
+      inputs = tf.concat([x, r * h], axis=self._feature_axis)
+      n = channels + self._filters
+      m = self._filters
+      W = tf.get_variable('kernel', self._kernel + [n, m])
+      y = tf.nn.convolution(inputs, W, 'SAME', data_format=self._data_format)
+      if self._normalize:
+        y = tf.contrib.layers.layer_norm(y)
+      else:
+        y += tf.get_variable('bias', [m], initializer=tf.zeros_initializer())
+      h = u * h + (1 - u) * self._activation(y)
+
+    return h, h

ops.py

@@ -0,0 +1,2 @@
+from tensorflow.python.client import device_lib
+print(device_lib.list_local_devices())

resnetv2.py

@@ -0,0 +1,128 @@
+import os
+import numpy as np
+import cv2
+from model import *
+from utils import *
+import tensorflow as tf
+
+overall_random_seed = 23 # EP:5, sinus:23
+np.random.seed(overall_random_seed)
+tf.set_random_seed(overall_random_seed)
+
+def del_all_flags(FLAGS):
+    flags_dict = FLAGS._flags()
+    keys_list = [keys for keys in flags_dict]
+    for keys in keys_list:
+        FLAGS.__delattr__(keys)
+del_all_flags(tf.flags.FLAGS)
+tf.reset_default_graph()
+tf.set_random_seed(overall_random_seed)
+
+flags = tf.app.flags
+flags.DEFINE_integer("epoch",30, "Epoch to train [25]")
+flags.DEFINE_integer("batch_size", 16, "The size of batch images [64]")
+flags.DEFINE_integer("seed", overall_random_seed, "random seed")
+flags.DEFINE_integer("input_height", 240, "The size of image to use (will be center cropped). [108]")
+flags.DEFINE_integer("input_width", 240, "The size of image to use (will be center cropped). If None, same value as input_height [None]")
+flags.DEFINE_integer("crop_height", 192, "The size of image to crop")
+flags.DEFINE_integer("crop_width", 192, "")
+flags.DEFINE_integer("temporal_len",4,"the number of consecutive frames to input")
+
+# flags.DEFINE_string("train_dataset", "../sinus_data/cadaver", "train dataset direction")
+flags.DEFINE_string("train_dataset", "../sinus_data/syn_cadaver", "train dataset direction")
+flags.DEFINE_string("frame_dataset", "../sinus_data/cadaver/frame_dataset", "frame dataset direction")
+flags.DEFINE_string("video_dir", "../sinus_data/cadaver/videos", "train dataset direction")
+flags.DEFINE_string("datasets", "cf1cf2", "")
+
+flags.DEFINE_string("img_pattern", "*.jpg", "Glob pattern of filename of input images [*]")
+flags.DEFINE_string("label_pattern", "*.png", "Glob pattern of filename of input labels [*]")
+
+flags.DEFINE_string("checkpoint_dir", "./checkpoint", "Directory name to save the checkpoints [checkpoint]")
+flags.DEFINE_string("save_checkpoint_dir", "", "Directory name to save the checkpoints [checkpoint]")
+# flags.DEFINE_string("pretrain_dir", "../pretrain/resnet_v2_50_2017_04_14", "")
+flags.DEFINE_string("pretrain_dir", "../pretrain/mobilenet_v1_1.0_224", "")
+
+#$$$$ SL
+flags.DEFINE_string("model_type", "deeplab_mobilenet", "")#unet, deeplab_mobilenet, deeplab_resnet
+
+flags.DEFINE_integer("continue_train",0,"")
+flags.DEFINE_integer("pass_hidden",0,"")
+flags.DEFINE_integer("seq_label",0,"")
+flags.DEFINE_integer("teacher_mode",0,"")
+flags.DEFINE_integer("disable_gcn",0,"")
+
+# flags.DEFINE_integer("fold_id",0, "")
+
+flags.DEFINE_integer("rnn_mode",1, "")
+flags.DEFINE_integer("decay_epoch",15, "Epoch to decay learning rate")
+flags.DEFINE_float("learning_rate",0.000125,"")
+# flags.DEFINE_float("learning_rate",0.0000625,"")
+
+flags.DEFINE_string("gpu", '0', "gpu")
+FLAGS = flags.FLAGS
+
+
+os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu
+def main(_):
+  pp.pprint(flags.FLAGS.__flags)
+
+  if not os.path.exists(FLAGS.checkpoint_dir):
+    os.makedirs(FLAGS.checkpoint_dir)
+  if not os.path.exists(FLAGS.save_checkpoint_dir) and not FLAGS.save_checkpoint_dir=="":
+    os.makedirs(FLAGS.save_checkpoint_dir)
+
+  # cvt number to bool
+  continue_train = False if FLAGS.continue_train==0 else True
+  pass_hidden = False if FLAGS.pass_hidden==0 else True
+  seq_label = False if FLAGS.seq_label==0 else True
+  teacher_mode = False if FLAGS.teacher_mode==0 else True
+  disable_gcn = False if FLAGS.disable_gcn==0 else True
+
+  color_table = load_color_table('./labels.json')
+  
+  #gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333)
+  # run_config = tf.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
+  run_config = tf.ConfigProto()
+  run_config.gpu_options.allow_growth=True
+  tf.reset_default_graph()
+  tf.set_random_seed(overall_random_seed)
+  with tf.Session(config=run_config) as sess:
+
+    net = DeepLab(
+          sess,
+          input_width=FLAGS.input_width,
+          input_height=FLAGS.input_height,
+          crop_width=FLAGS.crop_width,
+          crop_height=FLAGS.crop_height,
+          batch_size=FLAGS.batch_size,
+          seed=FLAGS.seed,
+          temporal_len=FLAGS.temporal_len,
+          img_pattern=FLAGS.img_pattern,
+          label_pattern=FLAGS.label_pattern,
+          checkpoint_dir=FLAGS.checkpoint_dir,
+          save_checkpoint_dir=FLAGS.save_checkpoint_dir,
+          pretrain_dir=FLAGS.pretrain_dir,
+          datasets=FLAGS.datasets,
+          train_dataset=FLAGS.train_dataset,
+          frame_dataset=FLAGS.frame_dataset,
+          video_dir=FLAGS.video_dir,
+          continue_train=continue_train, ###
+          pass_hidden=pass_hidden,
+          seq_label=seq_label,
+          teacher_mode=teacher_mode,
+          disable_gcn=disable_gcn,
+          model_type=FLAGS.model_type,
+          rnn_mode=FLAGS.rnn_mode,
+          learning_rate=FLAGS.learning_rate, 
+          # fold_id=FLAGS.fold_id, ###
+          num_class=2,
+          color_table=color_table,
+          test_video=False,is_train=True)
+
+    net.train(FLAGS)
+      
+      
+
+    
+if __name__ == '__main__':
+  tf.app.run()

resources/env.yml

@@ -0,0 +1,310 @@
+"""
+Some codes from https://github.com/Newmu/dcgan_code
+"""
+from __future__ import division
+import math
+import json
+import random
+import pprint
+import scipy.misc
+import numpy as np
+from time import gmtime, strftime
+from six.moves import xrange
+import cv2
+import os
+from image_inpaint import *
+
+import tensorflow as tf
+import tensorflow.contrib.slim as slim
+pp = pprint.PrettyPrinter()
+
+get_stddev = lambda x, k_h, k_w: 1/math.sqrt(k_w*k_h*x.get_shape()[-1])
+
+
+def load_color_table(json_file):
+    # load color table
+    f= open(json_file, "r", encoding='utf-8')
+    colors = json.loads(f.read())
+    class_num=len(colors)
+    R,G,B=[[],[],[]]
+    for c in colors:
+        R.append(c['color'][0])
+        G.append(c['color'][1])
+        B.append(c['color'][2])
+    return [R,G,B]        
+    
+def idxmap2colormap(im_idx,color_table):
+    R,G,B = color_table
+    class_num = len(R)
+    imR = np.zeros_like(im_idx,np.uint8)
+    imG = np.zeros_like(im_idx,np.uint8)
+    imB = np.zeros_like(im_idx,np.uint8)
+    for i in range(class_num):
+        imR[im_idx==i]=R[i]
+        imG[im_idx==i]=G[i]
+        imB[im_idx==i]=B[i]
+    imcolor = np.dstack((imR,imG,imB))
+    return imcolor
+
+def show_all_variables():
+  model_vars = tf.trainable_variables()
+  slim.model_analyzer.analyze_vars(model_vars, print_info=True)
+
+def save_images(images, size, image_path):
+  return imsave(images, size, image_path)
+
+def imread(path,resize_wh=None, nearest_interpolate=False, grayscale = False):
+#   print("#######",path)
+  image = cv2.imread(path)
+  if grayscale and image.shape[2]>0:
+      image = image[:,:,0]
+  if resize_wh is not None:
+      if nearest_interpolate:
+          image = cv2.resize(image,resize_wh,interpolation=cv2.INTER_NEAREST)
+      else:
+          image = cv2.resize(image,resize_wh)
+  return image
+
+# read from folder
+def sequence_read(path_train, dir_frame, temporal_len, interval=2, resize_wh=None, nearest_interpolate=False, grayscale = False):
+    file = os.path.basename(path_train)
+    vname,idx = file[:-4].split('_')
+    if(os.path.exists(path_train)):
+        frames=[imread(path_train, resize_wh, nearest_interpolate, grayscale)]  
+    else:
+        path_train = os.path.join(dir_frame,vname+'_'+idx+file[-4:])
+        frames=[imread(path_train, resize_wh, nearest_interpolate, grayscale)] 
+    # print("$$$$$$$0",len(frames),path_train,temporal_len)
+    for t in range(1,temporal_len):
+        idxt = str(int(idx)-interval*t)
+        patht = os.path.join(dir_frame,vname+'_'+idxt+file[-4:])
+        if(os.path.exists(patht)):
+            img = imread(patht, resize_wh, nearest_interpolate, grayscale)
+            frames.append(img)
+        else:
+            # print("iamhere",t)
+            break
+    # print("$$$$$$$1",patht)
+    # print("$$$$$$$0",len(frames),vname+'_'+idxt+file[-4:],file)
+    if len(frames) == temporal_len:
+        # print("&&&&&&good")
+        return frames
+    else:
+        # print("&&&&&&bad")
+        interval = -interval
+        frames=[imread(path_train, resize_wh, nearest_interpolate, grayscale)]  
+        for t in range(1,temporal_len):
+            idxt = str(int(idx)-interval*t)
+            patht = os.path.join(dir_frame,vname+'_'+idxt+file[-4:])
+            if(os.path.exists(patht)):
+                img = imread(patht, resize_wh, nearest_interpolate, grayscale)
+                frames.append(img)
+        if len(frames) == temporal_len:
+            return frames
+        else:
+            return None
+
+def full_sequence_read(imgfile, labelfile, temporal_len, resize_wh=None, nearest_interpolate=False, grayscale = False):
+    if "EP" in os.path.basename(imgfile):
+        inpaint_dir = "./get_miccai_dataset/inpaint_images"
+    else:
+        inpaint_dir = "../sinus_data/cadaver/inpaint_images"
+    frames, labels, gt_valid_id = inpaint_image(imgfile, labelfile, inpaint_dir, temporal_len, resize_wh=resize_wh)
+    return frames, labels
+
+# def full_sequence_read(imgfile, labelfile, temporal_len, resize_wh=None):
+#     syn_path = "./syn_images"
+#     _imgfile = os.path.join(syn_path,os.path.basename(imgfile))
+#     _labelfile = os.path.join(syn_path,os.path.basename(labelfile))
+#     # print(cv2.imread(_imgfile).shape,cv2.imread(_labelfile,0).shape)
+#     frames = np.reshape(cv2.imread(_imgfile),(temporal_len,resize_wh[1],resize_wh[0],3))
+#     labels = np.reshape(cv2.imread(_labelfile,0),(temporal_len,resize_wh[1],resize_wh[0]))
+#     # cv2.imwrite(os.path.join("./samples",os.path.basename(imgfile)),np.concatenate(frames,axis=0))
+#     # cv2.imwrite(os.path.join("./samples",os.path.basename(labelfile)),np.concatenate(labels,axis=0)*255)
+#     return frames, labels
+
+def imsave(images, size, path):
+  image = np.squeeze(merge(images, size))
+  return scipy.misc.imsave(path, image)
+
+def center_crop(x, crop_h, crop_w,
+                resize_h=64, resize_w=64):
+  if crop_w is None:
+    crop_w = crop_h
+  h, w = x.shape[:2]
+  j = int(round((h - crop_h)/2.))
+  i = int(round((w - crop_w)/2.))
+  return scipy.misc.imresize(
+      x[j:j+crop_h, i:i+crop_w], [resize_h, resize_w])
+
+def evaluate_seg_result(result_path, label_path, save_name='test_rst.txt', cum_time=None):
+    dices = []
+    ious = []
+    ct_dices = []
+    ct_ious = []
+    names=[]
+    # files=os.listdir(label_path)
+    files=os.listdir(result_path)
+    for file in files:
+        if not file.endswith(".png"):
+            continue
+        
+        #    
+        gt = cv2.imread(os.path.join(label_path,file))
+         
+        gt = gt[:,:,0]
+
+        if 'EP' in file:
+            gt[gt>0]=1
+
+        ## coutour loss
+        contour_mask = np.zeros_like(gt)
+        try:        
+            contours,_ = cv2.findContours(gt*255,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)
+        except:
+            _,contours,_ = cv2.findContours(gt*255,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)
+        cv2.drawContours(contour_mask,contours,-1,(1,1,1),20)
+        #
+        ct_gt = gt*contour_mask
+        
+        #
+        output = cv2.imread(os.path.join(result_path,file))
+        output = cv2.resize(output,(gt.shape[1],gt.shape[0]),interpolation=cv2.INTER_NEAREST)
+    
+        output=output[:,:,1]/255
+        #
+        ct_output = output*contour_mask
+    
+
+        #
+        if (np.count_nonzero(output)+np.count_nonzero(gt)) is 0:
+            dice = 1
+            iou = 1
+        else:
+            dice = (2*np.count_nonzero(gt*output))/(np.count_nonzero(output)+np.count_nonzero(gt)+0.000001) 
+                
+            iou = np.count_nonzero(gt*output)/(np.count_nonzero(output+gt)+0.000001)
+        #
+        if (np.count_nonzero(ct_output)+np.count_nonzero(ct_gt)) is 0:
+            ct_dice = 1
+            ct_iou = 1
+        else:
+            ct_dice = (2*np.count_nonzero(ct_gt*ct_output))/(np.count_nonzero(ct_output)+np.count_nonzero(ct_gt)+0.000001) 
+                
+            ct_iou = np.count_nonzero(ct_gt*ct_output)/(np.count_nonzero(ct_output+ct_gt)+0.000001)
+        
+        
+        
+        dices.append(dice)
+        ious.append(iou)
+        ct_dices.append(ct_dice)
+        ct_ious.append(ct_iou)
+        names.append(file[:-4])
+    
+
+    mean_dice = np.mean(dices)
+    mean_iou = np.mean(ious)
+    ct_mean_dice = np.mean(ct_dices)
+    ct_mean_iou = np.mean(ct_ious)
+
+    mean_time = np.mean(cum_time)
+    num_time = len(cum_time)
+    
+    print("mean_dice={},mean_iou={},ct_mean_dice={},ct_mean_iou={}".format(mean_dice,mean_iou,ct_mean_dice,ct_mean_iou))
+    print("mean time: {}ms".format(mean_time))
+    file = open(save_name, 'w')
+    file.write("mean_dice={},mean_iou={},ct_mean_dice={},ct_mean_iou={},mean_time={},num_time={}\n".format(mean_dice,mean_iou,ct_mean_dice,ct_mean_iou,mean_time,num_time))
+    file.close()
+
+
+def bilinear_sampler(imgs, coords):
+  """Construct a new image by bilinear sampling from the input image.
+
+  Points falling outside the source image boundary have value 0.
+
+  Args:
+    imgs: source image to be sampled from [batch, height_s, width_s, channels]
+    coords: coordinates of source pixels to sample from [batch, height_t,
+      width_t, 2]. height_t/width_t correspond to the dimensions of the output
+      image (don't need to be the same as height_s/width_s). The two channels
+      correspond to x and y coordinates respectively.
+  Returns:
+    A new sampled image [batch, height_t, width_t, channels]
+  """
+  def _repeat(x, n_repeats):
+    rep = tf.transpose(
+        tf.expand_dims(tf.ones(shape=tf.stack([
+            n_repeats,
+        ])), 1), [1, 0])
+    rep = tf.cast(rep, 'float32')
+    x = tf.matmul(tf.reshape(x, (-1, 1)), rep)
+    return tf.reshape(x, [-1])
+
+  with tf.name_scope('image_sampling'):
+    coords_x, coords_y = tf.split(coords, [1, 1], axis=3)
+    inp_size = imgs.get_shape()
+    coord_size = coords.get_shape()
+    out_size = coords.get_shape().as_list()
+    out_size[3] = imgs.get_shape().as_list()[3]
+
+    coords_x = tf.cast(coords_x, 'float32')
+    coords_y = tf.cast(coords_y, 'float32')
+
+    x0 = tf.floor(coords_x)
+    x1 = x0 + 1
+    y0 = tf.floor(coords_y)
+    y1 = y0 + 1
+
+    y_max = tf.cast(tf.shape(imgs)[1] - 1, 'float32')
+    x_max = tf.cast(tf.shape(imgs)[2] - 1, 'float32')
+    zero = tf.zeros([1], dtype='float32')
+
+    x0_safe = tf.clip_by_value(x0, zero, x_max)
+    y0_safe = tf.clip_by_value(y0, zero, y_max)
+    x1_safe = tf.clip_by_value(x1, zero, x_max)
+    y1_safe = tf.clip_by_value(y1, zero, y_max)
+
+    ## bilinear interp weights, with points outside the grid having weight 0
+    wt_x0 = (x1 - coords_x) * tf.cast(tf.equal(x0, x0_safe), 'float32')
+    wt_x1 = (coords_x - x0) * tf.cast(tf.equal(x1, x1_safe), 'float32')
+    wt_y0 = (y1 - coords_y) * tf.cast(tf.equal(y0, y0_safe), 'float32')
+    wt_y1 = (coords_y - y0) * tf.cast(tf.equal(y1, y1_safe), 'float32')
+
+    ## indices in the flat image to sample from
+    dim2 = tf.cast(inp_size[2], 'float32')
+    dim1 = tf.cast(inp_size[2] * inp_size[1], 'float32')
+    base = tf.reshape(
+        _repeat(
+            tf.cast(tf.range(coord_size[0]), 'float32') * dim1,
+            coord_size[1] * coord_size[2]),
+        [out_size[0], out_size[1], out_size[2], 1])
+
+    base_y0 = base + y0_safe * dim2
+    base_y1 = base + y1_safe * dim2
+    idx00 = tf.reshape(x0_safe + base_y0, [-1])
+    idx01 = x0_safe + base_y1
+    idx10 = x1_safe + base_y0
+    idx11 = x1_safe + base_y1
+
+    ## sample from imgs
+    #import pdb;pdb.set_trace()
+    imgs_flat = tf.reshape(imgs, tf.stack([-1, inp_size[3]]))
+    imgs_flat = tf.cast(imgs_flat, 'float32')
+    im00 = tf.reshape(tf.gather(imgs_flat, tf.cast(idx00, 'int32')), out_size)
+    im01 = tf.reshape(tf.gather(imgs_flat, tf.cast(idx01, 'int32')), out_size)
+    im10 = tf.reshape(tf.gather(imgs_flat, tf.cast(idx10, 'int32')), out_size)
+    im11 = tf.reshape(tf.gather(imgs_flat, tf.cast(idx11, 'int32')), out_size)
+
+    w00 = wt_x0 * wt_y0
+    w01 = wt_x0 * wt_y1
+    w10 = wt_x1 * wt_y0
+    w11 = wt_x1 * wt_y1
+
+    output = tf.add_n([
+        w00 * im00, w01 * im01,
+        w10 * im10, w11 * im11
+    ])
+
+    wmask = w00+w01+w10+w11
+
+    return output,wmask