From 8c1786c1233093552dd4eda902aec1d853f67f3a Mon Sep 17 00:00:00 2001 From: Vishnu Banna Date: Fri, 29 Oct 2021 12:09:13 -0400 Subject: [PATCH 01/96] decoder params --- .../vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py index 78fd63d59ba..e327f6b8698 100644 --- a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py +++ b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py @@ -371,7 +371,7 @@ def build(self, inputs): for level, depth in zip(self._iterator, self._depths): if level > 5: proc_filters = lambda x: x * 2 - resample_filters = lambda x: x // 2 + resample_filters = lambda x: x elif self._csp_stack == 0: proc_filters = lambda x: x resample_filters = lambda x: x // 2 From 2e68bf1b4225c7a3b66ffb80124a75f767dfc0d2 Mon Sep 17 00:00:00 2001 From: Vishnu Banna Date: Fri, 29 Oct 2021 12:18:37 -0400 Subject: [PATCH 02/96] decoder params --- official/vision/beta/projects/yolo/configs/yolo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index c340f026425..59679d71e19 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -483,7 +483,7 @@ def scaled_yolo() -> cfg.ExperimentConfig: 'momentum_start': 0.8, 'nesterov': True, 'warmup_steps': steps_per_epoch * warmup_epochs, - 'weight_decay': 0.0005 * train_batch_size / 64.0, + 'weight_decay': 0.0005, } }, 'learning_rate': { @@ -595,7 +595,7 @@ def large_yolo() -> cfg.ExperimentConfig: 'momentum_start': 0.9, 'nesterov': True, 'warmup_steps': steps_per_epoch * warmup_epochs, - 'weight_decay': 0.0005 * train_batch_size / 64.0, + 'weight_decay': 0.0005, } }, 'learning_rate': { From e0f852d871fba8685de393563502d131f4747da5 Mon Sep 17 00:00:00 2001 From: Vishnu Banna Date: Mon, 1 Nov 2021 14:19:44 -0400 Subject: [PATCH 03/96] model-p5 train_path --- .../experiments/scaled-yolo/detection/yolo_l_p5_896_tpu.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_l_p5_896_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_l_p5_896_tpu.yaml index 2f86e012e3e..0d5eb4efa5d 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_l_p5_896_tpu.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_l_p5_896_tpu.yaml @@ -57,7 +57,7 @@ task: box: [48,102], box: [119,96], box: [97,189], box: [217,184], box: [171,384], box: [324,451], box: [616,618], box: [800,800]] train_data: - input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/train*' + input_path: 'gs://cam2-datasets/coco/train*' shuffle_buffer_size: 10000 parser: mosaic: @@ -73,4 +73,4 @@ task: aug_rand_translate: 0.5 area_thresh: 0.1 validation_data: - input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/val*' + input_path: 'gs://cam2-datasets/coco/val*' From 018f97550af33947637c46933b0bcf7bbdab9058 Mon Sep 17 00:00:00 2001 From: Vishnu Banna Date: Thu, 4 Nov 2021 18:46:34 -0400 Subject: [PATCH 04/96] model --- .../detection-finetune/yolo_csp_640_tpu.yaml | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_csp_640_tpu.yaml diff --git a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_csp_640_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_csp_640_tpu.yaml new file mode 100644 index 00000000000..a2968362a18 --- /dev/null +++ b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_csp_640_tpu.yaml @@ -0,0 +1,79 @@ +# --experiment_type=scaled_yolo +# mAP 47.6 +runtime: + distribution_strategy: 'tpu' + mixed_precision_dtype: 'float32' + tpu_enable_xla_dynamic_padder: false +task: + model: + input_size: [640, 640, 3] + backbone: + type: 'darknet' + darknet: + model_id: 'altered_cspdarknet53' + max_level: 5 + min_level: 3 + decoder: + type: yolo_decoder + yolo_decoder: + version: v4 + type: csp + head: + smart_bias: true + detection_generator: + box_type: + 'all': scaled + scale_xy: + 'all': 2.0 + max_boxes: 300 + nms_type: iou + iou_thresh: 0.001 + nms_thresh: 0.65 + loss: + use_scaled_loss: true + update_on_repeat: true + box_loss_type: + 'all': ciou + ignore_thresh: + 'all': 0.0 + iou_normalizer: + 'all': 0.05 + cls_normalizer: + 'all': 0.3 + object_normalizer: + '5': 0.28 + '4': 0.70 + '3': 2.80 + objectness_smooth: + 'all': 1.0 + norm_activation: + use_sync_bn: true + num_classes: 80 + anchor_boxes: + anchors_per_scale: 3 + boxes: [box: [12, 16], box: [19, 36], box: [40, 28], + box: [36, 75], box: [76, 55], box: [72, 146], + box: [142, 110], box: [192, 243], box: [459, 401]] + train_data: + input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/train*' + shuffle_buffer_size: 10000 + parser: + mosaic: + mosaic_frequency: 1.0 + mixup_frequency: 0.2 + mosaic_crop_mode: 'scale' + mosaic_center: 0.25 + aug_scale_min: 0.1 + aug_scale_max: 1.9 + max_num_instances: 300 + letter_box: true + random_flip: true + aug_rand_translate: 0.1 + area_thresh: 0.1 + validation_data: + input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/val*' +trainer: + train_steps: 831600 # epoch 300 to 450 + learning_rate: + cosine: + decay_steps: 831600 # epoch 300 to 450 \ No newline at end of file From 73c186f6a3fae72e2c6a490f04bdc7577af7319c Mon Sep 17 00:00:00 2001 From: Vishnu Banna Date: Thu, 4 Nov 2021 18:57:31 -0400 Subject: [PATCH 05/96] model --- .../scaled-yolo/detection-finetune/yolo_csp_640_tpu.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_csp_640_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_csp_640_tpu.yaml index a2968362a18..6336d1cf56d 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_csp_640_tpu.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_csp_640_tpu.yaml @@ -74,6 +74,7 @@ task: input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/val*' trainer: train_steps: 831600 # epoch 300 to 450 - learning_rate: - cosine: - decay_steps: 831600 # epoch 300 to 450 \ No newline at end of file + optimizer_config: + learning_rate: + cosine: + decay_steps: 831600 # epoch 300 to 450 \ No newline at end of file From 4d87447ba64874d902e19c3f985d51b02840568a Mon Sep 17 00:00:00 2001 From: Vishnu Banna Date: Thu, 4 Nov 2021 18:59:18 -0400 Subject: [PATCH 06/96] model --- .../scaled-yolo/detection-finetune/yolo_csp_640_tpu.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_csp_640_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_csp_640_tpu.yaml index 6336d1cf56d..844c1f5b9d0 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_csp_640_tpu.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_csp_640_tpu.yaml @@ -55,7 +55,7 @@ task: box: [36, 75], box: [76, 55], box: [72, 146], box: [142, 110], box: [192, 243], box: [459, 401]] train_data: - input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/train*' + input_path: 'gs://cam2-datasets/coco/train*' shuffle_buffer_size: 10000 parser: mosaic: @@ -71,7 +71,7 @@ task: aug_rand_translate: 0.1 area_thresh: 0.1 validation_data: - input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/val*' + input_path: 'gs://cam2-datasets/coco/val*' trainer: train_steps: 831600 # epoch 300 to 450 optimizer_config: From db8580b19f0f9b05a9b20147fa544f6643fb6a37 Mon Sep 17 00:00:00 2001 From: Vishnu Banna Date: Thu, 4 Nov 2021 19:04:35 -0400 Subject: [PATCH 07/96] model --- .../scaled-yolo/detection-finetune/yolo_l_p5_896_tpu.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_l_p5_896_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_l_p5_896_tpu.yaml index 39f4434183c..72b3be6bdf9 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_l_p5_896_tpu.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_l_p5_896_tpu.yaml @@ -57,7 +57,7 @@ task: box: [48,102], box: [119,96], box: [97,189], box: [217,184], box: [171,384], box: [324,451], box: [616,618], box: [800,800]] train_data: - input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/train*' + input_path: 'gs://cam2-datasets/coco/train*' shuffle_buffer_size: 10000 parser: mosaic: @@ -73,7 +73,7 @@ task: aug_rand_translate: 0.5 area_thresh: 0.1 validation_data: - input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/val*' + input_path: 'gs://cam2-datasets/coco/val*' trainer: train_steps: 831600 # epoch 300 to 450 learning_rate: From d74920c05ef9c9632076665050e6814618887de3 Mon Sep 17 00:00:00 2001 From: Vishnu Banna Date: Thu, 4 Nov 2021 19:06:53 -0400 Subject: [PATCH 08/96] model --- .../scaled-yolo/detection-finetune/yolo_l_p5_896_tpu.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_l_p5_896_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_l_p5_896_tpu.yaml index 72b3be6bdf9..a520acf4e91 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_l_p5_896_tpu.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection-finetune/yolo_l_p5_896_tpu.yaml @@ -74,8 +74,9 @@ task: area_thresh: 0.1 validation_data: input_path: 'gs://cam2-datasets/coco/val*' - trainer: - train_steps: 831600 # epoch 300 to 450 +trainer: + train_steps: 831600 # epoch 300 to 450 + optimizer_config: learning_rate: cosine: decay_steps: 831600 # epoch 300 to 450 \ No newline at end of file From 74a52d7146790314ea71d94963db8a8c0b64b45f Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Sun, 7 Nov 2021 10:08:45 -0500 Subject: [PATCH 09/96] yolox_head init --- .../yolo/modeling/heads/yolox_head.py | 229 ++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 official/vision/beta/projects/yolo/modeling/heads/yolox_head.py diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py new file mode 100644 index 00000000000..eb3fa04fff8 --- /dev/null +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -0,0 +1,229 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Yolox heads.""" +import tensorflow as tf +from tensorflow.keras.models import Sequential +from yolo.modeling.layers import nn_blocks +from tensorflow.keras.layers import Concatenate +from yolo.ops import box_ops + +class YOLOXHead(tf.keras.layers.Layer): + """YOLOX Prediction Head.""" + + def __init__( + self, + num_classes, + width=1.0, + strides=[8, 16, 32], + in_channels=[256, 512, 1024], + act='silu', + depthwise=False, + **kwargs + ): + + + """ + Args: + act (str): activation type of conv. Defalut value: "silu". + depthwise (bool): whether apply depthwise conv in conv branch. Defalut value: False. + """ + + super().__init__(**kwargs) + + self._n_anchors = 1 + self._num_classes = num_classes + self._decode_in_inference = True + + self._cls_convs = [] + self._reg_convs = [] + + self._cls_preds = [] + self._reg_preds = [] + self._obj_preds = [] + self._stems = [] + + self.prior_prob = 1e-2 + self.bias=-tf.math.log((1-self.prior_prob)/self.prior_prob) + + Conv = nn_blocks.DWConv if depthwise else nn_blocks.ConvBN + for i in range(len(in_channels)): + self._stems.append( + nn_blocks.ConvBN( + filters=int(256 * width), + kernel_size=1, + strides=(1, 1), + padding='same', # TODO + use_bn = True, + activation=act, + ), + ) + + self._cls_convs.append ( + Sequential( + [ + nn_blocks.ConvBN( + filters=int(256 * width), + kernel_size=3, + strides=(1, 1), + use_bn = True, + activation=act, + ), + nn_blocks.ConvBN( + filters=int(256 * width), + kernel_size=3, + strides=(1, 1), + use_bn = True, + activation=act, + ), + ] + ) + ) + + + self._reg_convs.append( + Sequential( + [ + Conv( + filters=int(256 * width), + kernel_size=3, + strides=(1, 1), + use_bn = True, + activation=act, + ), + Conv( + filters=int(256 * width), + kernel_size=3, + strides=(1, 1), + use_bn = True, + activation=act, + ), + ] + ) + ) + + + + self._cls_preds.append( + tf.keras.layers.Conv2D( + filters=self._n_anchors * self._num_classes, + kernel_size=1, + strides=(1, 1), + padding='valid', + bias_initializer=tf.keras.initializers.constant(self.bias) + ), + ) + self._reg_preds.append( + tf.keras.layers.Conv2D( + filters=4, + kernel_size=1, + strides=(1, 1), + padding='valid', + ), + ) + self._obj_preds.append( + tf.keras.layers.Conv2D( + filters=self._n_anchors * 1, + kernel_size=1, + strides=(1, 1), + padding='valid', + bias_initializer=tf.keras.initializers.constant(self.bias) + ), + ) + + # self.use_l1 = False + # self.l1_loss = tf.keras.losses.MAE # TODO need reduce_mean after the loss + # self.bcewithlog_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True) + # self.iou_loss = box_ops.compute_iou + # self.strides = strides + # self.grids = [tf.zeros(1)] * len(in_channels) + @tf.function + def call(self, inputs, *args, **kwargs): + outputs=[] + for k,x in enumerate(inputs): + x = self.stems[k](x) + cls_x = x + reg_x = x + + cls_feat = self._cls_convs[k](cls_x) + cls_output = self._cls_preds[k](cls_feat) + + reg_feat = self._reg_convs[k](reg_x) + reg_output = self._reg_preds[k](reg_feat) + obj_output = self._obj_preds[k](reg_feat) + output=Concatenate(-1)([reg_output,obj_output,cls_output]) + outputs.append(output) + + return outputs +# def call(self, xin, labels=None, imgs=None): +# outputs = [] +# origin_preds = [] +# x_shifts = [] +# y_shifts = [] +# expanded_strides = [] + +# for k, (cls_conv, reg_conv, stride_this_level, x) in enumerate( +# zip(self.cls_convs, self.reg_convs, self.strides, xin) +# ): +# x = self.stems[k](x) +# cls_x = x +# reg_x = x + +# # cls - feature and output +# cls_feat = cls_conv(cls_x) +# cls_output = self.cls_preds[k](cls_feat) + +# # reg & obj - feature and output +# reg_feat = reg_conv(reg_x) +# reg_output = self.reg_preds[k](reg_feat) +# obj_output = self.obj_preds[k](reg_feat) +# output=Concatenate(-1)([reg_output,obj_output,cls_output]) +# outputs.append(output) +# return outputs + # if self.training: + # output = tf.concat([reg_output, obj_output, cls_output], 1) + # output, grid = self.get_output_and_grid( + # output, k, stride_this_level, xin[0].type() + # ) + # x_shifts.append(grid[:, :, 0]) + # y_shifts.append(grid[:, :, 1]) + # expanded_strides.append( + # tf.zeros(1, grid.shape[1]) + # .fill_(stride_this_level) + # .type_as(xin[0]) + # ) + # if self.use_l1: + # batch_size = reg_output.shape[0] + # hsize, wsize = reg_output.shape[-2:] + # reg_output = reg_output.view( + # batch_size, self.n_anchors, 4, hsize, wsize + # ) + # reg_output = reg_output.permute(0, 1, 3, 4, 2).reshape( + # # reg_output: batch_size * n_anchors * 4 * height * width + # # permute: batch_size * n_anchors * height * width * 4 + # # reshape: [batch_size, N_anchors height width, 4] + # # This is also called "origin_pred" + # batch_size, -1, 4 + # ) + # origin_preds.append(reg_output.clone()) + + # else: # not training + # output = tf.concat( + # [reg_output, obj_output.sigmoid(), cls_output.sigmoid()], 1 + # ) + + # outputs.append(output) + # return outputs + \ No newline at end of file From 8d5c3bcc6fd50c9b63359bb5c55228256d52b046 Mon Sep 17 00:00:00 2001 From: Vishnu Banna Date: Sun, 7 Nov 2021 15:30:00 -0500 Subject: [PATCH 10/96] models --- .../vision/beta/projects/yolo/modeling/backbones/darknet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/modeling/backbones/darknet.py b/official/vision/beta/projects/yolo/modeling/backbones/darknet.py index 7c3086bd66d..0cec8883d56 100644 --- a/official/vision/beta/projects/yolo/modeling/backbones/darknet.py +++ b/official/vision/beta/projects/yolo/modeling/backbones/darknet.py @@ -225,7 +225,7 @@ def __call__(self, config, kwargs): False ], [ - 'DarkRes', 'csp', 1, True, 64, None, None, None, None, 'mish', -1, + 'DarkRes', 'csp', 1, False, 64, None, None, None, None, 'mish', -1, 1, 1, False ], [ From 966902299a9d23f6041357a0dc32a1ae6f2ebfd0 Mon Sep 17 00:00:00 2001 From: Vishnu Banna Date: Sun, 7 Nov 2021 15:39:43 -0500 Subject: [PATCH 11/96] models --- .../yolov4/detection/yolov4_tiny_416_tpu.yaml | 134 ++++++++++++++++++ .../vision/beta/projects/yolo/configs/yolo.py | 109 ++++++++++++++ 2 files changed, 243 insertions(+) create mode 100755 official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml new file mode 100755 index 00000000000..c4cf3474384 --- /dev/null +++ b/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml @@ -0,0 +1,134 @@ +# --experiment_type=yolo_darknet +# mAP 43.0 +runtime: + distribution_strategy: 'tpu' + mixed_precision_dtype: 'bfloat16' +task: + smart_bias_lr: 0.0 + model: + darknet_based_model: true + input_size: [416, 416, 3] + backbone: + type: 'darknet' + darknet: + model_id: 'cspdarknettiny' + max_level: 5 + min_level: 4 + decoder: + type: yolo_decoder + yolo_decoder: + version: v4 + type: tiny + head: + smart_bias: true + detection_generator: + box_type: + 'all': original + scale_xy: + '5': 1.05 + '4': 1.1 + '3': 1.2 + max_boxes: 200 + nms_type: iou + iou_thresh: 0.001 + nms_thresh: 0.60 + loss: + use_scaled_loss: false + box_loss_type: + 'all': ciou + ignore_thresh: + 'all': 0.7 + iou_normalizer: + 'all': 0.07 + cls_normalizer: + 'all': 1.0 + object_normalizer: + 'all': 1.0 + objectness_smooth: + 'all': 0.0 + max_delta: + 'all': 5.0 + norm_activation: + activation: leaky + norm_epsilon: 0.0001 + norm_momentum: 0.99 + use_sync_bn: true + num_classes: 80 + anchor_boxes: + anchors_per_scale: 3 + boxes: [box: [10, 14], box: [23, 27], box: [37, 58], + box: [81, 82], box: [135, 169], box: [344, 319]] + train_data: + global_batch_size: 64 + dtype: float32 + input_path: 'gs://cam2-datasets/coco/train*' + is_training: true + drop_remainder: true + seed: 1000 + parser: + mosaic: + mosaic_frequency: 0.75 + mixup_frequency: 0.0 + mosaic_crop_mode: 'crop' + mosaic_center: 0.2 + aug_scale_min: 0.2 + aug_scale_max: 1.6 + jitter: 0.3 + max_num_instances: 200 + letter_box: false + random_flip: true + aug_rand_saturation: 1.5 + aug_rand_brightness: 1.5 + aug_rand_hue: 0.1 + aug_scale_min: 0.1 + aug_scale_max: 1.9 + aug_rand_translate: 0.0 + jitter: 0.3 + area_thresh: 0.1 + random_pad: true + use_tie_breaker: true + anchor_thresh: 0.4 + validation_data: + global_batch_size: 8 + dtype: float32 + input_path: 'gs://cam2-datasets/coco/val*' + is_training: false + drop_remainder: true + parser: + max_num_instances: 200 + letter_box: false + use_tie_breaker: true + anchor_thresh: 0.4 + weight_decay: 0.000 + annotation_file: null +trainer: + train_steps: 555000 + validation_steps: 625 + steps_per_loop: 1850 + summary_interval: 1850 + validation_interval: 9250 + checkpoint_interval: 1850 + optimizer_config: + ema: + average_decay: 0.9998 + trainable_weights_only: false + dynamic_decay: true + learning_rate: + type: stepwise + stepwise: + boundaries: [400000] + name: PiecewiseConstantDecay + values: [0.00131, 0.000131] + optimizer: + type: sgd_torch + sgd_torch: + momentum: 0.949 + momentum_start: 0.949 + nesterov: true + warmup_steps: 1000 + weight_decay: 0.0005 + name: SGD + warmup: + type: 'linear' + linear: + warmup_steps: 1000 # learning rate rises from 0 to 0.0013 over 1000 steps diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index 59679d71e19..b9df1afe8f3 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -619,4 +619,113 @@ def large_yolo() -> cfg.ExperimentConfig: 'task.validation_data.is_training != None' ]) + return config + + +@exp_factory.register_config_factory('yolo_tiny') +def yolo_tiny() -> cfg.ExperimentConfig: + """COCO object detection with YOLOv3 and v4.""" + train_batch_size = 256 + eval_batch_size = 8 + train_epochs = 600 + steps_per_epoch = COCO_TRAIN_EXAMPLES // train_batch_size + validation_interval = 10 + + max_num_instances = 200 + config = cfg.ExperimentConfig( + runtime=cfg.RuntimeConfig(mixed_precision_dtype='bfloat16'), + task=YoloTask( + smart_bias_lr=0.1, + init_checkpoint='', + init_checkpoint_modules='backbone', + annotation_file=None, + weight_decay=0.0, + model=Yolo( + darknet_based_model=True, + norm_activation=common.NormActivation(use_sync_bn=True, + activation="leaky"), + head=YoloHead(smart_bias=True), + loss=YoloLoss(use_scaled_loss=False, update_on_repeat=True)), + train_data=DataConfig( + input_path=os.path.join(COCO_INPUT_PATH_BASE, 'train*'), + is_training=True, + global_batch_size=train_batch_size, + dtype='float32', + parser=Parser( + letter_box=False, + aug_rand_saturation=1.5, + aug_rand_brightness=1.5, + aug_rand_hue=0.1, + use_tie_breaker=True, + best_match_only=False, + anchor_thresh=0.4, + area_thresh=0.1, + max_num_instances=max_num_instances, + mosaic=Mosaic( + mosaic_frequency=0.75, + mixup_frequency=0.0, + mosaic_crop_mode='crop', + mosaic_center=0.2))), + validation_data=DataConfig( + input_path=os.path.join(COCO_INPUT_PATH_BASE, 'val*'), + is_training=False, + global_batch_size=eval_batch_size, + drop_remainder=True, + dtype='float32', + parser=Parser( + letter_box=False, + use_tie_breaker=True, + best_match_only=False, + anchor_thresh=0.4, + area_thresh=0.1, + max_num_instances=max_num_instances, + ))), + trainer=cfg.TrainerConfig( + train_steps=train_epochs * steps_per_epoch, + validation_steps=COCO_VAL_EXAMPLES // eval_batch_size, + validation_interval=validation_interval * steps_per_epoch, + steps_per_loop=steps_per_epoch, + summary_interval=steps_per_epoch, + checkpoint_interval=steps_per_epoch, + optimizer_config=optimization.OptimizationConfig({ + 'ema': { + 'average_decay': 0.9998, + 'trainable_weights_only': False, + 'dynamic_decay': True, + }, + 'optimizer': { + 'type': 'sgd_torch', + 'sgd_torch': { + 'momentum': 0.9, + 'momentum_start': 0.9, + 'nesterov': True, + 'warmup_steps': 1000, + 'weight_decay': 0.0005, + } + }, + 'learning_rate': { + 'type': 'stepwise', + 'stepwise': { + 'boundaries': [ + 0.8 * train_epochs * steps_per_epoch + ], + 'values': [ + 0.00261 * train_batch_size / 64.0, + 0.000261 * train_batch_size / 64.0, + ] + } + }, + 'warmup': { + 'type': 'linear', + 'linear': { + 'warmup_steps': 1000, + 'warmup_learning_rate': 0 + } + } + })), + restrictions=[ + 'task.train_data.is_training != None', + 'task.validation_data.is_training != None' + ]) + return config \ No newline at end of file From 7f1e4b5294d855efa8f47f36eee9d275d6c6eeda Mon Sep 17 00:00:00 2001 From: Vishnu Banna Date: Sun, 7 Nov 2021 15:49:33 -0500 Subject: [PATCH 12/96] models --- .../detection/yolo_tiny_416_tpu.yaml | 75 ++++++++++++++++++ .../detection/yolo_xl_p5_896_tpu.yaml | 76 ------------------- .../yolov4/detection/yolov4_tiny_416_tpu.yaml | 37 +-------- 3 files changed, 78 insertions(+), 110 deletions(-) create mode 100755 official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_tiny_416_tpu.yaml delete mode 100644 official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_xl_p5_896_tpu.yaml diff --git a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_tiny_416_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_tiny_416_tpu.yaml new file mode 100755 index 00000000000..61bcd460901 --- /dev/null +++ b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_tiny_416_tpu.yaml @@ -0,0 +1,75 @@ +# --experiment_type=yolo_darknet +# mAP 43.0 +runtime: + distribution_strategy: 'tpu' + mixed_precision_dtype: 'bfloat16' +task: + model: + input_size: [416, 416, 3] + backbone: + type: 'darknet' + darknet: + model_id: 'cspdarknettiny' + max_level: 5 + min_level: 4 + decoder: + type: yolo_decoder + yolo_decoder: + version: v4 + type: tiny + head: + smart_bias: true + detection_generator: + box_type: + 'all': scaled + scale_xy: + 'all': 2.0 + max_boxes: 300 + nms_type: iou + iou_thresh: 0.001 + nms_thresh: 0.65 + loss: + use_scaled_loss: true + update_on_repeat: true + box_loss_type: + 'all': ciou + ignore_thresh: + 'all': 0.0 + iou_normalizer: + 'all': 0.05 + cls_normalizer: + 'all': 0.3 + object_normalizer: + '5': 0.28 + '4': 0.70 + '3': 2.80 + objectness_smooth: + 'all': 1.0 + norm_activation: + activation: leaky + norm_epsilon: 0.0001 + norm_momentum: 0.99 + use_sync_bn: true + num_classes: 80 + anchor_boxes: + anchors_per_scale: 3 + boxes: [box: [10, 14], box: [23, 27], box: [37, 58], + box: [81, 82], box: [135, 169], box: [344, 319]] + train_data: + input_path: 'gs://cam2-datasets/coco/train*' + shuffle_buffer_size: 10000 + parser: + mosaic: + mosaic_frequency: 1.0 + mixup_frequency: 0.0 + mosaic_crop_mode: 'scale' + mosaic_center: 0.25 + aug_scale_min: 0.1 + aug_scale_max: 1.9 + max_num_instances: 300 + letter_box: true + random_flip: true + aug_rand_translate: 0.1 + area_thresh: 0.1 + validation_data: + input_path: 'gs://cam2-datasets/coco/val*' diff --git a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_xl_p5_896_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_xl_p5_896_tpu.yaml deleted file mode 100644 index 4fc3dfe647c..00000000000 --- a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_xl_p5_896_tpu.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# --experiment_type=large_yolo -# mAP 50.5% -runtime: - distribution_strategy: 'tpu' - mixed_precision_dtype: 'float32' - tpu_enable_xla_dynamic_padder: false -task: - model: - input_size: [896, 896, 3] - backbone: - type: 'darknet' - darknet: - model_id: 'csp-large' - max_level: 5 - min_level: 3 - width_scale: 1.00 - depth_scale: 1.00 - decoder: - type: yolo_decoder - yolo_decoder: - version: v4 - type: csp_xlarge - head: - smart_bias: true - detection_generator: - box_type: - 'all': scaled - scale_xy: - 'all': 2.0 - max_boxes: 300 - nms_type: iou - iou_thresh: 0.001 - nms_thresh: 0.65 - loss: - use_scaled_loss: true - update_on_repeat: true - box_loss_type: - 'all': ciou - ignore_thresh: - 'all': 0.0 - iou_normalizer: - 'all': 0.05 - cls_normalizer: - 'all': 0.5 - object_normalizer: - '5': 0.4 - '4': 1.0 - '3': 4.0 - objectness_smooth: - 'all': 1.0 - norm_activation: - use_sync_bn: true - num_classes: 80 - anchor_boxes: - anchors_per_scale: 4 - boxes: [box: [13,17], box: [31,25], box: [24,51], box: [61,45], - box: [48,102], box: [119,96], box: [97,189], box: [217,184], - box: [171,384], box: [324,451], box: [616,618], box: [800,800]] - train_data: - input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/train*' - shuffle_buffer_size: 10000 - parser: - mosaic: - mosaic_frequency: 1.0 - mixup_frequency: 0.0 - mosaic_crop_mode: 'scale' - mosaic_center: 0.0 - aug_scale_min: 0.5 - aug_scale_max: 1.5 - max_num_instances: 300 - letter_box: true - random_flip: true - aug_rand_translate: 0.5 - area_thresh: 0.0 - validation_data: - input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/val*' diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml index c4cf3474384..dfac2bdb36c 100755 --- a/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml @@ -28,7 +28,7 @@ task: '5': 1.05 '4': 1.1 '3': 1.2 - max_boxes: 200 + max_boxes: 300 nms_type: iou iou_thresh: 0.001 nms_thresh: 0.60 @@ -59,12 +59,12 @@ task: boxes: [box: [10, 14], box: [23, 27], box: [37, 58], box: [81, 82], box: [135, 169], box: [344, 319]] train_data: - global_batch_size: 64 + global_batch_size: 256 + shuffle_buffer_size: 10000 dtype: float32 input_path: 'gs://cam2-datasets/coco/train*' is_training: true drop_remainder: true - seed: 1000 parser: mosaic: mosaic_frequency: 0.75 @@ -101,34 +101,3 @@ task: anchor_thresh: 0.4 weight_decay: 0.000 annotation_file: null -trainer: - train_steps: 555000 - validation_steps: 625 - steps_per_loop: 1850 - summary_interval: 1850 - validation_interval: 9250 - checkpoint_interval: 1850 - optimizer_config: - ema: - average_decay: 0.9998 - trainable_weights_only: false - dynamic_decay: true - learning_rate: - type: stepwise - stepwise: - boundaries: [400000] - name: PiecewiseConstantDecay - values: [0.00131, 0.000131] - optimizer: - type: sgd_torch - sgd_torch: - momentum: 0.949 - momentum_start: 0.949 - nesterov: true - warmup_steps: 1000 - weight_decay: 0.0005 - name: SGD - warmup: - type: 'linear' - linear: - warmup_steps: 1000 # learning rate rises from 0 to 0.0013 over 1000 steps From 7c1add689483d4efd0006c25d25030156b8b0b5f Mon Sep 17 00:00:00 2001 From: Vishnu Banna Date: Sun, 7 Nov 2021 15:49:48 -0500 Subject: [PATCH 13/96] models --- .../experiments/scaled-yolo/detection/yolo_tiny_416_tpu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_tiny_416_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_tiny_416_tpu.yaml index 61bcd460901..f31ecd0bb16 100755 --- a/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_tiny_416_tpu.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/scaled-yolo/detection/yolo_tiny_416_tpu.yaml @@ -48,7 +48,7 @@ task: norm_activation: activation: leaky norm_epsilon: 0.0001 - norm_momentum: 0.99 + norm_momentum: 0.97 use_sync_bn: true num_classes: 80 anchor_boxes: From e27571bf6fcf5dceb7e1fc334ce08f2695dd1072 Mon Sep 17 00:00:00 2001 From: Vishnu Banna Date: Sun, 7 Nov 2021 16:24:28 -0500 Subject: [PATCH 14/96] models --- .../experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml index dfac2bdb36c..049a7eb8a1f 100755 --- a/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml @@ -26,8 +26,7 @@ task: 'all': original scale_xy: '5': 1.05 - '4': 1.1 - '3': 1.2 + '4': 1.05 max_boxes: 300 nms_type: iou iou_thresh: 0.001 From a9ce08d305942ee36a1e029db55dc1172fed0686 Mon Sep 17 00:00:00 2001 From: Vishnu Banna Date: Mon, 8 Nov 2021 12:58:31 -0500 Subject: [PATCH 15/96] models --- .../experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml index 049a7eb8a1f..c992bc5f8e1 100755 --- a/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolov4/detection/yolov4_tiny_416_tpu.yaml @@ -70,8 +70,8 @@ task: mixup_frequency: 0.0 mosaic_crop_mode: 'crop' mosaic_center: 0.2 - aug_scale_min: 0.2 - aug_scale_max: 1.6 + aug_scale_min: 0.7 + aug_scale_max: 1.3 jitter: 0.3 max_num_instances: 200 letter_box: false @@ -79,8 +79,8 @@ task: aug_rand_saturation: 1.5 aug_rand_brightness: 1.5 aug_rand_hue: 0.1 - aug_scale_min: 0.1 - aug_scale_max: 1.9 + aug_scale_min: 0.5 + aug_scale_max: 1.5 aug_rand_translate: 0.0 jitter: 0.3 area_thresh: 0.1 From 42c1cf603427f8be956c5d61d39ccacc35a8d6fc Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Tue, 9 Nov 2021 17:26:06 -0500 Subject: [PATCH 16/96] Update DWConv in nn_blocks --- .../yolo/modeling/layers/nn_blocks.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py index 5fc98ea2f63..e21ec4b38d1 100644 --- a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py +++ b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py @@ -1725,3 +1725,34 @@ def call(self, x, training=None): x[..., 1::2, 1::2, :] ], axis=-1) + + +class DWConv(tf.keras.layers.Layer): + def __init__ (self, + filters, + kernel_size, + strides=1, + padding='valid', # TODO: need to make sure this is correct + groups=1, + use_biase=False, + use_bn=True, + activation='silu', + **kwargs): + super().__init__(**kwargs) + self._conv1 = ConvBN(filters, + kernel_size, + strides=strides, + use_bn=use_bn, + activation=activation) + self._conv2 = ConvBN(filters, + kernel_size=1, + strides=1, + use_bn=use_bn, + activation=activation) + + def call(self, inputs, **kwargs): + x = self._conv1(inputs) + x = self._conv2(x) + return x + + \ No newline at end of file From 40b7483529b736e90729f3f12f6614b71651957f Mon Sep 17 00:00:00 2001 From: Vishnu Banna <43182884+vishnubanna@users.noreply.github.com> Date: Tue, 9 Nov 2021 19:10:39 -0500 Subject: [PATCH 17/96] Update yolo_decoder.py --- .../beta/projects/yolo/modeling/decoders/yolo_decoder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py index e327f6b8698..5b2c434ac3d 100644 --- a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py +++ b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py @@ -78,8 +78,8 @@ spp=dict( embed_spp=True, use_fpn=False, - max_level_process_len=2, - path_process_len=1), + max_level_process_len=None, + path_process_len=6), ), } @@ -633,4 +633,4 @@ def build_yolo_decoder( base_model.update(base_dict) model = YoloDecoder(input_specs, **base_model, **kwargs) - return model \ No newline at end of file + return model From 01b6930978787193ed1a7a16ad725c0db1340345 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Wed, 10 Nov 2021 21:29:58 -0500 Subject: [PATCH 18/96] yolox head init --- .../yolo/configs/experiments/yolox/yolox.yaml | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml new file mode 100644 index 00000000000..fe5ce334195 --- /dev/null +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -0,0 +1,74 @@ +# --experiment_type=scaled_yolo +# mAP 47.6 +runtime: + distribution_strategy: 'tpu' + mixed_precision_dtype: 'float32' + tpu_enable_xla_dynamic_padder: false +task: + model: + input_size: [640, 640, 3] + backbone: + type: 'darknet' + darknet: + model_id: 'darknet53' + max_level: 5 + min_level: 3 + decoder: + type: yolo_decoder + yolo_decoder: + version: v4 + type: csp + head: + smart_bias: true + detection_generator: + box_type: + 'all': scaled + scale_xy: + 'all': 2.0 + max_boxes: 300 + nms_type: iou + iou_thresh: 0.001 + nms_thresh: 0.65 + loss: + use_scaled_loss: true + update_on_repeat: true + box_loss_type: + 'all': ciou + ignore_thresh: + 'all': 0.0 + iou_normalizer: + 'all': 0.05 + cls_normalizer: + 'all': 0.3 + object_normalizer: + '5': 0.28 + '4': 0.70 + '3': 2.80 + objectness_smooth: + 'all': 1.0 + norm_activation: + use_sync_bn: true + num_classes: 80 + anchor_boxes: + anchors_per_scale: 3 + boxes: [box: [12, 16], box: [19, 36], box: [40, 28], + box: [36, 75], box: [76, 55], box: [72, 146], + box: [142, 110], box: [192, 243], box: [459, 401]] + train_data: + input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/train*' + shuffle_buffer_size: 10000 + parser: + mosaic: + mosaic_frequency: 1.0 + mixup_frequency: 0.0 + mosaic_crop_mode: 'scale' + mosaic_center: 0.25 + aug_scale_min: 0.1 + aug_scale_max: 1.9 + max_num_instances: 300 + letter_box: true + random_flip: true + aug_rand_translate: 0.1 + area_thresh: 0.1 + validation_data: + input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/val*' From 5e4badd4b7d19506b1f0afbf63b0c5d871c0504b Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Thu, 11 Nov 2021 15:53:36 -0500 Subject: [PATCH 19/96] Create yolox.py --- .../beta/projects/yolo/configs/yolox.py | 391 ++++++++++++++++++ 1 file changed, 391 insertions(+) create mode 100644 official/vision/beta/projects/yolo/configs/yolox.py diff --git a/official/vision/beta/projects/yolo/configs/yolox.py b/official/vision/beta/projects/yolo/configs/yolox.py new file mode 100644 index 00000000000..c4414ba535e --- /dev/null +++ b/official/vision/beta/projects/yolo/configs/yolox.py @@ -0,0 +1,391 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""YOLO configuration definition.""" +import dataclasses +import os +from typing import Any, List, Optional, Union + +import numpy as np + +from official.core import config_definitions as cfg +from official.core import exp_factory +from official.modeling import hyperparams +from official.vision.beta.configs import common +from official.vision.beta.projects.yolo import optimization +from official.vision.beta.projects.yolo.configs import backbones +from official.vision.beta.projects.yolo.configs import decoders + + +# pytype: disable=annotation-type-mismatch + +MIN_LEVEL = 1 +MAX_LEVEL = 7 +GLOBAL_SEED = 1000 + + +def _build_dict(min_level, max_level, value): + vals = {str(key): value for key in range(min_level, max_level + 1)} + vals['all'] = None + return lambda: vals + + +def _build_path_scales(min_level, max_level): + return lambda: {str(key): 2**key for key in range(min_level, max_level + 1)} + + +@dataclasses.dataclass +class FPNConfig(hyperparams.Config): + """FPN config.""" + all: Optional[Any] = None + + def get(self): + """Allow for a key for each level or a single key for all the levels.""" + values = self.as_dict() + if 'all' in values and values['all'] is not None: + for key in values: + if key != 'all': + values[key] = values['all'] + return values + + +# pylint: disable=missing-class-docstring +@dataclasses.dataclass +class TfExampleDecoder(hyperparams.Config): + regenerate_source_id: bool = False + coco91_to_80: bool = True + + +@dataclasses.dataclass +class TfExampleDecoderLabelMap(hyperparams.Config): + regenerate_source_id: bool = False + label_map: str = '' + + +@dataclasses.dataclass +class DataDecoder(hyperparams.OneOfConfig): + type: Optional[str] = 'simple_decoder' + simple_decoder: TfExampleDecoder = TfExampleDecoder() + label_map_decoder: TfExampleDecoderLabelMap = TfExampleDecoderLabelMap() + + +@dataclasses.dataclass +class Mosaic(hyperparams.Config): + mosaic_frequency: float = 0.0 + mixup_frequency: float = 0.0 + mosaic_center: float = 0.2 + mosaic_crop_mode: Optional[str] = None + aug_scale_min: float = 1.0 + aug_scale_max: float = 1.0 + jitter: float = 0.0 + + +@dataclasses.dataclass +class Parser(hyperparams.Config): + max_num_instances: int = 200 + letter_box: Optional[bool] = True + random_flip: bool = True + random_pad: float = False + jitter: float = 0.0 + aug_scale_min: float = 1.0 + aug_scale_max: float = 1.0 + aug_rand_saturation: float = 0.0 + aug_rand_brightness: float = 0.0 + aug_rand_hue: float = 0.0 + aug_rand_angle: float = 0.0 + aug_rand_translate: float = 0.0 + aug_rand_perspective: float = 0.0 + use_tie_breaker: bool = True + best_match_only: bool = False + anchor_thresh: float = -0.01 + area_thresh: float = 0.1 + mosaic: Mosaic = Mosaic() + + +@dataclasses.dataclass +class DataConfig(cfg.DataConfig): + """Input config for training.""" + global_batch_size: int = 64 + input_path: str = '' + tfds_name: str = '' + tfds_split: str = '' + global_batch_size: int = 1 + is_training: bool = True + dtype: str = 'float16' + decoder: DataDecoder = DataDecoder() + parser: Parser = Parser() + shuffle_buffer_size: int = 10000 + tfds_download: bool = True + cache: bool = False + drop_remainder: bool = True + + +@dataclasses.dataclass +class YOLOXHead(hyperparams.Config): + """Parameterization for the YOLOX Head.""" + num_classes: int = None + width: float = 1.0 + strides: List[int] = [8, 16, 32] + in_channels: List[int] = [256, 512, 1024] + depthwise: bool = False + activation: str = 'silu' + + +@dataclasses.dataclass +class YoloDetectionGenerator(hyperparams.Config): + box_type: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 'original')) + scale_xy: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) + path_scales: FPNConfig = dataclasses.field( + default_factory=_build_path_scales(MIN_LEVEL, MAX_LEVEL)) + nms_type: str = 'greedy' + iou_thresh: float = 0.001 + nms_thresh: float = 0.6 + max_boxes: int = 200 + pre_nms_points: int = 5000 + + +@dataclasses.dataclass +class YoloLoss(hyperparams.Config): + ignore_thresh: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 0.0)) + truth_thresh: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) + box_loss_type: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 'ciou')) + iou_normalizer: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) + cls_normalizer: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) + object_normalizer: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) + max_delta: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, np.inf)) + objectness_smooth: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 0.0)) + label_smoothing: float = 0.0 + use_scaled_loss: bool = True + update_on_repeat: bool = True + + +@dataclasses.dataclass +class Box(hyperparams.Config): + box: List[int] = dataclasses.field(default=list) + + +@dataclasses.dataclass +class AnchorBoxes(hyperparams.Config): + boxes: Optional[List[Box]] = None + level_limits: Optional[List[int]] = None + anchors_per_scale: int = 3 + + def get(self, min_level, max_level): + """Distribute them in order to each level. + + Args: + min_level: `int` the lowest output level. + max_level: `int` the heighest output level. + Returns: + anchors_per_level: A `Dict[List[int]]` of the anchor boxes for each level. + self.level_limits: A `List[int]` of the box size limits to link to each + level under anchor free conditions. + """ + if self.level_limits is None: + boxes = [box.box for box in self.boxes] + else: + boxes = [[1.0, 1.0]] * ((max_level - min_level) + 1) + self.anchors_per_scale = 1 + + anchors_per_level = dict() + start = 0 + for i in range(min_level, max_level + 1): + anchors_per_level[str(i)] = boxes[start:start + self.anchors_per_scale] + start += self.anchors_per_scale + return anchors_per_level, self.level_limits + + +@dataclasses.dataclass +class YOLOX(hyperparams.Config): + input_size: Optional[List[int]] = dataclasses.field( + default_factory=lambda: [640, 640, 3]) + backbone: backbones.Backbone = backbones.Backbone( + type='darknet', darknet=backbones.Darknet(model_id='darknet53')) + decoder: decoders.Decoder = decoders.Decoder( + type='yolo_decoder', + yolo_decoder=decoders.YoloDecoder(version='vx', type='regular')) + head: YOLOXHead = YOLOXHead() + detection_generator: YoloDetectionGenerator = YoloDetectionGenerator() + loss: YoloLoss = YoloLoss() + norm_activation: common.NormActivation = common.NormActivation( + activation='mish', + use_sync_bn=True, + norm_momentum=0.99, + norm_epsilon=0.001) + num_classes: int = 80 + anchor_boxes: AnchorBoxes = AnchorBoxes() + darknet_based_model: bool = False + + +@dataclasses.dataclass +class YOLOXTask(cfg.TaskConfig): + per_category_metrics: bool = False + smart_bias_lr: float = 0.0 + model: YOLOX = YOLOX() + train_data: DataConfig = DataConfig(is_training=True) + validation_data: DataConfig = DataConfig(is_training=False) + weight_decay: float = 0.0 + annotation_file: Optional[str] = None + init_checkpoint: Optional[str] = None + init_checkpoint_modules: Union[ + str, List[str]] = 'all' # all, backbone, and/or decoder + gradient_clip_norm: float = 0.0 + seed = GLOBAL_SEED + + +COCO_INPUT_PATH_BASE = 'coco' +COCO_TRAIN_EXAMPLES = 118287 +COCO_VAL_EXAMPLES = 5000 + + +@exp_factory.register_config_factory('yolox') +def yolox() -> cfg.ExperimentConfig: + """Yolox general config.""" + return cfg.ExperimentConfig( + task=YOLOXTask(), + restrictions=[ + 'task.train_data.is_training != None', + 'task.validation_data.is_training != None' + ]) + + +@exp_factory.register_config_factory('yolox') +def yolox() -> cfg.ExperimentConfig: + """COCO object detection with YOLOvx.""" + train_batch_size = 64 + eval_batch_size = 8 + train_epochs = 300 + steps_per_epoch = COCO_TRAIN_EXAMPLES // train_batch_size + validation_interval = 5 + + max_num_instances = 200 + config = cfg.ExperimentConfig( + runtime=cfg.RuntimeConfig(mixed_precision_dtype='bfloat16'), + task=YOLOXTask( + smart_bias_lr=0.1, + init_checkpoint='', + init_checkpoint_modules='backbone', + annotation_file=None, + weight_decay=0.0, + model=YOLOX( + darknet_based_model=True, + norm_activation=common.NormActivation(use_sync_bn=True), + head=YoloHead(num_classes=80), + loss=YoloLoss(use_scaled_loss=False, update_on_repeat=True), + anchor_boxes=AnchorBoxes( + anchors_per_scale=3, + boxes=[ + Box(box=[12, 16]), + Box(box=[19, 36]), + Box(box=[40, 28]), + Box(box=[36, 75]), + Box(box=[76, 55]), + Box(box=[72, 146]), + Box(box=[142, 110]), + Box(box=[192, 243]), + Box(box=[459, 401]) + ])), + train_data=DataConfig( + input_path=os.path.join(COCO_INPUT_PATH_BASE, 'train*'), + is_training=True, + global_batch_size=train_batch_size, + dtype='float32', + parser=Parser( + letter_box=False, + aug_rand_saturation=1.5, + aug_rand_brightness=1.5, + aug_rand_hue=0.1, + use_tie_breaker=True, + best_match_only=False, + anchor_thresh=0.4, + area_thresh=0.1, + max_num_instances=max_num_instances, + mosaic=Mosaic( + mosaic_frequency=0.75, + mixup_frequency=0.0, + mosaic_crop_mode='crop', + mosaic_center=0.2))), + validation_data=DataConfig( + input_path=os.path.join(COCO_INPUT_PATH_BASE, 'val*'), + is_training=False, + global_batch_size=eval_batch_size, + drop_remainder=True, + dtype='float32', + parser=Parser( + letter_box=False, + use_tie_breaker=True, + best_match_only=False, + anchor_thresh=0.4, + area_thresh=0.1, + max_num_instances=max_num_instances, + ))), + trainer=cfg.TrainerConfig( + train_steps=train_epochs * steps_per_epoch, + validation_steps=COCO_VAL_EXAMPLES // eval_batch_size, + validation_interval=validation_interval * steps_per_epoch, + steps_per_loop=steps_per_epoch, + summary_interval=steps_per_epoch, + checkpoint_interval=steps_per_epoch, + optimizer_config=optimization.OptimizationConfig({ + 'ema': { + 'average_decay': 0.9998, + 'trainable_weights_only': False, + 'dynamic_decay': True, + }, + 'optimizer': { + 'type': 'sgd_torch', + 'sgd_torch': { + 'momentum': 0.949, + 'momentum_start': 0.949, + 'nesterov': True, + 'warmup_steps': 1000, + 'weight_decay': 0.0005, + } + }, + 'learning_rate': { + 'type': 'stepwise', + 'stepwise': { + 'boundaries': [ + 240 * steps_per_epoch + ], + 'values': [ + 0.00131 * train_batch_size / 64.0, + 0.000131 * train_batch_size / 64.0, + ] + } + }, + 'warmup': { + 'type': 'linear', + 'linear': { + 'warmup_steps': 1000, + 'warmup_learning_rate': 0 + } + } + })), + restrictions=[ + 'task.train_data.is_training != None', + 'task.validation_data.is_training != None' + ]) + + return config From 813443eef1496c0181764e04f30331c5cbd9fbe5 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:23:01 -0500 Subject: [PATCH 20/96] Create yolox.py --- .../vision/beta/projects/yolo/tasks/yolox.py | 408 ++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100644 official/vision/beta/projects/yolo/tasks/yolox.py diff --git a/official/vision/beta/projects/yolo/tasks/yolox.py b/official/vision/beta/projects/yolo/tasks/yolox.py new file mode 100644 index 00000000000..dcdb5cbb993 --- /dev/null +++ b/official/vision/beta/projects/yolo/tasks/yolox.py @@ -0,0 +1,408 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Contains classes used to train Yolo.""" + +import collections +from typing import Optional + +from absl import logging +import tensorflow as tf + +from official.core import base_task +from official.core import config_definitions +from official.core import input_reader +from official.core import task_factory +from official.modeling import performance +from official.vision.beta.dataloaders import tfds_factory +from official.vision.beta.dataloaders import tf_example_label_map_decoder +from official.vision.beta.evaluation import coco_evaluator +from official.vision.beta.ops import box_ops +from official.vision.beta.projects.yolo import optimization +from official.vision.beta.projects.yolo.configs import yolo as exp_cfg +from official.vision.beta.projects.yolo.dataloaders import tf_example_decoder +from official.vision.beta.projects.yolo.dataloaders import yolo_input +from official.vision.beta.projects.yolo.modeling import factory +from official.vision.beta.projects.yolo.ops import mosaic +from official.vision.beta.projects.yolo.ops import preprocessing_ops +from official.vision.beta.projects.yolo.tasks import task_utils + +OptimizationConfig = optimization.OptimizationConfig +RuntimeConfig = config_definitions.RuntimeConfig + + +@task_factory.register_task_cls(exp_cfg.YOLOXTask) +class YOLOXTask(base_task.Task): + """A single-replica view of training procedure. + + YOLO task provides artifacts for training/evalution procedures, including + loading/iterating over Datasets, initializing the model, calculating the loss, + post-processing, and customized metrics with reduction. + """ + + def __init__(self, params, logging_dir: Optional[str] = None): + super().__init__(params, logging_dir) + self.coco_metric = None + self._loss_fn = None + self._model = None + self._coco_91_to_80 = False + self._metrics = [] + + # globally set the random seed + preprocessing_ops.set_random_seeds(seed=params.seed) + return + + def build_model(self): + """Build an instance of Yolo.""" + + model_base_cfg = self.task_config.model + l2_weight_decay = self.task_config.weight_decay / 2.0 + + input_size = model_base_cfg.input_size.copy() + input_specs = tf.keras.layers.InputSpec(shape=[None] + input_size) + l2_regularizer = ( + tf.keras.regularizers.l2(l2_weight_decay) if l2_weight_decay else None) + model, losses = factory.build_yolox( + input_specs, model_base_cfg) + + # save for later usage within the task. + self._loss_fn = losses + self._model = model + return model + + def _get_data_decoder(self, params): + """Get a decoder object to decode the dataset.""" + if params.tfds_name: + decoder = tfds_factory.get_detection_decoder(params.tfds_name) + else: + decoder_cfg = params.decoder.get() + if params.decoder.type == 'simple_decoder': + self._coco_91_to_80 = decoder_cfg.coco91_to_80 + decoder = tf_example_decoder.TfExampleDecoder( + coco91_to_80=decoder_cfg.coco91_to_80, + regenerate_source_id=decoder_cfg.regenerate_source_id) + elif params.decoder.type == 'label_map_decoder': + decoder = tf_example_label_map_decoder.TfExampleDecoderLabelMap( + label_map=decoder_cfg.label_map, + regenerate_source_id=decoder_cfg.regenerate_source_id) + else: + raise ValueError('Unknown decoder type: {}!'.format( + params.decoder.type)) + return decoder + + def build_inputs(self, params, input_context=None): + """Build input dataset.""" + model = self.task_config.model + + # get anchor boxes dict based on models min and max level + backbone = model.backbone.get() + anchor_dict, level_limits = model.anchor_boxes.get(backbone.min_level, + backbone.max_level) + + params.seed = self.task_config.seed + # set shared patamters between mosaic and yolo_input + base_config = dict( + letter_box=params.parser.letter_box, + aug_rand_translate=params.parser.aug_rand_translate, + aug_rand_angle=params.parser.aug_rand_angle, + aug_rand_perspective=params.parser.aug_rand_perspective, + area_thresh=params.parser.area_thresh, + random_flip=params.parser.random_flip, + seed=params.seed, + ) + + # get the decoder + decoder = self._get_data_decoder(params) + + # init Mosaic + sample_fn = mosaic.Mosaic( + output_size=model.input_size, + mosaic_frequency=params.parser.mosaic.mosaic_frequency, + mixup_frequency=params.parser.mosaic.mixup_frequency, + jitter=params.parser.mosaic.jitter, + mosaic_center=params.parser.mosaic.mosaic_center, + mosaic_crop_mode=params.parser.mosaic.mosaic_crop_mode, + aug_scale_min=params.parser.mosaic.aug_scale_min, + aug_scale_max=params.parser.mosaic.aug_scale_max, + **base_config) + + # init Parser + parser = yolo_input.Parser( + output_size=model.input_size, + anchors=anchor_dict, + use_tie_breaker=params.parser.use_tie_breaker, + jitter=params.parser.jitter, + aug_scale_min=params.parser.aug_scale_min, + aug_scale_max=params.parser.aug_scale_max, + aug_rand_hue=params.parser.aug_rand_hue, + aug_rand_saturation=params.parser.aug_rand_saturation, + aug_rand_brightness=params.parser.aug_rand_brightness, + max_num_instances=params.parser.max_num_instances, + scale_xy=model.detection_generator.scale_xy.get(), + expanded_strides=model.detection_generator.path_scales.get(), + darknet=model.darknet_based_model, + best_match_only=params.parser.best_match_only, + anchor_t=params.parser.anchor_thresh, + random_pad=params.parser.random_pad, + level_limits=level_limits, + dtype=params.dtype, + **base_config) + + # init the dataset reader + reader = input_reader.InputReader( + params, + dataset_fn=tf.data.TFRecordDataset, + decoder_fn=decoder.decode, + sample_fn=sample_fn.mosaic_fn(is_training=params.is_training), + parser_fn=parser.parse_fn(params.is_training)) + dataset = reader.read(input_context=input_context) + return dataset + + def build_metrics(self, training=True): + """Build detection metrics.""" + metrics = [] + + backbone = self.task_config.model.backbone.get() + metric_names = collections.defaultdict(list) + for key in range(backbone.min_level, backbone.max_level + 1): + key = str(key) + metric_names[key].append('loss') + metric_names[key].append('avg_iou') + metric_names[key].append('avg_obj') + + metric_names['net'].append('box') + metric_names['net'].append('class') + metric_names['net'].append('conf') + + for _, key in enumerate(metric_names.keys()): + metrics.append(task_utils.ListMetrics(metric_names[key], name=key)) + + self._metrics = metrics + if not training: + annotation_file = self.task_config.annotation_file + if self._coco_91_to_80: + annotation_file = None + self.coco_metric = coco_evaluator.COCOEvaluator( + annotation_file=annotation_file, + include_mask=False, + need_rescale_bboxes=False, + per_category_metrics=self._task_config.per_category_metrics) + + return metrics + + def build_losses(self, outputs, labels, aux_losses=None): + """Build YOLO losses.""" + return self._loss_fn(labels, outputs) + + def train_step(self, inputs, model, optimizer, metrics=None): + """Train Step. + + Forward step and backwards propagate the model. + + Args: + inputs: a dictionary of input tensors. + model: the model, forward pass definition. + optimizer: the optimizer for this training step. + metrics: a nested structure of metrics objects. + + Returns: + A dictionary of logs. + """ + image, label = inputs + + with tf.GradientTape(persistent=False) as tape: + # Compute a prediction + y_pred = model(image, training=True) + + # Cast to float32 for gradietn computation + y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred) + + # Get the total loss + (scaled_loss, metric_loss, + loss_metrics) = self.build_losses(y_pred['raw_output'], label) + + # Scale the loss for numerical stability + if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer): + scaled_loss = optimizer.get_scaled_loss(scaled_loss) + + # Compute the gradient + train_vars = model.trainable_variables + gradients = tape.gradient(scaled_loss, train_vars) + + # Get unscaled loss if we are using the loss scale optimizer on fp16 + if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer): + gradients = optimizer.get_unscaled_gradients(gradients) + + # Apply gradients to the model + optimizer.apply_gradients(zip(gradients, train_vars)) + logs = {self.loss: metric_loss} + + # Compute all metrics + if metrics: + for m in metrics: + m.update_state(loss_metrics[m.name]) + logs.update({m.name: m.result()}) + return logs + + def _reorg_boxes(self, boxes, info, num_detections): + """Scale and Clean boxes prior to Evaluation.""" + mask = tf.sequence_mask(num_detections, maxlen=tf.shape(boxes)[1]) + mask = tf.cast(tf.expand_dims(mask, axis = -1), boxes.dtype) + + # Denormalize the boxes by the shape of the image + inshape = tf.expand_dims(info[:, 1, :], axis = 1) + ogshape = tf.expand_dims(info[:, 0, :], axis = 1) + scale = tf.expand_dims(info[:, 2, :], axis = 1) + offset = tf.expand_dims(info[:, 3, :], axis = 1) + + boxes = box_ops.denormalize_boxes(boxes, inshape) + boxes = box_ops.clip_boxes(boxes, inshape) + boxes += tf.tile(offset, [1, 1, 2]) + boxes /= tf.tile(scale, [1, 1, 2]) + boxes = box_ops.clip_boxes(boxes, ogshape) + + # Mask the boxes for usage + boxes *= mask + boxes += (mask - 1) + return boxes + + def validation_step(self, inputs, model, metrics=None): + """Validatation step. + + Args: + inputs: a dictionary of input tensors. + model: the keras.Model. + metrics: a nested structure of metrics objects. + + Returns: + A dictionary of logs. + """ + image, label = inputs + + # Step the model once + y_pred = model(image, training=False) + y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred) + (_, metric_loss, loss_metrics) = self.build_losses(y_pred['raw_output'], + label) + logs = {self.loss: metric_loss} + + # Reorganize and rescale the boxes + info = label['groundtruths']['image_info'] + boxes = self._reorg_boxes(y_pred['bbox'], info, y_pred["num_detections"]) + + # Build the input for the coc evaluation metric + coco_model_outputs = { + 'detection_boxes': boxes, + 'detection_scores': y_pred['confidence'], + 'detection_classes': y_pred['classes'], + 'num_detections': y_pred['num_detections'], + 'source_id': label['groundtruths']['source_id'], + 'image_info': label['groundtruths']['image_info'] + } + + # Compute all metrics + if metrics: + logs.update( + {self.coco_metric.name: (label['groundtruths'], coco_model_outputs)}) + for m in metrics: + m.update_state(loss_metrics[m.name]) + logs.update({m.name: m.result()}) + return logs + + def aggregate_logs(self, state=None, step_outputs=None): + """Get Metric Results.""" + if not state: + self.coco_metric.reset_states() + state = self.coco_metric + self.coco_metric.update_state(step_outputs[self.coco_metric.name][0], + step_outputs[self.coco_metric.name][1]) + return state + + def reduce_aggregated_logs(self, aggregated_logs, global_step=None): + """Reduce logs and remove unneeded items. Update with COCO results.""" + res = self.coco_metric.result() + return res + + def initialize(self, model: tf.keras.Model): + """Loading pretrained checkpoint.""" + + if not self.task_config.init_checkpoint: + logging.info('Training from Scratch.') + return + + ckpt_dir_or_file = self.task_config.init_checkpoint + if tf.io.gfile.isdir(ckpt_dir_or_file): + ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file) + + # Restoring checkpoint. + if self.task_config.init_checkpoint_modules == 'all': + ckpt = tf.train.Checkpoint(**model.checkpoint_items) + status = ckpt.read(ckpt_dir_or_file) + status.expect_partial().assert_existing_objects_matched() + else: + ckpt_items = {} + if 'backbone' in self.task_config.init_checkpoint_modules: + ckpt_items.update(backbone=model.backbone) + if 'decoder' in self.task_config.init_checkpoint_modules: + ckpt_items.update(decoder=model.decoder) + + ckpt = tf.train.Checkpoint(**ckpt_items) + status = ckpt.read(ckpt_dir_or_file) + status.expect_partial().assert_existing_objects_matched() + + logging.info('Finished loading pretrained checkpoint from %s', + ckpt_dir_or_file) + + def create_optimizer(self, + optimizer_config: OptimizationConfig, + runtime_config: Optional[RuntimeConfig] = None): + """Creates an TF optimizer from configurations. + + Args: + optimizer_config: the parameters of the Optimization settings. + runtime_config: the parameters of the runtime. + + Returns: + A tf.optimizers.Optimizer object. + """ + opt_factory = optimization.YoloOptimizerFactory(optimizer_config) + # pylint: disable=protected-access + ema = opt_factory._use_ema + opt_factory._use_ema = False + + opt_type = opt_factory._optimizer_type + if opt_type == 'sgd_torch': + optimizer = opt_factory.build_optimizer(opt_factory.build_learning_rate()) + optimizer.set_bias_lr( + opt_factory.get_bias_lr_schedule(self._task_config.smart_bias_lr)) + optimizer.search_and_set_variable_groups(self._model.trainable_variables) + else: + optimizer = opt_factory.build_optimizer(opt_factory.build_learning_rate()) + opt_factory._use_ema = ema + + if ema: + logging.info('EMA is enabled.') + optimizer = opt_factory.add_ema(optimizer) + + # pylint: enable=protected-access + + if runtime_config and runtime_config.loss_scale: + use_float16 = runtime_config.mixed_precision_dtype == 'float16' + optimizer = performance.configure_optimizer( + optimizer, + use_graph_rewrite=False, + use_float16=use_float16, + loss_scale=runtime_config.loss_scale) + + return optimizer From ce48c8682a6b18f4e62ee2c3ea356dbaa8d6cf9b Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Sun, 14 Nov 2021 14:57:08 -0500 Subject: [PATCH 21/96] yolox head update --- .../projects/yolo/common/registry_imports.py | 4 +- .../yolo/configs/experiments/yolox/yolox.yaml | 14 +- .../vision/beta/projects/yolo/configs/head.py | 40 ++++++ .../beta/projects/yolo/configs/yolox.py | 12 +- .../yolo/modeling/decoders/yolo_decoder.py | 13 ++ .../beta/projects/yolo/modeling/factory.py | 56 +++++++- .../yolo/modeling/heads/yolox_head.py | 135 +++++++++++------- .../yolo/modeling/layers/nn_blocks.py | 49 ++++++- .../beta/projects/yolo/modeling/yolo_model.py | 109 -------------- .../vision/beta/projects/yolo/tasks/yolox.py | 6 +- 10 files changed, 252 insertions(+), 186 deletions(-) create mode 100644 official/vision/beta/projects/yolo/configs/head.py diff --git a/official/vision/beta/projects/yolo/common/registry_imports.py b/official/vision/beta/projects/yolo/common/registry_imports.py index e40d39856a7..fed597937ff 100644 --- a/official/vision/beta/projects/yolo/common/registry_imports.py +++ b/official/vision/beta/projects/yolo/common/registry_imports.py @@ -20,7 +20,7 @@ # import configs from official.vision.beta.projects.yolo.configs import darknet_classification -from official.vision.beta.projects.yolo.configs import yolo as yolo_config +from official.vision.beta.projects.yolo.configs import yolox as yolo_config # import modeling components from official.vision.beta.projects.yolo.modeling.backbones import darknet @@ -28,7 +28,7 @@ # import tasks from official.vision.beta.projects.yolo.tasks import image_classification -from official.vision.beta.projects.yolo.tasks import yolo as yolo_task +from official.vision.beta.projects.yolo.tasks import yolox as yolo_task # import optimization packages from official.vision.beta.projects.yolo.optimization import optimizer_factory diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index fe5ce334195..d7ac25718b3 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -1,5 +1,5 @@ -# --experiment_type=scaled_yolo -# mAP 47.6 +# --experiment_type=yolox +# mAP runtime: distribution_strategy: 'tpu' mixed_precision_dtype: 'float32' @@ -16,10 +16,10 @@ task: decoder: type: yolo_decoder yolo_decoder: - version: v4 - type: csp + version: vx + type: regular head: - smart_bias: true + num_classes: 80 detection_generator: box_type: 'all': scaled @@ -55,7 +55,7 @@ task: box: [36, 75], box: [76, 55], box: [72, 146], box: [142, 110], box: [192, 243], box: [459, 401]] train_data: - input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/train*' + input_path: '/content/train*' shuffle_buffer_size: 10000 parser: mosaic: @@ -71,4 +71,4 @@ task: aug_rand_translate: 0.1 area_thresh: 0.1 validation_data: - input_path: '/readahead/200M/placer/prod/home/tensorflow-performance-data/datasets/coco/val*' + input_path: '/content/val*' diff --git a/official/vision/beta/projects/yolo/configs/head.py b/official/vision/beta/projects/yolo/configs/head.py new file mode 100644 index 00000000000..8fb55fad1cc --- /dev/null +++ b/official/vision/beta/projects/yolo/configs/head.py @@ -0,0 +1,40 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Decoders configurations.""" +import dataclasses +from typing import Optional +from official.modeling import hyperparams +from official.vision.beta.configs import head + + +@dataclasses.dataclass +class YOLOXHead(hyperparams.Config): + """Builds Yolo decoder. + + If the name is specified, or version is specified we ignore input parameters + and use version and name defaults. + """ + num_classes: Optional[str] = None + width: Optional[str] =1.0 + strides: Optional[str] = [8, 16, 32] + in_channels: Optional[str] = [256, 512, 1024] + depthwise: Optional[str] = False + activation: Optional[str] = 'silu' + + +@dataclasses.dataclass +class Head(decoders.Decoder): + type: Optional[str] = 'yolox_head' + yolox_head: YOLOXHead = YOLOXHead() diff --git a/official/vision/beta/projects/yolo/configs/yolox.py b/official/vision/beta/projects/yolo/configs/yolox.py index c4414ba535e..a1357bbd32a 100644 --- a/official/vision/beta/projects/yolo/configs/yolox.py +++ b/official/vision/beta/projects/yolo/configs/yolox.py @@ -136,8 +136,8 @@ class YOLOXHead(hyperparams.Config): """Parameterization for the YOLOX Head.""" num_classes: int = None width: float = 1.0 - strides: List[int] = [8, 16, 32] - in_channels: List[int] = [256, 512, 1024] + # strides: List[int] = [8, 16, 32] + # in_channels: List[int] = [256, 512, 1024] depthwise: bool = False activation: str = 'silu' @@ -270,8 +270,8 @@ def yolox() -> cfg.ExperimentConfig: ]) -@exp_factory.register_config_factory('yolox') -def yolox() -> cfg.ExperimentConfig: +@exp_factory.register_config_factory('yolox_regular') +def yolox_regular() -> cfg.ExperimentConfig: """COCO object detection with YOLOvx.""" train_batch_size = 64 eval_batch_size = 8 @@ -291,7 +291,7 @@ def yolox() -> cfg.ExperimentConfig: model=YOLOX( darknet_based_model=True, norm_activation=common.NormActivation(use_sync_bn=True), - head=YoloHead(num_classes=80), + head=YOLOXHead(num_classes=80), loss=YoloLoss(use_scaled_loss=False, update_on_repeat=True), anchor_boxes=AnchorBoxes( anchors_per_scale=3, @@ -388,4 +388,4 @@ def yolox() -> cfg.ExperimentConfig: 'task.validation_data.is_training != None' ]) - return config + return config \ No newline at end of file diff --git a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py index 5b2c434ac3d..afed067b5d4 100644 --- a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py +++ b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py @@ -81,6 +81,19 @@ max_level_process_len=None, path_process_len=6), ), + 'vx': + dict( + regular=dict( + embed_spp=True, + use_fpn=False, + max_level_process_len=None, + path_process_len=6), + fpn=dict( + embed_spp=True, + use_fpn=True, + max_level_process_len=None, + path_process_len=6), + ), } diff --git a/official/vision/beta/projects/yolo/modeling/factory.py b/official/vision/beta/projects/yolo/modeling/factory.py index a841131062a..1e1591c8095 100644 --- a/official/vision/beta/projects/yolo/modeling/factory.py +++ b/official/vision/beta/projects/yolo/modeling/factory.py @@ -15,12 +15,13 @@ """Contains common factory functions yolo neural networks.""" from absl import logging +from loguru import logger from official.vision.beta.modeling.backbones import factory as backbone_factory from official.vision.beta.modeling.decoders import factory as decoder_factory -from official.vision.beta.projects.yolo.configs import yolo +from official.vision.beta.projects.yolo.configs import yolo, yolox from official.vision.beta.projects.yolo.modeling import yolo_model -from official.vision.beta.projects.yolo.modeling.heads import yolo_head +from official.vision.beta.projects.yolo.modeling.heads import yolo_head, yolox_head from official.vision.beta.projects.yolo.modeling.layers import detection_generator @@ -66,6 +67,18 @@ def build_yolo_head(input_specs, model_config: yolo.Yolo, l2_regularization): smart_bias=model_config.head.smart_bias) return head +def build_yolox_head(input_specs, model_config: yolox.yolox): + """Builds yolo head.""" + head = yolox_head.YOLOXHead( + num_classes=model_config.num_classes, + # boxes_per_level=model_config.anchor_boxes.anchors_per_scale, + # norm_momentum=model_config.norm_activation.norm_momentum, + # norm_epsilon=model_config.norm_activation.norm_epsilon, + # kernel_regularizer=l2_regularization, + # smart_bias=model_config.head.smart_bias + ) + return head + def build_yolo(input_specs, model_config, l2_regularization): """Builds yolo model.""" @@ -91,5 +104,44 @@ def build_yolo(input_specs, model_config, l2_regularization): model.summary(print_fn=logging.info) + losses = detection_generator_obj.get_losses() + return model, losses + +def build_yolox(input_specs, model_config, l2_regularization): + """Builds yolox model.""" + backbone = model_config.backbone.get() + anchor_dict, _ = model_config.anchor_boxes.get( + backbone.min_level, backbone.max_level) + backbone = backbone_factory.build_backbone(input_specs, model_config.backbone, + model_config.norm_activation, + l2_regularization) + decoder = decoder_factory.build_decoder(backbone.output_specs, model_config, + l2_regularization) + + head = build_yolox_head(decoder.output_specs, model_config) + detection_generator_obj = build_yolo_detection_generator(model_config, + anchor_dict) + + model = yolo_model.Yolo( + backbone=backbone, + decoder=decoder, + head=head, + detection_generator=detection_generator_obj) + + # from tensorflow.keras import Input + # from tensorflow.keras import Model + # h, w = (640,640) + # inputs = Input(shape=(h, w,3)) + # outputs = model(inputs) + # model=Model(inputs,outputs) + # model.summary() + + model.build(input_specs.shape) + + model.summary(print_fn=logging.info) + logger.info(model.summary) + logger.info(model) + + losses = detection_generator_obj.get_losses() return model, losses diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index eb3fa04fff8..743bd853218 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -14,11 +14,14 @@ # Lint as: python3 """Yolox heads.""" +# import sys +# sys.path.append("/content/drive/MyDrive/tf-models/official/vision/beta/projects") +from loguru import logger import tensorflow as tf from tensorflow.keras.models import Sequential -from yolo.modeling.layers import nn_blocks +from official.vision.beta.projects.yolo.modeling.layers import nn_blocks from tensorflow.keras.layers import Concatenate -from yolo.ops import box_ops +from official.vision.beta.projects.yolo.ops import box_ops class YOLOXHead(tf.keras.layers.Layer): """YOLOX Prediction Head.""" @@ -42,106 +45,117 @@ def __init__( """ super().__init__(**kwargs) + # self._min_level = min_level + # self._max_level = max_level + + self._key_list = [ + str(key) for key in range(3, 6) + ] self._n_anchors = 1 self._num_classes = num_classes self._decode_in_inference = True - self._cls_convs = [] - self._reg_convs = [] + self._cls_convs = dict() + self._reg_convs = dict() - self._cls_preds = [] - self._reg_preds = [] - self._obj_preds = [] - self._stems = [] + self._cls_preds = dict() + self._reg_preds = dict() + self._obj_preds = dict() + self._stems = dict() self.prior_prob = 1e-2 self.bias=-tf.math.log((1-self.prior_prob)/self.prior_prob) - Conv = nn_blocks.DWConv if depthwise else nn_blocks.ConvBN - for i in range(len(in_channels)): - self._stems.append( - nn_blocks.ConvBN( + Conv = nn_blocks.DWConv if depthwise else nn_blocks.BaseConv + for k in self._key_list: + self._stems[k] = nn_blocks.BaseConv( filters=int(256 * width), kernel_size=1, - strides=(1, 1), + strides=1, padding='same', # TODO - use_bn = True, + # use_bn = True, activation=act, - ), ) - - self._cls_convs.append ( - Sequential( + # for i in range(len(in_channels)): + # self._stems.append( + # nn_blocks.BaseConv( + # filters=int(256 * width), + # kernel_size=1, + # strides=(1, 1), + # padding='same', # TODO + # # use_bn = True, + # activation=act, + # ), + # ) + + self._cls_convs[k] = Sequential( [ - nn_blocks.ConvBN( + Conv( filters=int(256 * width), kernel_size=3, - strides=(1, 1), - use_bn = True, + strides=1, + # use_bn = True, activation=act, ), - nn_blocks.ConvBN( + Conv( filters=int(256 * width), kernel_size=3, - strides=(1, 1), - use_bn = True, + strides=1, + # use_bn = True, activation=act, ), ] ) - ) + - self._reg_convs.append( - Sequential( + self._reg_convs[k] = Sequential( [ Conv( filters=int(256 * width), kernel_size=3, - strides=(1, 1), - use_bn = True, + strides=1, + # use_bn = True, activation=act, ), Conv( filters=int(256 * width), kernel_size=3, - strides=(1, 1), - use_bn = True, + strides=1, + # use_bn = True, activation=act, ), ] ) - ) + - self._cls_preds.append( - tf.keras.layers.Conv2D( + self._cls_preds[k] = tf.keras.layers.Conv2D( filters=self._n_anchors * self._num_classes, kernel_size=1, - strides=(1, 1), - padding='valid', + strides=1, + padding='same', bias_initializer=tf.keras.initializers.constant(self.bias) - ), - ) - self._reg_preds.append( - tf.keras.layers.Conv2D( + ) + + self._reg_preds[k] = tf.keras.layers.Conv2D( filters=4, kernel_size=1, - strides=(1, 1), - padding='valid', - ), - ) - self._obj_preds.append( - tf.keras.layers.Conv2D( + strides=1, + padding='same', + ) + + self._obj_preds[k] = tf.keras.layers.Conv2D( filters=self._n_anchors * 1, kernel_size=1, - strides=(1, 1), - padding='valid', + strides=1, + padding='same', bias_initializer=tf.keras.initializers.constant(self.bias) - ), - ) + ) + + # self.use_l1 = False # self.l1_loss = tf.keras.losses.MAE # TODO need reduce_mean after the loss @@ -149,11 +163,21 @@ def __init__( # self.iou_loss = box_ops.compute_iou # self.strides = strides # self.grids = [tf.zeros(1)] * len(in_channels) - @tf.function + # @tf.function def call(self, inputs, *args, **kwargs): - outputs=[] - for k,x in enumerate(inputs): - x = self.stems[k](x) + outputs=dict() + # logger.info("inputs = {}".format(inputs)) + # inputs_list = list() + # for key, value in inputs.items(): + # inputs_list.append(value) + # logger.info("inputs_list = {}".format(inputs_list)) + # inputs_tuple = tuple(inputs_list) + # logger.info("inputs_tuple = {}".format(inputs_tuple)) + + + # for k,x in inputs.items(): + for k in self._key_list: + x = self._stems[k](inputs[k]) cls_x = x reg_x = x @@ -164,7 +188,8 @@ def call(self, inputs, *args, **kwargs): reg_output = self._reg_preds[k](reg_feat) obj_output = self._obj_preds[k](reg_feat) output=Concatenate(-1)([reg_output,obj_output,cls_output]) - outputs.append(output) + outputs[k] = output + #TODO flatten return outputs # def call(self, xin, labels=None, imgs=None): diff --git a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py index e21ec4b38d1..752f4179bfb 100644 --- a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py +++ b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +from loguru import logger """Contains common building blocks for yolo neural networks.""" from typing import Callable, List, Tuple @@ -19,7 +19,10 @@ from official.modeling import tf_utils from official.vision.beta.ops import spatial_transform_ops - +import tensorflow.keras.backend as K +from tensorflow.keras import Sequential +from tensorflow.keras.layers import Layer,Conv2D,LeakyReLU,\ + MaxPool2D,UpSampling2D,Activation,ReLU,BatchNormalization,concatenate @tf.keras.utils.register_keras_serializable(package='yolo') class Identity(tf.keras.layers.Layer): @@ -1726,6 +1729,48 @@ def call(self, x, training=None): ], axis=-1) +class SiLU(Layer): + def __init__(self, *args, **kwargs): + super(SiLU,self).__init__(*args, **kwargs) + + def call(self,x,**kwargs): + return x*K.sigmoid(x) + + def get_config(self): + config = super(SiLU, self).get_config() + return config + + def compute_output_shape(self, input_shape): + return input_shape + +def get_activation(name="silu", ): + if name == "silu": + module = SiLU() + elif name == "relu": + module = ReLU() + elif name == "lrelu": + module = LeakyReLU(0.1 ) + else: + raise AttributeError("Unsupported act type: {}".format(name)) + return module + +class BaseConv(tf.keras.layers.Layer): + def __init__(self,filters, kernel_size, strides=1,padding='same',groups=1,use_bias=False, apply_batchnorm=True, activation="silu"): + super(BaseConv, self).__init__() + self.conv=Conv2D(filters,kernel_size=kernel_size,padding=padding,strides=strides,groups=groups, + use_bias=use_bias, + ) + self.apply_batchnorm=apply_batchnorm + self.bn=BatchNormalization(momentum=0.03,epsilon=1e-3) + self.activation=get_activation(activation) + + def call(self,inputs,**kwargs): + x=self.conv(inputs) + if self.apply_batchnorm: + x=self.bn(x) + x=self.activation(x) + + return x class DWConv(tf.keras.layers.Layer): def __init__ (self, diff --git a/official/vision/beta/projects/yolo/modeling/yolo_model.py b/official/vision/beta/projects/yolo/modeling/yolo_model.py index 06f79750ea8..eafe40adde8 100644 --- a/official/vision/beta/projects/yolo/modeling/yolo_model.py +++ b/official/vision/beta/projects/yolo/modeling/yolo_model.py @@ -1,110 +1 @@ # Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Yolo models.""" - -from typing import Mapping, Union -import tensorflow as tf -from official.vision.beta.projects.yolo.modeling.layers import nn_blocks - - -class Yolo(tf.keras.Model): - """The YOLO model class.""" - - def __init__(self, - backbone, - decoder, - head, - detection_generator, - **kwargs): - """Detection initialization function. - - Args: - backbone: `tf.keras.Model` a backbone network. - decoder: `tf.keras.Model` a decoder network. - head: `RetinaNetHead`, the RetinaNet head. - detection_generator: the detection generator. - **kwargs: keyword arguments to be passed. - """ - super(Yolo, self).__init__(**kwargs) - - self._config_dict = { - 'backbone': backbone, - 'decoder': decoder, - 'head': head, - 'detection_generator': detection_generator - } - - # model components - self._backbone = backbone - self._decoder = decoder - self._head = head - self._detection_generator = detection_generator - self._fused = False - return - - def call(self, inputs, training=False): - maps = self.backbone(inputs) - decoded_maps = self.decoder(maps) - raw_predictions = self.head(decoded_maps) - if training: - return {'raw_output': raw_predictions} - else: - # Post-processing. - predictions = self.detection_generator(raw_predictions) - predictions.update({'raw_output': raw_predictions}) - return predictions - - @property - def backbone(self): - return self._backbone - - @property - def decoder(self): - return self._decoder - - @property - def head(self): - return self._head - - @property - def detection_generator(self): - return self._detection_generator - - def get_config(self): - return self._config_dict - - @classmethod - def from_config(cls, config): - return cls(**config) - - @property - def checkpoint_items( - self) -> Mapping[str, Union[tf.keras.Model, tf.keras.layers.Layer]]: - """Returns a dictionary of items to be additionally checkpointed.""" - items = dict(backbone=self.backbone, head=self.head) - if self.decoder is not None: - items.update(decoder=self.decoder) - return items - - def fuse(self): - """Fuses all Convolution and Batchnorm layers to get better latency.""" - print('Fusing Conv Batch Norm Layers.') - if not self._fused: - self._fused = True - for layer in self.submodules: - if isinstance(layer, nn_blocks.ConvBN): - layer.fuse() - self.summary() - return diff --git a/official/vision/beta/projects/yolo/tasks/yolox.py b/official/vision/beta/projects/yolo/tasks/yolox.py index dcdb5cbb993..2479294b6e1 100644 --- a/official/vision/beta/projects/yolo/tasks/yolox.py +++ b/official/vision/beta/projects/yolo/tasks/yolox.py @@ -30,7 +30,7 @@ from official.vision.beta.evaluation import coco_evaluator from official.vision.beta.ops import box_ops from official.vision.beta.projects.yolo import optimization -from official.vision.beta.projects.yolo.configs import yolo as exp_cfg +from official.vision.beta.projects.yolo.configs import yolox as exp_cfg from official.vision.beta.projects.yolo.dataloaders import tf_example_decoder from official.vision.beta.projects.yolo.dataloaders import yolo_input from official.vision.beta.projects.yolo.modeling import factory @@ -74,7 +74,7 @@ def build_model(self): l2_regularizer = ( tf.keras.regularizers.l2(l2_weight_decay) if l2_weight_decay else None) model, losses = factory.build_yolox( - input_specs, model_base_cfg) + input_specs, model_base_cfg, l2_regularizer) # save for later usage within the task. self._loss_fn = losses @@ -405,4 +405,4 @@ def create_optimizer(self, use_float16=use_float16, loss_scale=runtime_config.loss_scale) - return optimizer + return optimizer \ No newline at end of file From 2ac97f2c1ebf79b195b94f33a184e88a091d39dc Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Sun, 14 Nov 2021 14:58:02 -0500 Subject: [PATCH 22/96] yolox head udpated --- .../beta/projects/yolo/modeling/yolo_model.py | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/official/vision/beta/projects/yolo/modeling/yolo_model.py b/official/vision/beta/projects/yolo/modeling/yolo_model.py index eafe40adde8..06f79750ea8 100644 --- a/official/vision/beta/projects/yolo/modeling/yolo_model.py +++ b/official/vision/beta/projects/yolo/modeling/yolo_model.py @@ -1 +1,110 @@ # Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Yolo models.""" + +from typing import Mapping, Union +import tensorflow as tf +from official.vision.beta.projects.yolo.modeling.layers import nn_blocks + + +class Yolo(tf.keras.Model): + """The YOLO model class.""" + + def __init__(self, + backbone, + decoder, + head, + detection_generator, + **kwargs): + """Detection initialization function. + + Args: + backbone: `tf.keras.Model` a backbone network. + decoder: `tf.keras.Model` a decoder network. + head: `RetinaNetHead`, the RetinaNet head. + detection_generator: the detection generator. + **kwargs: keyword arguments to be passed. + """ + super(Yolo, self).__init__(**kwargs) + + self._config_dict = { + 'backbone': backbone, + 'decoder': decoder, + 'head': head, + 'detection_generator': detection_generator + } + + # model components + self._backbone = backbone + self._decoder = decoder + self._head = head + self._detection_generator = detection_generator + self._fused = False + return + + def call(self, inputs, training=False): + maps = self.backbone(inputs) + decoded_maps = self.decoder(maps) + raw_predictions = self.head(decoded_maps) + if training: + return {'raw_output': raw_predictions} + else: + # Post-processing. + predictions = self.detection_generator(raw_predictions) + predictions.update({'raw_output': raw_predictions}) + return predictions + + @property + def backbone(self): + return self._backbone + + @property + def decoder(self): + return self._decoder + + @property + def head(self): + return self._head + + @property + def detection_generator(self): + return self._detection_generator + + def get_config(self): + return self._config_dict + + @classmethod + def from_config(cls, config): + return cls(**config) + + @property + def checkpoint_items( + self) -> Mapping[str, Union[tf.keras.Model, tf.keras.layers.Layer]]: + """Returns a dictionary of items to be additionally checkpointed.""" + items = dict(backbone=self.backbone, head=self.head) + if self.decoder is not None: + items.update(decoder=self.decoder) + return items + + def fuse(self): + """Fuses all Convolution and Batchnorm layers to get better latency.""" + print('Fusing Conv Batch Norm Layers.') + if not self._fused: + self._fused = True + for layer in self.submodules: + if isinstance(layer, nn_blocks.ConvBN): + layer.fuse() + self.summary() + return From 5050eab6e17cab257eaafc4015ddf08c1971aad5 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Mon, 15 Nov 2021 15:09:16 -0500 Subject: [PATCH 23/96] yolox decoder update --- .../beta/projects/yolo/modeling/decoders/yolo_decoder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py index afed067b5d4..bad0c1bf258 100644 --- a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py +++ b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py @@ -87,12 +87,12 @@ embed_spp=True, use_fpn=False, max_level_process_len=None, - path_process_len=6), + path_process_len=5), fpn=dict( embed_spp=True, use_fpn=True, max_level_process_len=None, - path_process_len=6), + path_process_len=5), ), } From 8db04202a9a4c29832b08d11bfc13a0951a45294 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Tue, 16 Nov 2021 11:31:36 -0500 Subject: [PATCH 24/96] yolox head update --- .../yolo/modeling/heads/yolox_head.py | 101 +----------------- 1 file changed, 2 insertions(+), 99 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index 743bd853218..85d35b1cc31 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -14,8 +14,6 @@ # Lint as: python3 """Yolox heads.""" -# import sys -# sys.path.append("/content/drive/MyDrive/tf-models/official/vision/beta/projects") from loguru import logger import tensorflow as tf from tensorflow.keras.models import Sequential @@ -45,8 +43,6 @@ def __init__( """ super().__init__(**kwargs) - # self._min_level = min_level - # self._max_level = max_level self._key_list = [ str(key) for key in range(3, 6) @@ -77,17 +73,6 @@ def __init__( # use_bn = True, activation=act, ) - # for i in range(len(in_channels)): - # self._stems.append( - # nn_blocks.BaseConv( - # filters=int(256 * width), - # kernel_size=1, - # strides=(1, 1), - # padding='same', # TODO - # # use_bn = True, - # activation=act, - # ), - # ) self._cls_convs[k] = Sequential( [ @@ -107,8 +92,6 @@ def __init__( ), ] ) - - self._reg_convs[k] = Sequential( [ @@ -129,9 +112,6 @@ def __init__( ] ) - - - self._cls_preds[k] = tf.keras.layers.Conv2D( filters=self._n_anchors * self._num_classes, kernel_size=1, @@ -154,28 +134,10 @@ def __init__( padding='same', bias_initializer=tf.keras.initializers.constant(self.bias) ) - - - - # self.use_l1 = False - # self.l1_loss = tf.keras.losses.MAE # TODO need reduce_mean after the loss - # self.bcewithlog_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True) - # self.iou_loss = box_ops.compute_iou - # self.strides = strides - # self.grids = [tf.zeros(1)] * len(in_channels) - # @tf.function - def call(self, inputs, *args, **kwargs): - outputs=dict() - # logger.info("inputs = {}".format(inputs)) - # inputs_list = list() - # for key, value in inputs.items(): - # inputs_list.append(value) - # logger.info("inputs_list = {}".format(inputs_list)) - # inputs_tuple = tuple(inputs_list) - # logger.info("inputs_tuple = {}".format(inputs_tuple)) - # for k,x in inputs.items(): + def call(self, inputs, *args, **kwargs): + outputs=dict() for k in self._key_list: x = self._stems[k](inputs[k]) cls_x = x @@ -192,63 +154,4 @@ def call(self, inputs, *args, **kwargs): #TODO flatten return outputs -# def call(self, xin, labels=None, imgs=None): -# outputs = [] -# origin_preds = [] -# x_shifts = [] -# y_shifts = [] -# expanded_strides = [] - -# for k, (cls_conv, reg_conv, stride_this_level, x) in enumerate( -# zip(self.cls_convs, self.reg_convs, self.strides, xin) -# ): -# x = self.stems[k](x) -# cls_x = x -# reg_x = x - -# # cls - feature and output -# cls_feat = cls_conv(cls_x) -# cls_output = self.cls_preds[k](cls_feat) - -# # reg & obj - feature and output -# reg_feat = reg_conv(reg_x) -# reg_output = self.reg_preds[k](reg_feat) -# obj_output = self.obj_preds[k](reg_feat) -# output=Concatenate(-1)([reg_output,obj_output,cls_output]) -# outputs.append(output) -# return outputs - # if self.training: - # output = tf.concat([reg_output, obj_output, cls_output], 1) - # output, grid = self.get_output_and_grid( - # output, k, stride_this_level, xin[0].type() - # ) - # x_shifts.append(grid[:, :, 0]) - # y_shifts.append(grid[:, :, 1]) - # expanded_strides.append( - # tf.zeros(1, grid.shape[1]) - # .fill_(stride_this_level) - # .type_as(xin[0]) - # ) - # if self.use_l1: - # batch_size = reg_output.shape[0] - # hsize, wsize = reg_output.shape[-2:] - # reg_output = reg_output.view( - # batch_size, self.n_anchors, 4, hsize, wsize - # ) - # reg_output = reg_output.permute(0, 1, 3, 4, 2).reshape( - # # reg_output: batch_size * n_anchors * 4 * height * width - # # permute: batch_size * n_anchors * height * width * 4 - # # reshape: [batch_size, N_anchors height width, 4] - # # This is also called "origin_pred" - # batch_size, -1, 4 - # ) - # origin_preds.append(reg_output.clone()) - - # else: # not training - # output = tf.concat( - # [reg_output, obj_output.sigmoid(), cls_output.sigmoid()], 1 - # ) - - # outputs.append(output) - # return outputs \ No newline at end of file From 1d30b0dc0feddfee6f601719a2e4743d9a2428c0 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Fri, 19 Nov 2021 17:24:57 -0500 Subject: [PATCH 25/96] Anchor.py copied from tensorflowmodels --- .../vision/beta/projects/yolo/ops/anchor.py | 372 +++++++++--------- 1 file changed, 192 insertions(+), 180 deletions(-) diff --git a/official/vision/beta/projects/yolo/ops/anchor.py b/official/vision/beta/projects/yolo/ops/anchor.py index dfe675984a7..91528bab8d9 100644 --- a/official/vision/beta/projects/yolo/ops/anchor.py +++ b/official/vision/beta/projects/yolo/ops/anchor.py @@ -16,41 +16,39 @@ import numpy as np import tensorflow as tf -from official.vision.beta.projects.yolo.ops import box_ops -from official.vision.beta.projects.yolo.ops import loss_utils -from official.vision.beta.projects.yolo.ops import preprocessing_ops +from yolo.ops import box_ops +from yolo.ops import preprocessing_ops +from yolo.ops import loss_utils INF = 10000000 - def get_best_anchor(y_true, anchors, stride, width=1, height=1, iou_thresh=0.25, - best_match_only=False, + best_match_only=False, use_tie_breaker=True): """Get the correct anchor that is assoiciated with each box using IOU. - + Args: y_true: tf.Tensor[] for the list of bounding boxes in the yolo format. - anchors: list or tensor for the anchor boxes to be used in prediction found - via Kmeans. - stride: `int` stride for the anchors. + anchors: list or tensor for the anchor boxes to be used in prediction + found via Kmeans. width: int for the image width. height: int for the image height. - iou_thresh: `float` the minimum iou threshold to use for selecting boxes for - each level. - best_match_only: `bool` if the box only has one match and it is less than - the iou threshold, when set to True, this match will be dropped as no - anchors can be linked to it. - use_tie_breaker: `bool` if there is many anchors for a given box, then - attempt to use all of them, if False, only the first matching box will be - used. - Returns: - tf.Tensor: y_true with the anchor associated with each ground truth box - known + iou_thresh: `float` the minimum iou threshold to use for selecting boxes for + each level. + best_match_only: `bool` if the box only has one match and it is less than + the iou threshold, when set to True, this match will be dropped as no + anchors can be linked to it. + use_tie_breaker: `bool` if there is many anchors for a given box, then + attempt to use all of them, if False, only the first matching box will + be used. + Return: + tf.Tensor: y_true with the anchor associated with each ground truth + box known """ with tf.name_scope('get_best_anchor'): width = tf.cast(width, dtype=tf.float32) @@ -61,7 +59,7 @@ def get_best_anchor(y_true, true_wh = tf.cast(y_true[..., 2:4], dtype=tf.float32) * scaler # scale down from large anchor to small anchor type - anchors = tf.cast(anchors, dtype=tf.float32) / stride + anchors = tf.cast(anchors, dtype=tf.float32)/stride k = tf.shape(anchors)[0] @@ -93,7 +91,9 @@ def get_best_anchor(y_true, iou_type=3, ) values, indexes = tf.math.top_k( - iou_raw, k=tf.cast(k, dtype=tf.int32), sorted=True) + iou_raw, + k=tf.cast(k, dtype=tf.int32), + sorted=True) ind_mask = tf.cast(values >= iou_thresh, dtype=indexes.dtype) # pad the indexs such that all values less than the thresh are -1 @@ -104,67 +104,63 @@ def get_best_anchor(y_true, elif use_tie_breaker: iou_index = tf.concat([ tf.expand_dims(indexes[..., 0], axis=-1), - ((indexes[..., 1:] + 1) * ind_mask[..., 1:]) - 1 - ], - axis=-1) + ((indexes[..., 1:] + 1) * ind_mask[..., 1:]) - 1], axis=-1) else: iou_index = tf.concat([ tf.expand_dims(indexes[..., 0], axis=-1), - tf.zeros_like(indexes[..., 1:]) - 1 - ], - axis=-1) + tf.zeros_like(indexes[..., 1:]) - 1], axis=-1) return tf.cast(iou_index, dtype=tf.float32), tf.cast(values, dtype=tf.float32) - class YoloAnchorLabeler: - """Anchor labeler for the Yolo Models.""" - - def __init__(self, - anchors=None, - anchor_free_level_limits=None, - level_strides=None, - center_radius=None, - max_num_instances=200, - match_threshold=0.25, - best_matches_only=False, - use_tie_breaker=True, - darknet=False, - dtype='float32'): - """Initialization for anchor labler. - - Args: + """Anchor labeler for the Yolo Models""" + + def __init__(self, + anchors = None, + anchor_free_level_limits = None, + level_strides = None, + center_radius = None, + max_num_instances = 200, + match_threshold = 0.25, + best_matches_only = False, + use_tie_breaker = True, + darknet = False, + dtype = 'float32'): + """Initialization for anchor labler. + + Args: anchors: `Dict[List[Union[int, float]]]` values for each anchor box. - anchor_free_level_limits: `List` the box sizes that will be allowed at - each FPN level as is done in the FCOS and YOLOX paper for anchor free + anchor_free_level_limits: `List` the box sizes that will be allowed at + each FPN level as is done in the FCOS and YOLOX paper for anchor free box assignment. - level_strides: `Dict[int]` for how much the model scales down the images - at the each level. - center_radius: `Dict[float]` for radius around each box center to search + level_strides: `Dict[int]` for how much the model scales down the + images at the each level. + center_radius: `Dict[float]` for radius around each box center to search for extra centers in each level. max_num_instances: `int` for the number of boxes to compute loss on. - match_threshold: `float` indicating the threshold over which an anchor - will be considered for prediction, at zero, all the anchors will be used - and at 1.0 only the best will be used. for anchor thresholds larger than - 1.0 we stop using the IOU for anchor comparison and resort directly to - comparing the width and height, this is used for the scaled models. - best_matches_only: `boolean` indicating how boxes are selected for - optimization. - use_tie_breaker: `boolean` indicating whether to use the anchor threshold + match_threshold: `float` indicating the threshold over which an anchor + will be considered for prediction, at zero, all the anchors will be used + and at 1.0 only the best will be used. for anchor thresholds larger than + 1.0 we stop using the IOU for anchor comparison and resort directly to + comparing the width and height, this is used for the scaled models. + best_matches_only: `boolean` indicating how boxes are selected for + optimization. + use_tie_breaker: `boolean` indicating whether to use the anchor threshold value. - darknet: `boolean` indicating which data pipeline to use. Setting to True + darknet: `boolean` indicating which data pipeline to use. Setting to True swaps the pipeline to output images realtive to Yolov4 and older. - dtype: `str` indicating the output datatype of the datapipeline selecting + dtype: `str` indicating the output datatype of the datapipeline selecting from {"float32", "float16", "bfloat16"}. """ self.anchors = anchors self.masks = self._get_mask() + self.use_tie_breaker = use_tie_breaker self.anchor_free_level_limits = self._get_level_limits( - anchor_free_level_limits) + anchor_free_level_limits) if darknet and self.anchor_free_level_limits is None: center_radius = None - + self.keys = self.anchors.keys() if self.anchor_free_level_limits is not None: maxim = 2000 @@ -172,8 +168,7 @@ def __init__(self, self.num_instances = {key: maxim for key in self.keys} elif not darknet: self.num_instances = { - key: (6 - i) * max_num_instances for i, key in enumerate(self.keys) - } + key: (6 - i) * max_num_instances for i, key in enumerate(self.keys)} else: self.num_instances = {key: max_num_instances for key in self.keys} @@ -181,7 +176,6 @@ def __init__(self, self.level_strides = level_strides self.match_threshold = match_threshold self.best_matches_only = best_matches_only - self.use_tie_breaker = use_tie_breaker self.dtype = dtype def _get_mask(self): @@ -196,15 +190,26 @@ def _get_mask(self): masks[str(i)] = list(range(start, per_scale + start)) start += per_scale return masks - + def _get_level_limits(self, level_limits): """For each level receptive feild range for anchor free box placement.""" if level_limits is not None: level_limits_dict = {} level_limits = [0.0] + level_limits + [np.inf] + k = 0 for i, key in enumerate(self.anchors.keys()): - level_limits_dict[key] = level_limits[i:i + 2] + level_limits_dict[key] = [] + + base = k + for j, lst in enumerate(self.anchors[key]): + # level_limits_dict[key].append(level_limits[k:k + 2]) + if self.use_tie_breaker: + base = k + level_limits_dict[key].append([level_limits[base], level_limits[k + 1]]) + k += 1 + level_limits_dict[key] = tf.convert_to_tensor(level_limits_dict[key]) + print(level_limits_dict) else: level_limits_dict = None return level_limits_dict @@ -213,112 +218,112 @@ def _tie_breaking_search(self, anchors, mask, boxes, classes): """After search, link each anchor ind to the correct map in ground truth.""" mask = tf.cast(tf.reshape(mask, [1, 1, 1, -1]), anchors.dtype) anchors = tf.expand_dims(anchors, axis=-1) - viable = tf.where(tf.squeeze(anchors == mask, axis=0)) + viable = tf.where(tf.squeeze(anchors == mask, axis = 0)) - gather_id, _, anchor_id = tf.split(viable, 3, axis=-1) + gather_id, _, anchor_id = tf.split(viable, 3, axis = -1) boxes = tf.gather_nd(boxes, gather_id) classes = tf.gather_nd(classes, gather_id) - - classes = tf.expand_dims(classes, axis=-1) + + classes = tf.expand_dims(classes, axis = -1) classes = tf.cast(classes, boxes.dtype) anchor_id = tf.cast(anchor_id, boxes.dtype) return boxes, classes, anchor_id - def _get_anchor_id(self, - key, - boxes, - classes, - width, - height, - stride, - iou_index=None): - """Find the object anchor assignments in an anchor based paradigm.""" - + def _get_anchor_id(self, key, boxes, classes, width, height, stride, + iou_index = None): + """Find the object anchor assignments in an anchor based paradigm. """ + # find the best anchor anchors = self.anchors[key] num_anchors = len(anchors) if self.best_matches_only: # get the best anchor for each box - iou_index, _ = get_best_anchor( - boxes, - anchors, - stride, - width=width, - height=height, - best_match_only=True, - iou_thresh=self.match_threshold) + iou_index, _ = get_best_anchor(boxes, anchors, stride, + width=width, height=height, + best_match_only=True, + iou_thresh=self.match_threshold) mask = range(num_anchors) - else: + else: # search is done across FPN levels, get the mask of anchor indexes - # corralated to this level. + # corralated to this level. mask = self.masks[key] # search for the correct box to use - (boxes, classes, - anchors) = self._tie_breaking_search(iou_index, mask, boxes, classes) + (boxes, classes, anchors) = self._tie_breaking_search(iou_index, mask, + boxes, classes) return boxes, classes, anchors, num_anchors def _get_centers(self, boxes, classes, anchors, width, height, scale_xy): - """Find the object center assignments in an anchor based paradigm.""" + """Find the object center assignments in an anchor based paradigm. """ offset = tf.cast(0.5 * (scale_xy - 1), boxes.dtype) - grid_xy, _ = tf.split(boxes, 2, axis=-1) + grid_xy, _ = tf.split(boxes, 2, axis = -1) wh_scale = tf.cast(tf.convert_to_tensor([width, height]), boxes.dtype) grid_xy = grid_xy * wh_scale centers = tf.math.floor(grid_xy) - if offset != 0.0: - clamp = lambda x, ma: tf.maximum( # pylint:disable=g-long-lambda + if offset != 0.0: + clamp = lambda x, ma: tf.maximum( tf.minimum(x, tf.cast(ma, x.dtype)), tf.zeros_like(x)) grid_xy_index = grid_xy - centers positive_shift = ((grid_xy_index < offset) & (grid_xy > 1.)) - negative_shift = ((grid_xy_index > (1 - offset)) & (grid_xy < - (wh_scale - 1.))) + negative_shift = ( + (grid_xy_index > (1 - offset)) & (grid_xy < (wh_scale - 1.))) - zero, _ = tf.split(tf.ones_like(positive_shift), 2, axis=-1) - shift_mask = tf.concat([zero, positive_shift, negative_shift], axis=-1) - offset = tf.cast([[0, 0], [1, 0], [0, 1], [-1, 0], [0, -1]], - offset.dtype) * offset + zero , _ = tf.split(tf.ones_like(positive_shift), 2, axis = -1) + shift_mask = tf.concat( + [zero, positive_shift, negative_shift], axis = -1) + offset = tf.cast([[0, 0], [1, 0], + [0, 1], [-1, 0], + [0, -1]], offset.dtype) * offset num_shifts = tf.shape(shift_mask) num_shifts = num_shifts[-1] - boxes = tf.tile(tf.expand_dims(boxes, axis=-2), [1, num_shifts, 1]) - classes = tf.tile(tf.expand_dims(classes, axis=-2), [1, num_shifts, 1]) - anchors = tf.tile(tf.expand_dims(anchors, axis=-2), [1, num_shifts, 1]) + boxes = tf.tile(tf.expand_dims(boxes, axis = -2), [1, num_shifts, 1]) + classes = tf.tile(tf.expand_dims(classes, axis = -2), [1, num_shifts, 1]) + anchors = tf.tile(tf.expand_dims(anchors, axis = -2), [1, num_shifts, 1]) shift_mask = tf.cast(shift_mask, boxes.dtype) - shift_ind = shift_mask * tf.range(0, num_shifts, dtype=boxes.dtype) + shift_ind = shift_mask * tf.range(0, num_shifts, dtype = boxes.dtype) shift_ind = shift_ind - (1 - shift_mask) - shift_ind = tf.expand_dims(shift_ind, axis=-1) + shift_ind = tf.expand_dims(shift_ind, axis = -1) - boxes_and_centers = tf.concat([boxes, classes, anchors, shift_ind], - axis=-1) + boxes_and_centers = tf.concat( + [boxes, classes, anchors, shift_ind], axis = -1) boxes_and_centers = tf.reshape(boxes_and_centers, [-1, 7]) - _, center_ids = tf.split(boxes_and_centers, [6, 1], axis=-1) + _, center_ids = tf.split(boxes_and_centers, [6, 1], axis = -1) + #center_ids = tf.squeeze(center_ids, axis = -1) select = tf.where(center_ids >= 0) - select, _ = tf.split(select, 2, axis=-1) + select, _ = tf.split(select, 2, axis = -1) boxes_and_centers = tf.gather_nd(boxes_and_centers, select) + # center_ids = tf.cast(center_ids, tf.int32) center_ids = tf.gather_nd(center_ids, select) center_ids = tf.cast(center_ids, tf.int32) shifts = tf.gather_nd(offset, center_ids) - boxes, classes, anchors, _ = tf.split( - boxes_and_centers, [4, 1, 1, 1], axis=-1) - grid_xy, _ = tf.split(boxes, 2, axis=-1) + boxes, classes, anchors, _ = tf.split(boxes_and_centers, + [4, 1, 1, 1], axis = -1) + grid_xy, _ = tf.split(boxes, 2, axis = -1) centers = tf.math.floor(grid_xy * wh_scale - shifts) centers = clamp(centers, wh_scale - 1) - - x, y = tf.split(centers, 2, axis=-1) - centers = tf.cast(tf.concat([y, x, anchors], axis=-1), tf.int32) + + x, y = tf.split(centers, 2, axis = -1) + centers = tf.cast(tf.concat([y, x, anchors], axis = -1), tf.int32) return boxes, classes, centers - def _get_anchor_free(self, key, boxes, classes, height, width, stride, + def _get_anchor_free(self, + key, + boxes, + classes, + height, + width, + stride, center_radius): """Find the box assignements in an anchor free paradigm.""" level_limits = self.anchor_free_level_limits[key] @@ -327,6 +332,7 @@ def _get_anchor_free(self, key, boxes, classes, height, width, stride, grid_points = tf.squeeze(grid_points, axis=0) box_list = boxes class_list = classes + num_anchors = 1 grid_points = (grid_points + 0.5) * stride x_centers, y_centers = grid_points[..., 0], grid_points[..., 1] @@ -337,7 +343,7 @@ def _get_anchor_free(self, key, boxes, classes, height, width, stride, boxes = tf.reshape(boxes, [1, 1, -1, 4]) tlbr_boxes = tf.reshape(tlbr_boxes, [1, 1, -1, 4]) if self.use_tie_breaker: - area = tf.reduce_prod(boxes[..., 2:], axis=-1) + area = tf.reduce_prod(boxes[..., 2:], axis = -1) # check if the box is in the receptive feild of the this fpn level b_t = y_centers - tlbr_boxes[..., 0] @@ -345,13 +351,18 @@ def _get_anchor_free(self, key, boxes, classes, height, width, stride, b_b = tlbr_boxes[..., 2] - y_centers b_r = tlbr_boxes[..., 3] - x_centers box_delta = tf.stack([b_t, b_l, b_b, b_r], axis=-1) + is_in_boxes = tf.reduce_min(box_delta, axis=-1) > 0.0 if level_limits is not None: max_reg_targets_per_im = tf.reduce_max(box_delta, axis=-1) - gt_min = max_reg_targets_per_im >= level_limits[0] - gt_max = max_reg_targets_per_im <= level_limits[1] - is_in_boxes = tf.logical_and(gt_min, gt_max) - else: - is_in_boxes = tf.reduce_min(box_delta, axis=-1) > 0.0 + level_limits = tf.cast(level_limits, max_reg_targets_per_im.dtype) + num_anchors = tf.shape(level_limits)[0] + + max_reg_targets_per_im = tf.expand_dims(max_reg_targets_per_im, axis = -1) + + gt_min = max_reg_targets_per_im >= level_limits[..., 0] + gt_max = max_reg_targets_per_im <= level_limits[..., 1] + is_in_level = tf.logical_and(gt_min, gt_max) + is_in_boxes_all = tf.reduce_any(is_in_boxes, axis=(0, 1), keepdims=True) # check if the center is in the receptive feild of the this fpn level @@ -368,17 +379,21 @@ def _get_anchor_free(self, key, boxes, classes, height, width, stride, is_in_boxes_and_center = tf.logical_and(is_in_boxes, is_in_centers) is_in_boxes_and_center = tf.logical_and(is_in_index, is_in_boxes_and_center) - if self.use_tie_breaker: - boxes_all = tf.cast(is_in_boxes_and_center, area.dtype) - boxes_all = ((boxes_all * area) + ((1 - boxes_all) * INF)) - boxes_min = tf.reduce_min(boxes_all, axis=-1, keepdims=True) - boxes_min = tf.where(boxes_min == INF, -1.0, boxes_min) - is_in_boxes_and_center = boxes_all == boxes_min + if level_limits is not None: + is_in_boxes_and_center = tf.expand_dims(is_in_boxes_and_center, axis = -1) + is_in_boxes_and_center = tf.logical_and(is_in_level, is_in_boxes_and_center) + + # if self.use_tie_breaker: + # boxes_all = tf.cast(is_in_boxes_and_center, area.dtype) + # boxes_all = ((boxes_all * area) + ((1 - boxes_all) * INF)) + # boxes_min = tf.reduce_min(boxes_all, axis = -1, keepdims = True) + # boxes_min = tf.where(boxes_min == INF, -1.0, boxes_min) + # is_in_boxes_and_center = boxes_all == boxes_min # construct the index update grid - reps = tf.reduce_sum(tf.cast(is_in_boxes_and_center, tf.int16), axis=-1) + reps = tf.reduce_sum(tf.cast(is_in_boxes_and_center, tf.int16), axis=-2) indexes = tf.cast(tf.where(is_in_boxes_and_center), tf.int32) - y, x, t = tf.split(indexes, 3, axis=-1) + y, x, t, a = tf.split(indexes, 4, axis=-1) boxes = tf.gather_nd(box_list, t) classes = tf.cast(tf.gather_nd(class_list, t), boxes.dtype) @@ -389,36 +404,37 @@ def _get_anchor_free(self, key, boxes, classes, height, width, stride, # return the samples and the indexes samples = tf.concat([boxes, conf, classes], axis=-1) - indexes = tf.concat([y, x, tf.zeros_like(t)], axis=-1) - return indexes, samples - - def build_label_per_path(self, - key, - boxes, - classes, - width, - height, - iou_index=None): + indexes = tf.concat([y, x, a], axis=-1) + return indexes, samples, num_anchors + + def build_label_per_path(self, + key, + boxes, + classes, + width, + height, + iou_index = None): """Builds the labels for one path.""" stride = self.level_strides[key] scale_xy = self.center_radius[key] if self.center_radius is not None else 1 - width = tf.cast(width // stride, boxes.dtype) - height = tf.cast(height // stride, boxes.dtype) - + width = tf.cast(width//stride, boxes.dtype) + height = tf.cast(height//stride, boxes.dtype) + if self.anchor_free_level_limits is None: - (boxes, classes, anchors, num_anchors) = self._get_anchor_id( - key, boxes, classes, width, height, stride, iou_index=iou_index) - boxes, classes, centers = self._get_centers(boxes, classes, anchors, + (boxes, classes, + anchors, num_anchors) = self._get_anchor_id(key, boxes, classes, + width, height, stride, + iou_index = iou_index) + boxes, classes, centers = self._get_centers(boxes, classes, anchors, width, height, scale_xy) ind_mask = tf.ones_like(classes) - updates = tf.concat([boxes, ind_mask, classes], axis=-1) + updates = tf.concat([boxes, ind_mask, classes], axis = -1) else: - num_anchors = 1 - (centers, updates) = self._get_anchor_free(key, boxes, classes, height, + (centers, updates, num_anchors) = self._get_anchor_free(key, boxes, classes, height, width, stride, scale_xy) - boxes, ind_mask, classes = tf.split(updates, [4, 1, 1], axis=-1) - + boxes, ind_mask, classes = tf.split(updates, [4, 1, 1], axis = -1) + width = tf.cast(width, tf.int32) height = tf.cast(height, tf.int32) full = tf.zeros([height, width, num_anchors, 1], dtype=classes.dtype) @@ -426,30 +442,30 @@ def build_label_per_path(self, num_instances = int(self.num_instances[key]) centers = preprocessing_ops.pad_max_instances( - centers, num_instances, pad_value=0, pad_axis=0) + centers, num_instances, pad_value=0, pad_axis=0) updates = preprocessing_ops.pad_max_instances( - updates, num_instances, pad_value=0, pad_axis=0) + updates, num_instances, pad_value=0, pad_axis=0) updates = tf.cast(updates, self.dtype) full = tf.cast(full, self.dtype) return centers, updates, full def __call__(self, boxes, classes, width, height): - """Builds the labels for a single image, not functional in batch mode. - - Args: - boxes: `Tensor` of shape [None, 4] indicating the object locations in an - image. + """Builds the labels for a single image, not functional in batch mode. + + Args: + boxes: `Tensor` of shape [None, 4] indicating the object locations in + an image. classes: `Tensor` of shape [None] indicating the each objects classes. - width: `int` for the images width. + width: `int` for the images width. height: `int` for the images height. - - Returns: - centers: `Tensor` of shape [None, 3] of indexes in the final grid where - boxes are located. + num_instances: `int` for the maximum number of expanded boxes to allow. + Returns: + centers: `Tensor` of shape [None, 3] of indexes in the final grid where + boxes are located. updates: `Tensor` of shape [None, 8] the value to place in the final grid. - full: `Tensor` of [width/stride, height/stride, num_anchors, 1] holding - a mask of where boxes are locates for confidence losses. + full: `Tensor` of [width/stride, height/stride, num_anchors, 1] holding + a mask of where boxes are locates for confidence losses. """ indexes = {} updates = {} @@ -457,7 +473,7 @@ def __call__(self, boxes, classes, width, height): iou_index = None boxes = box_ops.yxyx_to_xcycwh(boxes) - if not self.best_matches_only and self.anchor_free_level_limits is None: + if not self.best_matches_only and self.anchor_free_level_limits is None: # stitch and search boxes across fpn levels anchorsvec = [] for stitch in self.anchors: @@ -465,17 +481,13 @@ def __call__(self, boxes, classes, width, height): stride = tf.cast([width, height], boxes.dtype) # get the best anchor for each box - iou_index, _ = get_best_anchor( - boxes, - anchorsvec, - stride, - width=1.0, - height=1.0, - best_match_only=False, - use_tie_breaker=self.use_tie_breaker, - iou_thresh=self.match_threshold) + iou_index, _ = get_best_anchor(boxes, anchorsvec, stride, + width=1.0, height=1.0, + best_match_only=False, + use_tie_breaker=self.use_tie_breaker, + iou_thresh=self.match_threshold) for key in self.keys: indexes[key], updates[key], true_grids[key] = self.build_label_per_path( - key, boxes, classes, width, height, iou_index=iou_index) - return indexes, updates, true_grids + key, boxes, classes, width, height, iou_index = iou_index) + return indexes, updates, true_grids From ba9e168fad2f9e6d8915f66d6e986b52f1ffc436 Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Tue, 30 Nov 2021 23:16:32 -0500 Subject: [PATCH 26/96] Update yolox_head.py --- .../vision/beta/projects/yolo/modeling/heads/yolox_head.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index 85d35b1cc31..9078288829a 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -38,6 +38,7 @@ def __init__( """ Args: + num_classes: `int`, number of classes per category. act (str): activation type of conv. Defalut value: "silu". depthwise (bool): whether apply depthwise conv in conv branch. Defalut value: False. """ @@ -69,7 +70,7 @@ def __init__( filters=int(256 * width), kernel_size=1, strides=1, - padding='same', # TODO + padding='same', # use_bn = True, activation=act, ) @@ -154,4 +155,4 @@ def call(self, inputs, *args, **kwargs): #TODO flatten return outputs - \ No newline at end of file + From 761b61c7e0cce34d2f06700a9fbd3b166a4bcfe6 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Thu, 2 Dec 2021 15:06:17 -0500 Subject: [PATCH 27/96] Update yolox.yaml --- .../projects/yolo/configs/experiments/yolox/yolox.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index d7ac25718b3..f3a69b33ae8 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -22,7 +22,7 @@ task: num_classes: 80 detection_generator: box_type: - 'all': scaled + 'all': anchor_free scale_xy: 'all': 2.0 max_boxes: 300 @@ -50,10 +50,8 @@ task: use_sync_bn: true num_classes: 80 anchor_boxes: - anchors_per_scale: 3 - boxes: [box: [12, 16], box: [19, 36], box: [40, 28], - box: [36, 75], box: [76, 55], box: [72, 146], - box: [142, 110], box: [192, 243], box: [459, 401]] + level_limits: [64, 128] + anchors_per_scale: 1 train_data: input_path: '/content/train*' shuffle_buffer_size: 10000 From 341873870500fd3fd4a0216a978219ba92b830ae Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Thu, 2 Dec 2021 15:19:04 -0500 Subject: [PATCH 28/96] Update yolox.yaml --- .../beta/projects/yolo/configs/experiments/yolox/yolox.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index f3a69b33ae8..95bb15d220a 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -1,9 +1,9 @@ # --experiment_type=yolox # mAP runtime: - distribution_strategy: 'tpu' + distribution_strategy: 'mirrored' mixed_precision_dtype: 'float32' - tpu_enable_xla_dynamic_padder: false + num_gpus: 1 task: model: input_size: [640, 640, 3] From 344fce66157a7f918dd5b948ba9e8b544b54a7ea Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Fri, 3 Dec 2021 18:23:48 -0500 Subject: [PATCH 29/96] Update yolox_head.py --- .../yolo/modeling/heads/yolox_head.py | 97 +++++++++++-------- 1 file changed, 57 insertions(+), 40 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index 9078288829a..19775e0e847 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -26,12 +26,24 @@ class YOLOXHead(tf.keras.layers.Layer): def __init__( self, - num_classes, - width=1.0, + min_level, + max_level, + classes=80, + boxes_per_level=1, + output_extras=0, + norm_momentum=0.99, + norm_epsilon=0.001, + kernel_initializer='VarianceScaling', + kernel_regularizer=None, + bias_regularizer=None, + activation='silu', + smart_bias=False, + use_separable_conv=False, + width_scaling = 1.0, strides=[8, 16, 32], in_channels=[256, 512, 1024], - act='silu', depthwise=False, + prior_prob = 1e-2, **kwargs ): @@ -44,14 +56,20 @@ def __init__( """ super().__init__(**kwargs) + self._min_level = min_level + self._max_level = max_level self._key_list = [ - str(key) for key in range(3, 6) + str(key) for key in range(self._min_level, self._max_level + 1) ] - self._n_anchors = 1 - self._num_classes = num_classes - self._decode_in_inference = True + self._classes = classes + self._boxes_per_level = boxes_per_level + self._output_extras = output_extras + + self._smart_bias = smart_bias + self._use_separable_conv = use_separable_conv + self._prior_prob = prior_prob self._cls_convs = dict() self._reg_convs = dict() @@ -61,35 +79,35 @@ def __init__( self._obj_preds = dict() self._stems = dict() - self.prior_prob = 1e-2 - self.bias=-tf.math.log((1-self.prior_prob)/self.prior_prob) + + self._bias= -tf.math.log((1 - self._prior_prob) / self._prior_prob) Conv = nn_blocks.DWConv if depthwise else nn_blocks.BaseConv for k in self._key_list: self._stems[k] = nn_blocks.BaseConv( - filters=int(256 * width), - kernel_size=1, - strides=1, + filters=int(256 * width_scaling), + kernel_size=(1,1), + strides=(1,1), padding='same', # use_bn = True, - activation=act, + activation=activation, ) self._cls_convs[k] = Sequential( [ Conv( - filters=int(256 * width), - kernel_size=3, - strides=1, + filters=int(256 * width_scaling), + kernel_size=(3,3), + strides=(1,1), # use_bn = True, - activation=act, + activation=activation, ), Conv( - filters=int(256 * width), - kernel_size=3, - strides=1, + filters=int(256 * width_scaling), + kernel_size=(3,3), + strides=(1,1), # use_bn = True, - activation=act, + activation=activation, ), ] ) @@ -97,43 +115,43 @@ def __init__( self._reg_convs[k] = Sequential( [ Conv( - filters=int(256 * width), - kernel_size=3, - strides=1, + filters=int(256 * width_scaling), + kernel_size=(3,3), + strides=(1,1), # use_bn = True, - activation=act, + activation=activation, ), Conv( - filters=int(256 * width), - kernel_size=3, - strides=1, + filters=int(256 * width_scaling), + kernel_size=(3,3), + strides=(1,1), # use_bn = True, - activation=act, + activation=activation, ), ] ) self._cls_preds[k] = tf.keras.layers.Conv2D( - filters=self._n_anchors * self._num_classes, - kernel_size=1, - strides=1, + filters=self._boxes_per_level * self._classes, + kernel_size=(1,1), + strides=(1,1), padding='same', - bias_initializer=tf.keras.initializers.constant(self.bias) + bias_initializer=tf.keras.initializers.constant(self._bias) ) self._reg_preds[k] = tf.keras.layers.Conv2D( filters=4, - kernel_size=1, - strides=1, + kernel_size=(1,1), + strides=(1,1), padding='same', ) self._obj_preds[k] = tf.keras.layers.Conv2D( - filters=self._n_anchors * 1, - kernel_size=1, - strides=1, + filters=self._boxes_per_level * 1, + kernel_size=(1,1), + strides=(1,1), padding='same', - bias_initializer=tf.keras.initializers.constant(self.bias) + bias_initializer=tf.keras.initializers.constant(self._bias) ) @@ -155,4 +173,3 @@ def call(self, inputs, *args, **kwargs): #TODO flatten return outputs - From 233aea7888adf7897d675739ab6f39974a21631b Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Sun, 5 Dec 2021 13:28:34 -0500 Subject: [PATCH 30/96] Update factory.py --- .../beta/projects/yolo/modeling/factory.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/factory.py b/official/vision/beta/projects/yolo/modeling/factory.py index 1e1591c8095..c6d831fb53b 100644 --- a/official/vision/beta/projects/yolo/modeling/factory.py +++ b/official/vision/beta/projects/yolo/modeling/factory.py @@ -67,16 +67,19 @@ def build_yolo_head(input_specs, model_config: yolo.Yolo, l2_regularization): smart_bias=model_config.head.smart_bias) return head -def build_yolox_head(input_specs, model_config: yolox.yolox): +def build_yolox_head(input_specs, model_config: yolox.yolox, l2_regularization): """Builds yolo head.""" + min_level = min(map(int, input_specs.keys())) + max_level = max(map(int, input_specs.keys())) head = yolox_head.YOLOXHead( - num_classes=model_config.num_classes, - # boxes_per_level=model_config.anchor_boxes.anchors_per_scale, - # norm_momentum=model_config.norm_activation.norm_momentum, - # norm_epsilon=model_config.norm_activation.norm_epsilon, - # kernel_regularizer=l2_regularization, - # smart_bias=model_config.head.smart_bias - ) + min_level=min_level, + max_level=max_level, + classes=model_config.num_classes, + boxes_per_level=model_config.anchor_boxes.anchors_per_scale, + norm_momentum=model_config.norm_activation.norm_momentum, + norm_epsilon=model_config.norm_activation.norm_epsilon, + kernel_regularizer=l2_regularization, + smart_bias=model_config.head.smart_bias) return head @@ -118,7 +121,7 @@ def build_yolox(input_specs, model_config, l2_regularization): decoder = decoder_factory.build_decoder(backbone.output_specs, model_config, l2_regularization) - head = build_yolox_head(decoder.output_specs, model_config) + head = build_yolox_head(decoder.output_specs, model_config, l2_regularization) detection_generator_obj = build_yolo_detection_generator(model_config, anchor_dict) From 303fb86fa704bc1fa3fb1fc6894b7ad4f7126c4e Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Sun, 5 Dec 2021 13:29:16 -0500 Subject: [PATCH 31/96] Update yolox.py --- official/vision/beta/projects/yolo/configs/yolox.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/yolox.py b/official/vision/beta/projects/yolo/configs/yolox.py index a1357bbd32a..9809dd49624 100644 --- a/official/vision/beta/projects/yolo/configs/yolox.py +++ b/official/vision/beta/projects/yolo/configs/yolox.py @@ -134,12 +134,13 @@ class DataConfig(cfg.DataConfig): @dataclasses.dataclass class YOLOXHead(hyperparams.Config): """Parameterization for the YOLOX Head.""" - num_classes: int = None + #classes: int = None width: float = 1.0 # strides: List[int] = [8, 16, 32] # in_channels: List[int] = [256, 512, 1024] depthwise: bool = False activation: str = 'silu' + smart_bias: bool = True @dataclasses.dataclass @@ -291,7 +292,7 @@ def yolox_regular() -> cfg.ExperimentConfig: model=YOLOX( darknet_based_model=True, norm_activation=common.NormActivation(use_sync_bn=True), - head=YOLOXHead(num_classes=80), + head=YOLOXHead(smart_bias=True), loss=YoloLoss(use_scaled_loss=False, update_on_repeat=True), anchor_boxes=AnchorBoxes( anchors_per_scale=3, @@ -388,4 +389,4 @@ def yolox_regular() -> cfg.ExperimentConfig: 'task.validation_data.is_training != None' ]) - return config \ No newline at end of file + return config From 58234e87ceb7c7bc25f7111907e0410aa67b5252 Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Sun, 5 Dec 2021 13:30:06 -0500 Subject: [PATCH 32/96] Update yolox.yaml --- .../beta/projects/yolo/configs/experiments/yolox/yolox.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 95bb15d220a..a5088aff9f4 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -19,7 +19,8 @@ task: version: vx type: regular head: - num_classes: 80 + + smart_bias: true detection_generator: box_type: 'all': anchor_free From 7602a378bd4bdfdb1b1c114e57149f8c7ed18e90 Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Sun, 5 Dec 2021 13:30:24 -0500 Subject: [PATCH 33/96] Update yolox.yaml --- .../beta/projects/yolo/configs/experiments/yolox/yolox.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index a5088aff9f4..103358a9486 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -19,7 +19,6 @@ task: version: vx type: regular head: - smart_bias: true detection_generator: box_type: From 2bc6ceb7c7744d34310fec7aaa94c0be9ed81a27 Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Sun, 5 Dec 2021 14:07:22 -0500 Subject: [PATCH 34/96] Update yolox_head.py --- .../yolo/modeling/heads/yolox_head.py | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index 19775e0e847..c598ae73172 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -18,7 +18,6 @@ import tensorflow as tf from tensorflow.keras.models import Sequential from official.vision.beta.projects.yolo.modeling.layers import nn_blocks -from tensorflow.keras.layers import Concatenate from official.vision.beta.projects.yolo.ops import box_ops class YOLOXHead(tf.keras.layers.Layer): @@ -136,39 +135,49 @@ def __init__( kernel_size=(1,1), strides=(1,1), padding='same', - bias_initializer=tf.keras.initializers.constant(self._bias) - ) + bias_initializer=tf.keras.initializers.constant(self._bias)) self._reg_preds[k] = tf.keras.layers.Conv2D( filters=4, kernel_size=(1,1), strides=(1,1), - padding='same', - ) + padding='same') self._obj_preds[k] = tf.keras.layers.Conv2D( filters=self._boxes_per_level * 1, kernel_size=(1,1), strides=(1,1), padding='same', - bias_initializer=tf.keras.initializers.constant(self._bias) - ) + bias_initializer=tf.keras.initializers.constant(self._bias)) + + def build(self, input_shape): + self._cls_head = dict() + self._obj_head = dict() + self._reg_head = dict() + for key in self._key_list: + self._cls_head[key] = Sequential() + self._cls_head[key].add(self._stems[key]) + self._cls_head[key].add(self._cls_convs[key]) + self._cls_head[key].add(self._cls_preds[key]) + + self._obj_head[key] = Sequential() + self._obj_head[key].add(self._stems[key]) + self._obj_head[key].add(self._reg_convs[key]) + self._obj_head[key].add(self._obj_preds[key]) + + self._reg_head[key] = Sequential() + self._reg_head[key].add(self._stems[key]) + self._reg_head[key].add(self._reg_convs[key]) + self._reg_head[key].add(self._reg_preds[key]) def call(self, inputs, *args, **kwargs): outputs=dict() for k in self._key_list: - x = self._stems[k](inputs[k]) - cls_x = x - reg_x = x - - cls_feat = self._cls_convs[k](cls_x) - cls_output = self._cls_preds[k](cls_feat) - - reg_feat = self._reg_convs[k](reg_x) - reg_output = self._reg_preds[k](reg_feat) - obj_output = self._obj_preds[k](reg_feat) - output=Concatenate(-1)([reg_output,obj_output,cls_output]) + cls_output = self._cls_head[k](inputs[k]) + reg_output = self._reg_head[k](inputs[k]) + obj_output = self._obj_head[k](inputs[k]) + output=tf.concat([reg_output,obj_output,cls_output], axis = -1) outputs[k] = output #TODO flatten From d97fdb704a10f98493c84f81cad30cc2fe6dc81f Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Sun, 5 Dec 2021 14:11:29 -0500 Subject: [PATCH 35/96] Add some comments --- .../yolo/modeling/heads/yolox_head.py | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index c598ae73172..5fe4de47812 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -43,15 +43,30 @@ def __init__( in_channels=[256, 512, 1024], depthwise=False, prior_prob = 1e-2, - **kwargs - ): + **kwargs): + """YoloX Prediction Head initialization function. - """ Args: - num_classes: `int`, number of classes per category. - act (str): activation type of conv. Defalut value: "silu". - depthwise (bool): whether apply depthwise conv in conv branch. Defalut value: False. + min_level: `int`, the minimum backbone output level. + max_level: `int`, the maximum backbone output level. + classes: `int`, number of classes per category. + boxes_per_level: `int`, number of boxes to predict per level. + output_extras: `int`, number of additional output channels that the head. + should predict for non-object detection and non-image classification + tasks. + norm_momentum: `float`, normalization momentum for the moving average. + norm_epsilon: `float`, small float added to variance to avoid dividing by + zero. + kernel_initializer: kernel_initializer for convolutional layers. + kernel_regularizer: tf.keras.regularizers.Regularizer object for Conv2D. + bias_regularizer: tf.keras.regularizers.Regularizer object for Conv2d. + activation: `str`, the activation function to use typically leaky or mish. + smart_bias: `bool`, whether to use smart bias. + use_separable_conv: `bool` wether to use separable convs. + width_scaling: `float`, factor by which the filters should be scaled + depthwise (bool): whether apply depthwise conv in conv branch. Defalut value: False. + **kwargs: keyword arguments to be passed. """ super().__init__(**kwargs) @@ -179,6 +194,5 @@ def call(self, inputs, *args, **kwargs): obj_output = self._obj_head[k](inputs[k]) output=tf.concat([reg_output,obj_output,cls_output], axis = -1) outputs[k] = output - #TODO flatten - + #Outputs are not flattened here. return outputs From 828a85512ba212457355d101f625619d1fbc42b2 Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Mon, 6 Dec 2021 13:30:22 -0500 Subject: [PATCH 36/96] Create yolox_head_test.py --- .../yolo/modeling/heads/yolox_head_test.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py new file mode 100644 index 00000000000..01537bff285 --- /dev/null +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py @@ -0,0 +1,75 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Tests for yolo heads.""" + +# Import libraries +from absl.testing import parameterized +import tensorflow as tf +import sys +sys.path.append("/content/tf-models") +from official.vision.beta.projects.yolo.modeling.heads import yolox_head as heads + + +class YoloDecoderTest(parameterized.TestCase, tf.test.TestCase): + + def test_network_creation(self): + """Test creation of YOLO family models.""" + tf.keras.backend.set_image_data_format('channels_last') + input_shape = { + '3': [1, 52, 52, 256], + '4': [1, 26, 26, 512], + '5': [1, 13, 13, 1024] + } + classes = 100 + bps = 1 + head = heads.YOLOXHead(3, 5, classes=classes, boxes_per_level=bps) + + inputs = {} + for key in input_shape: + inputs[key] = tf.ones(input_shape[key], dtype=tf.float32) + + endpoints = head(inputs) + # print(endpoints) + + for key in endpoints.keys(): + expected_input_shape = input_shape[key] + expected_input_shape[-1] = (classes + 5) * bps + self.assertAllEqual(endpoints[key].shape.as_list(), expected_input_shape) + + def test_serialize_deserialize(self): + # Create a network object that sets all of its config options. + tf.keras.backend.set_image_data_format('channels_last') + input_shape = { + '3': [1, 52, 52, 256], + '4': [1, 26, 26, 512], + '5': [1, 13, 13, 1024] + } + classes = 100 + bps = 1 + head = heads.YOLOXHead(3, 5, classes=classes, boxes_per_level=bps) + + inputs = {} + for key in input_shape: + inputs[key] = tf.ones(input_shape[key], dtype=tf.float32) + + _ = head(inputs) + configs = head.get_config() + head_from_config = heads.YOLOXHead.from_config(configs) + self.assertAllEqual(head.get_config(), head_from_config.get_config()) + + +if __name__ == '__main__': + tf.test.main() From f4aba5c06230a2e193a393c3ed0843af69a12f64 Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Mon, 6 Dec 2021 13:31:54 -0500 Subject: [PATCH 37/96] Update yolox_head_test.py --- .../beta/projects/yolo/modeling/heads/yolox_head_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py index 01537bff285..890ff622078 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py @@ -18,8 +18,7 @@ # Import libraries from absl.testing import parameterized import tensorflow as tf -import sys -sys.path.append("/content/tf-models") + from official.vision.beta.projects.yolo.modeling.heads import yolox_head as heads From 3ed54264499a7a35cfec1b385eb82c1b259d5c6b Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Mon, 6 Dec 2021 13:33:35 -0500 Subject: [PATCH 38/96] Add config methods --- .../beta/projects/yolo/modeling/heads/yolox_head.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index 5fe4de47812..27b5ff9b532 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -196,3 +196,16 @@ def call(self, inputs, *args, **kwargs): outputs[k] = output #Outputs are not flattened here. return outputs + + def get_config(self): + config = dict( + min_level=self._min_level, + max_level=self._max_level, + classes=self._classes, + boxes_per_level=self._boxes_per_level, + output_extras=self._output_extras) + return config + + @classmethod + def from_config(cls, config, custom_objects=None): + return cls(**config) From 12b90217a4f76bb9cf1a62516832e236cbea1967 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Mon, 6 Dec 2021 14:33:56 -0500 Subject: [PATCH 39/96] Update Convs in yolox_head --- .../yolo/modeling/heads/yolox_head.py | 79 +++++++++++-------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index 27b5ff9b532..7cba1924324 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -41,7 +41,6 @@ def __init__( width_scaling = 1.0, strides=[8, 16, 32], in_channels=[256, 512, 1024], - depthwise=False, prior_prob = 1e-2, **kwargs): @@ -80,71 +79,90 @@ def __init__( self._classes = classes self._boxes_per_level = boxes_per_level self._output_extras = output_extras - + self._width_scaling = width_scaling self._smart_bias = smart_bias self._use_separable_conv = use_separable_conv self._prior_prob = prior_prob + self._stems = dict() + + self._bias= -tf.math.log((1 - self._prior_prob) / self._prior_prob) + + self._base_config = dict( + activation=activation, + norm_momentum=norm_momentum, + norm_epsilon=norm_epsilon, + kernel_initializer=kernel_initializer, + kernel_regularizer=kernel_regularizer, + bias_regularizer=bias_regularizer) + + + + def build(self, input_shape): + self._cls_convs = dict() self._reg_convs = dict() self._cls_preds = dict() self._reg_preds = dict() self._obj_preds = dict() - self._stems = dict() + self._cls_head = dict() + self._obj_head = dict() + self._reg_head = dict() - self._bias= -tf.math.log((1 - self._prior_prob) / self._prior_prob) - - Conv = nn_blocks.DWConv if depthwise else nn_blocks.BaseConv for k in self._key_list: - self._stems[k] = nn_blocks.BaseConv( - filters=int(256 * width_scaling), + self._stems[k] = nn_blocks.ConvBN( + filters=int(256 * self._width_scaling), kernel_size=(1,1), strides=(1,1), - padding='same', - # use_bn = True, - activation=activation, + use_bn=True, + use_separable_conv=self._use_separable_conv, + **self._base_config, ) self._cls_convs[k] = Sequential( [ - Conv( - filters=int(256 * width_scaling), + nn_blocks.ConvBN( + filters=int(256 * self._width_scaling), kernel_size=(3,3), strides=(1,1), - # use_bn = True, - activation=activation, + use_bn=True, + use_separable_conv=self._use_separable_conv, + **self._base_config, ), - Conv( - filters=int(256 * width_scaling), + nn_blocks.ConvBN( + filters=int(256 * self._width_scaling), kernel_size=(3,3), strides=(1,1), - # use_bn = True, - activation=activation, + use_bn=True, + use_separable_conv=self._use_separable_conv, + **self._base_config, ), ] ) self._reg_convs[k] = Sequential( [ - Conv( - filters=int(256 * width_scaling), + nn_blocks.ConvBN( + filters=int(256 * self._width_scaling), kernel_size=(3,3), strides=(1,1), - # use_bn = True, - activation=activation, + use_bn=True, + use_separable_conv=self._use_separable_conv, + **self._base_config, ), - Conv( - filters=int(256 * width_scaling), + nn_blocks.ConvBN( + filters=int(256 * self._width_scaling), kernel_size=(3,3), strides=(1,1), - # use_bn = True, - activation=activation, + use_bn=True, + use_separable_conv=self._use_separable_conv, + **self._base_config, ), ] ) - + self._cls_preds[k] = tf.keras.layers.Conv2D( filters=self._boxes_per_level * self._classes, kernel_size=(1,1), @@ -165,11 +183,6 @@ def __init__( padding='same', bias_initializer=tf.keras.initializers.constant(self._bias)) - def build(self, input_shape): - self._cls_head = dict() - self._obj_head = dict() - self._reg_head = dict() - for key in self._key_list: self._cls_head[key] = Sequential() self._cls_head[key].add(self._stems[key]) From 701424d05dfe3db98b673769e0f6fa499b7ee120 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Mon, 6 Dec 2021 17:04:23 -0500 Subject: [PATCH 40/96] yolo task updated --- .../projects/yolo/common/registry_imports.py | 7 ++- .../yolo/modeling/layers/nn_blocks.py | 49 +------------------ .../vision/beta/projects/yolo/tasks/yolox.py | 8 +-- 3 files changed, 10 insertions(+), 54 deletions(-) diff --git a/official/vision/beta/projects/yolo/common/registry_imports.py b/official/vision/beta/projects/yolo/common/registry_imports.py index fed597937ff..2879577c236 100644 --- a/official/vision/beta/projects/yolo/common/registry_imports.py +++ b/official/vision/beta/projects/yolo/common/registry_imports.py @@ -20,7 +20,8 @@ # import configs from official.vision.beta.projects.yolo.configs import darknet_classification -from official.vision.beta.projects.yolo.configs import yolox as yolo_config +from official.vision.beta.projects.yolo.configs import yolo as yolo_config +from official.vision.beta.projects.yolo.configs import yolox as yolox_config # import modeling components from official.vision.beta.projects.yolo.modeling.backbones import darknet @@ -28,7 +29,9 @@ # import tasks from official.vision.beta.projects.yolo.tasks import image_classification -from official.vision.beta.projects.yolo.tasks import yolox as yolo_task +from official.vision.beta.projects.yolo.tasks import yolo as yolo_task +from official.vision.beta.projects.yolo.tasks import yolox as yolox_task + # import optimization packages from official.vision.beta.projects.yolo.optimization import optimizer_factory diff --git a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py index 752f4179bfb..33295e4d126 100644 --- a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py +++ b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from loguru import logger """Contains common building blocks for yolo neural networks.""" from typing import Callable, List, Tuple @@ -1753,51 +1752,5 @@ def get_activation(name="silu", ): else: raise AttributeError("Unsupported act type: {}".format(name)) return module - -class BaseConv(tf.keras.layers.Layer): - def __init__(self,filters, kernel_size, strides=1,padding='same',groups=1,use_bias=False, apply_batchnorm=True, activation="silu"): - super(BaseConv, self).__init__() - self.conv=Conv2D(filters,kernel_size=kernel_size,padding=padding,strides=strides,groups=groups, - use_bias=use_bias, - ) - self.apply_batchnorm=apply_batchnorm - self.bn=BatchNormalization(momentum=0.03,epsilon=1e-3) - self.activation=get_activation(activation) - - def call(self,inputs,**kwargs): - x=self.conv(inputs) - if self.apply_batchnorm: - x=self.bn(x) - x=self.activation(x) - - return x - -class DWConv(tf.keras.layers.Layer): - def __init__ (self, - filters, - kernel_size, - strides=1, - padding='valid', # TODO: need to make sure this is correct - groups=1, - use_biase=False, - use_bn=True, - activation='silu', - **kwargs): - super().__init__(**kwargs) - self._conv1 = ConvBN(filters, - kernel_size, - strides=strides, - use_bn=use_bn, - activation=activation) - self._conv2 = ConvBN(filters, - kernel_size=1, - strides=1, - use_bn=use_bn, - activation=activation) - - def call(self, inputs, **kwargs): - x = self._conv1(inputs) - x = self._conv2(x) - return x - + \ No newline at end of file diff --git a/official/vision/beta/projects/yolo/tasks/yolox.py b/official/vision/beta/projects/yolo/tasks/yolox.py index 2479294b6e1..39b3bc1d498 100644 --- a/official/vision/beta/projects/yolo/tasks/yolox.py +++ b/official/vision/beta/projects/yolo/tasks/yolox.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Contains classes used to train Yolo.""" +"""Contains classes used to train Yolox.""" import collections from typing import Optional @@ -46,7 +46,7 @@ class YOLOXTask(base_task.Task): """A single-replica view of training procedure. - YOLO task provides artifacts for training/evalution procedures, including + YOLOX task provides artifacts for training/evalution procedures, including loading/iterating over Datasets, initializing the model, calculating the loss, post-processing, and customized metrics with reduction. """ @@ -64,7 +64,7 @@ def __init__(self, params, logging_dir: Optional[str] = None): return def build_model(self): - """Build an instance of Yolo.""" + """Build an instance of Yolox.""" model_base_cfg = self.task_config.model l2_weight_decay = self.task_config.weight_decay / 2.0 @@ -202,7 +202,7 @@ def build_metrics(self, training=True): return metrics def build_losses(self, outputs, labels, aux_losses=None): - """Build YOLO losses.""" + """Build YOLOX losses.""" return self._loss_fn(labels, outputs) def train_step(self, inputs, model, optimizer, metrics=None): From aeba188b6496ad416883d1c0e2b9000154464771 Mon Sep 17 00:00:00 2001 From: Gautam Fotedar Date: Mon, 6 Dec 2021 19:24:25 -0500 Subject: [PATCH 41/96] Incremental Linting Commit --- official/vision/beta/projects/yolo/configs/head.py | 2 +- .../beta/projects/yolo/modeling/decoders/yolo_decoder.py | 6 +++--- official/vision/beta/projects/yolo/modeling/factory.py | 2 +- official/vision/beta/projects/yolo/tasks/yolox.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/head.py b/official/vision/beta/projects/yolo/configs/head.py index 8fb55fad1cc..26a0678f8f1 100644 --- a/official/vision/beta/projects/yolo/configs/head.py +++ b/official/vision/beta/projects/yolo/configs/head.py @@ -27,7 +27,7 @@ class YOLOXHead(hyperparams.Config): and use version and name defaults. """ num_classes: Optional[str] = None - width: Optional[str] =1.0 + width: Optional[str] = 1.0 strides: Optional[str] = [8, 16, 32] in_channels: Optional[str] = [256, 512, 1024] depthwise: Optional[str] = False diff --git a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py index bad0c1bf258..68af65861fd 100644 --- a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py +++ b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py @@ -50,8 +50,8 @@ max_level_process_len=None, csp_stack=7, fpn_depth=7, - max_fpn_depth=5, - max_csp_stack=5, + max_fpn_depth=5, + max_csp_stack=5, path_process_len=8, fpn_filter_scale=1), csp_xlarge=dict( @@ -84,7 +84,7 @@ 'vx': dict( regular=dict( - embed_spp=True, + embed_spp=True, use_fpn=False, max_level_process_len=None, path_process_len=5), diff --git a/official/vision/beta/projects/yolo/modeling/factory.py b/official/vision/beta/projects/yolo/modeling/factory.py index c6d831fb53b..59ae209e6f5 100644 --- a/official/vision/beta/projects/yolo/modeling/factory.py +++ b/official/vision/beta/projects/yolo/modeling/factory.py @@ -130,7 +130,7 @@ def build_yolox(input_specs, model_config, l2_regularization): decoder=decoder, head=head, detection_generator=detection_generator_obj) - + # from tensorflow.keras import Input # from tensorflow.keras import Model # h, w = (640,640) diff --git a/official/vision/beta/projects/yolo/tasks/yolox.py b/official/vision/beta/projects/yolo/tasks/yolox.py index 2479294b6e1..81c00294d6b 100644 --- a/official/vision/beta/projects/yolo/tasks/yolox.py +++ b/official/vision/beta/projects/yolo/tasks/yolox.py @@ -405,4 +405,4 @@ def create_optimizer(self, use_float16=use_float16, loss_scale=runtime_config.loss_scale) - return optimizer \ No newline at end of file + return optimizer From b71a0a32d5e4d9cfa0bf40ecb8ac37f790906c7a Mon Sep 17 00:00:00 2001 From: Gautam Fotedar Date: Mon, 6 Dec 2021 19:57:49 -0500 Subject: [PATCH 42/96] Linting commit --- .../yolo/modeling/decoders/yolo_decoder.py | 12 +-- .../beta/projects/yolo/modeling/factory.py | 2 +- .../yolo/modeling/heads/yolox_head.py | 83 ++++++++++--------- 3 files changed, 49 insertions(+), 48 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py index 68af65861fd..26e2df77ef0 100644 --- a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py +++ b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py @@ -110,8 +110,8 @@ class YoloFPN(tf.keras.layers.Layer): def __init__(self, fpn_depth=4, - max_fpn_depth=None, - max_csp_stack=None, + max_fpn_depth=None, + max_csp_stack=None, use_spatial_attention=False, csp_stack=False, activation='leaky', @@ -215,7 +215,7 @@ def build(self, inputs): for level, depth in zip( reversed(range(self._min_level, self._max_level + 1)), self._depths): - + if level == self._min_level: self.resamples[str(level)] = nn_blocks.PathAggregationBlock( filters=depth // 2, @@ -464,8 +464,8 @@ def __init__(self, use_spatial_attention=False, csp_stack=False, fpn_depth=4, - max_fpn_depth=None, - max_csp_stack=None, + max_fpn_depth=None, + max_csp_stack=None, fpn_filter_scale=1, path_process_len=6, max_level_process_len=None, @@ -553,7 +553,7 @@ def __init__(self, } if self._use_fpn: inter_outs = YoloFPN( - fpn_depth=self._fpn_depth, + fpn_depth=self._fpn_depth, max_fpn_depth=self._max_fpn_depth, max_csp_stack=self._max_csp_stack, **self._base_config)(inputs) diff --git a/official/vision/beta/projects/yolo/modeling/factory.py b/official/vision/beta/projects/yolo/modeling/factory.py index 59ae209e6f5..d33eda7994c 100644 --- a/official/vision/beta/projects/yolo/modeling/factory.py +++ b/official/vision/beta/projects/yolo/modeling/factory.py @@ -130,7 +130,7 @@ def build_yolox(input_specs, model_config, l2_regularization): decoder=decoder, head=head, detection_generator=detection_generator_obj) - + # from tensorflow.keras import Input # from tensorflow.keras import Model # h, w = (640,640) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index 7cba1924324..6d5b0f6534c 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -38,10 +38,10 @@ def __init__( activation='silu', smart_bias=False, use_separable_conv=False, - width_scaling = 1.0, + width_scaling=1.0, strides=[8, 16, 32], in_channels=[256, 512, 1024], - prior_prob = 1e-2, + prior_prob=1e-2, **kwargs): """YoloX Prediction Head initialization function. @@ -64,7 +64,8 @@ def __init__( smart_bias: `bool`, whether to use smart bias. use_separable_conv: `bool` wether to use separable convs. width_scaling: `float`, factor by which the filters should be scaled - depthwise (bool): whether apply depthwise conv in conv branch. Defalut value: False. + depthwise (bool): whether apply depthwise conv in conv branch. + Default value: False. **kwargs: keyword arguments to be passed. """ @@ -86,7 +87,7 @@ def __init__( self._stems = dict() - self._bias= -tf.math.log((1 - self._prior_prob) / self._prior_prob) + self._bias = -tf.math.log((1 - self._prior_prob) / self._prior_prob) self._base_config = dict( activation=activation, @@ -96,13 +97,13 @@ def __init__( kernel_regularizer=kernel_regularizer, bias_regularizer=bias_regularizer) - + def build(self, input_shape): self._cls_convs = dict() self._reg_convs = dict() - + self._cls_preds = dict() self._reg_preds = dict() self._obj_preds = dict() @@ -110,12 +111,12 @@ def build(self, input_shape): self._cls_head = dict() self._obj_head = dict() self._reg_head = dict() - + for k in self._key_list: self._stems[k] = nn_blocks.ConvBN( filters=int(256 * self._width_scaling), - kernel_size=(1,1), - strides=(1,1), + kernel_size=(1, 1), + strides=(1, 1), use_bn=True, use_separable_conv=self._use_separable_conv, **self._base_config, @@ -123,22 +124,22 @@ def build(self, input_shape): self._cls_convs[k] = Sequential( [ - nn_blocks.ConvBN( - filters=int(256 * self._width_scaling), - kernel_size=(3,3), - strides=(1,1), - use_bn=True, - use_separable_conv=self._use_separable_conv, - **self._base_config, - ), - nn_blocks.ConvBN( - filters=int(256 * self._width_scaling), - kernel_size=(3,3), - strides=(1,1), - use_bn=True, - use_separable_conv=self._use_separable_conv, - **self._base_config, - ), + nn_blocks.ConvBN( + filters=int(256 * self._width_scaling), + kernel_size=(3, 3), + strides=(1, 1), + use_bn=True, + use_separable_conv=self._use_separable_conv, + **self._base_config, + ), + nn_blocks.ConvBN( + filters=int(256 * self._width_scaling), + kernel_size=(3, 3), + strides=(1, 1), + use_bn=True, + use_separable_conv=self._use_separable_conv, + **self._base_config, + ), ] ) @@ -146,40 +147,40 @@ def build(self, input_shape): [ nn_blocks.ConvBN( filters=int(256 * self._width_scaling), - kernel_size=(3,3), - strides=(1,1), + kernel_size=(3, 3), + strides=(1, 1), use_bn=True, use_separable_conv=self._use_separable_conv, **self._base_config, ), nn_blocks.ConvBN( filters=int(256 * self._width_scaling), - kernel_size=(3,3), - strides=(1,1), + kernel_size=(3, 3), + strides=(1, 1), use_bn=True, use_separable_conv=self._use_separable_conv, **self._base_config, ), ] ) - + self._cls_preds[k] = tf.keras.layers.Conv2D( filters=self._boxes_per_level * self._classes, - kernel_size=(1,1), - strides=(1,1), + kernel_size=(1, 1), + strides=(1, 1), padding='same', bias_initializer=tf.keras.initializers.constant(self._bias)) - + self._reg_preds[k] = tf.keras.layers.Conv2D( filters=4, - kernel_size=(1,1), - strides=(1,1), + kernel_size=(1, 1), + strides=(1, 1), padding='same') - + self._obj_preds[k] = tf.keras.layers.Conv2D( filters=self._boxes_per_level * 1, - kernel_size=(1,1), - strides=(1,1), + kernel_size=(1, 1), + strides=(1, 1), padding='same', bias_initializer=tf.keras.initializers.constant(self._bias)) @@ -200,12 +201,12 @@ def build(self, input_shape): self._reg_head[key].add(self._reg_preds[key]) def call(self, inputs, *args, **kwargs): - outputs=dict() - for k in self._key_list: + outputs = dict() + for k in self._key_list: cls_output = self._cls_head[k](inputs[k]) reg_output = self._reg_head[k](inputs[k]) obj_output = self._obj_head[k](inputs[k]) - output=tf.concat([reg_output,obj_output,cls_output], axis = -1) + output = tf.concat([reg_output, obj_output, cls_output], axis=-1) outputs[k] = output #Outputs are not flattened here. return outputs From 787474f40c35102d4345f95b892115fe98787695 Mon Sep 17 00:00:00 2001 From: Gautam Fotedar Date: Mon, 6 Dec 2021 23:39:27 -0500 Subject: [PATCH 43/96] Continued linting yolox_head branch --- .../yolo/modeling/heads/yolox_head.py | 2 +- .../yolo/modeling/layers/nn_blocks.py | 62 +++++++++++-------- .../vision/beta/projects/yolo/ops/anchor.py | 14 ++--- 3 files changed, 45 insertions(+), 33 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index 6d5b0f6534c..adac65f1dc8 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -202,7 +202,7 @@ def build(self, input_shape): def call(self, inputs, *args, **kwargs): outputs = dict() - for k in self._key_list: + for k in self._key_list: cls_output = self._cls_head[k](inputs[k]) reg_output = self._reg_head[k](inputs[k]) obj_output = self._obj_head[k](inputs[k]) diff --git a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py index 752f4179bfb..1973515e024 100644 --- a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py +++ b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py @@ -21,8 +21,8 @@ from official.vision.beta.ops import spatial_transform_ops import tensorflow.keras.backend as K from tensorflow.keras import Sequential -from tensorflow.keras.layers import Layer,Conv2D,LeakyReLU,\ - MaxPool2D,UpSampling2D,Activation,ReLU,BatchNormalization,concatenate +from tensorflow.keras.layers import Layer, Conv2D, LeakyReLU, \ + MaxPool2D, UpSampling2D, Activation, ReLU, BatchNormalization, concatenate @tf.keras.utils.register_keras_serializable(package='yolo') class Identity(tf.keras.layers.Layer): @@ -1731,10 +1731,10 @@ def call(self, x, training=None): class SiLU(Layer): def __init__(self, *args, **kwargs): - super(SiLU,self).__init__(*args, **kwargs) + super(SiLU, self).__init__(*args, **kwargs) - def call(self,x,**kwargs): - return x*K.sigmoid(x) + def call(self, x, **kwargs): + return x * K.sigmoid(x) def get_config(self): config = super(SiLU, self).get_config() @@ -1743,54 +1743,66 @@ def get_config(self): def compute_output_shape(self, input_shape): return input_shape -def get_activation(name="silu", ): +def get_activation(name="silu"): if name == "silu": module = SiLU() elif name == "relu": module = ReLU() elif name == "lrelu": - module = LeakyReLU(0.1 ) + module = LeakyReLU(0.1) else: raise AttributeError("Unsupported act type: {}".format(name)) return module class BaseConv(tf.keras.layers.Layer): - def __init__(self,filters, kernel_size, strides=1,padding='same',groups=1,use_bias=False, apply_batchnorm=True, activation="silu"): + def __init__(self, + filters, + kernel_size, + strides=1, + padding='same', + groups=1, + use_bias=False, + apply_batchnorm=True, + activation="silu"): super(BaseConv, self).__init__() - self.conv=Conv2D(filters,kernel_size=kernel_size,padding=padding,strides=strides,groups=groups, - use_bias=use_bias, - ) + self.conv = Conv2D(filters, + kernel_size=kernel_size, + padding=padding, + strides=strides, + groups=groups, + use_bias=use_bias, + ) self.apply_batchnorm=apply_batchnorm - self.bn=BatchNormalization(momentum=0.03,epsilon=1e-3) - self.activation=get_activation(activation) + self.bn = BatchNormalization(momentum=0.03, epsilon=1e-3) + self.activation = get_activation(activation) def call(self,inputs,**kwargs): - x=self.conv(inputs) + x = self.conv(inputs) if self.apply_batchnorm: - x=self.bn(x) - x=self.activation(x) + x = self.bn(x) + x = self.activation(x) return x class DWConv(tf.keras.layers.Layer): - def __init__ (self, - filters, - kernel_size, - strides=1, - padding='valid', # TODO: need to make sure this is correct + def __init__ (self, + filters, + kernel_size, + strides=1, + padding='valid',# TODO: need to make sure this is correct groups=1, use_biase=False, use_bn=True, activation='silu', **kwargs): super().__init__(**kwargs) - self._conv1 = ConvBN(filters, - kernel_size, + self._conv1 = ConvBN(filters, + kernel_size, strides=strides, use_bn=use_bn, activation=activation) - self._conv2 = ConvBN(filters, - kernel_size=1, + self._conv2 = ConvBN(filters, + kernel_size=1, strides=1, use_bn=use_bn, activation=activation) diff --git a/official/vision/beta/projects/yolo/ops/anchor.py b/official/vision/beta/projects/yolo/ops/anchor.py index 91528bab8d9..9fc20e8d248 100644 --- a/official/vision/beta/projects/yolo/ops/anchor.py +++ b/official/vision/beta/projects/yolo/ops/anchor.py @@ -38,13 +38,13 @@ def get_best_anchor(y_true, found via Kmeans. width: int for the image width. height: int for the image height. - iou_thresh: `float` the minimum iou threshold to use for selecting boxes for - each level. - best_match_only: `bool` if the box only has one match and it is less than - the iou threshold, when set to True, this match will be dropped as no - anchors can be linked to it. - use_tie_breaker: `bool` if there is many anchors for a given box, then - attempt to use all of them, if False, only the first matching box will + iou_thresh: `float` the minimum iou threshold to use for selecting boxes for + each level. + best_match_only: `bool` if the box only has one match and it is less than + the iou threshold, when set to True, this match will be dropped as no + anchors can be linked to it. + use_tie_breaker: `bool` if there is many anchors for a given box, then + attempt to use all of them, if False, only the first matching box will be used. Return: tf.Tensor: y_true with the anchor associated with each ground truth From ffd2a84b8ed62b22f9ebcf39ae3a65e7d3e62620 Mon Sep 17 00:00:00 2001 From: Gautam Fotedar Date: Tue, 7 Dec 2021 01:05:00 -0500 Subject: [PATCH 44/96] Additional linting commit --- .../yolo/modeling/decoders/yolo_decoder.py | 2 +- .../yolo/modeling/heads/yolox_head.py | 76 +++--- .../yolo/modeling/layers/nn_blocks.py | 37 ++- .../vision/beta/projects/yolo/ops/anchor.py | 241 +++++++++--------- .../vision/beta/projects/yolo/tasks/yolox.py | 10 +- 5 files changed, 185 insertions(+), 181 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py index 26e2df77ef0..b19b465a31c 100644 --- a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py +++ b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py @@ -215,7 +215,7 @@ def build(self, inputs): for level, depth in zip( reversed(range(self._min_level, self._max_level + 1)), self._depths): - + if level == self._min_level: self.resamples[str(level)] = nn_blocks.PathAggregationBlock( filters=depth // 2, diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index adac65f1dc8..fdd27eee775 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -64,7 +64,7 @@ def __init__( smart_bias: `bool`, whether to use smart bias. use_separable_conv: `bool` wether to use separable convs. width_scaling: `float`, factor by which the filters should be scaled - depthwise (bool): whether apply depthwise conv in conv branch. + depthwise (bool): whether apply depthwise conv in conv branch. Default value: False. **kwargs: keyword arguments to be passed. """ @@ -125,44 +125,44 @@ def build(self, input_shape): self._cls_convs[k] = Sequential( [ nn_blocks.ConvBN( - filters=int(256 * self._width_scaling), - kernel_size=(3, 3), - strides=(1, 1), - use_bn=True, - use_separable_conv=self._use_separable_conv, - **self._base_config, - ), + filters=int(256 * self._width_scaling), + kernel_size=(3, 3), + strides=(1, 1), + use_bn=True, + use_separable_conv=self._use_separable_conv, + **self._base_config, + ), nn_blocks.ConvBN( - filters=int(256 * self._width_scaling), - kernel_size=(3, 3), - strides=(1, 1), - use_bn=True, - use_separable_conv=self._use_separable_conv, - **self._base_config, - ), + filters=int(256 * self._width_scaling), + kernel_size=(3, 3), + strides=(1, 1), + use_bn=True, + use_separable_conv=self._use_separable_conv, + **self._base_config, + ), ] - ) + ) self._reg_convs[k] = Sequential( [ - nn_blocks.ConvBN( - filters=int(256 * self._width_scaling), - kernel_size=(3, 3), - strides=(1, 1), - use_bn=True, - use_separable_conv=self._use_separable_conv, - **self._base_config, - ), - nn_blocks.ConvBN( - filters=int(256 * self._width_scaling), - kernel_size=(3, 3), - strides=(1, 1), - use_bn=True, - use_separable_conv=self._use_separable_conv, - **self._base_config, - ), + nn_blocks.ConvBN( + filters=int(256 * self._width_scaling), + kernel_size=(3, 3), + strides=(1, 1), + use_bn=True, + use_separable_conv=self._use_separable_conv, + **self._base_config, + ), + nn_blocks.ConvBN( + filters=int(256 * self._width_scaling), + kernel_size=(3, 3), + strides=(1, 1), + use_bn=True, + use_separable_conv=self._use_separable_conv, + **self._base_config, + ), ] - ) + ) self._cls_preds[k] = tf.keras.layers.Conv2D( filters=self._boxes_per_level * self._classes, @@ -203,11 +203,11 @@ def build(self, input_shape): def call(self, inputs, *args, **kwargs): outputs = dict() for k in self._key_list: - cls_output = self._cls_head[k](inputs[k]) - reg_output = self._reg_head[k](inputs[k]) - obj_output = self._obj_head[k](inputs[k]) - output = tf.concat([reg_output, obj_output, cls_output], axis=-1) - outputs[k] = output + cls_output = self._cls_head[k](inputs[k]) + reg_output = self._reg_head[k](inputs[k]) + obj_output = self._obj_head[k](inputs[k]) + output = tf.concat([reg_output, obj_output, cls_output], axis=-1) + outputs[k] = output #Outputs are not flattened here. return outputs diff --git a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py index f14e1990a15..331e9f126a5 100644 --- a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py +++ b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py @@ -1729,27 +1729,26 @@ def call(self, x, training=None): axis=-1) class SiLU(Layer): - def __init__(self, *args, **kwargs): - super(SiLU, self).__init__(*args, **kwargs) + def __init__(self, *args, **kwargs): + super(SiLU, self).__init__(*args, **kwargs) - def call(self, x, **kwargs): - return x * K.sigmoid(x) + def call(self, x, **kwargs): + return x * K.sigmoid(x) - def get_config(self): - config = super(SiLU, self).get_config() - return config + def get_config(self): + config = super(SiLU, self).get_config() + return config - def compute_output_shape(self, input_shape): - return input_shape + def compute_output_shape(self, input_shape): + return input_shape def get_activation(name="silu"): - if name == "silu": - module = SiLU() - elif name == "relu": - module = ReLU() - elif name == "lrelu": - module = LeakyReLU(0.1) - else: - raise AttributeError("Unsupported act type: {}".format(name)) - return module - + if name == "silu": + module = SiLU() + elif name == "relu": + module = ReLU() + elif name == "lrelu": + module = LeakyReLU(0.1) + else: + raise AttributeError("Unsupported act type: {}".format(name)) + return module diff --git a/official/vision/beta/projects/yolo/ops/anchor.py b/official/vision/beta/projects/yolo/ops/anchor.py index 9fc20e8d248..0904c7b1069 100644 --- a/official/vision/beta/projects/yolo/ops/anchor.py +++ b/official/vision/beta/projects/yolo/ops/anchor.py @@ -28,10 +28,10 @@ def get_best_anchor(y_true, width=1, height=1, iou_thresh=0.25, - best_match_only=False, + best_match_only=False, use_tie_breaker=True): """Get the correct anchor that is assoiciated with each box using IOU. - + Args: y_true: tf.Tensor[] for the list of bounding boxes in the yolo format. anchors: list or tensor for the anchor boxes to be used in prediction @@ -45,7 +45,7 @@ def get_best_anchor(y_true, anchors can be linked to it. use_tie_breaker: `bool` if there is many anchors for a given box, then attempt to use all of them, if False, only the first matching box will - be used. + be used. Return: tf.Tensor: y_true with the anchor associated with each ground truth box known @@ -115,52 +115,52 @@ def get_best_anchor(y_true, class YoloAnchorLabeler: """Anchor labeler for the Yolo Models""" - def __init__(self, - anchors = None, - anchor_free_level_limits = None, - level_strides = None, - center_radius = None, - max_num_instances = 200, - match_threshold = 0.25, - best_matches_only = False, - use_tie_breaker = True, - darknet = False, - dtype = 'float32'): - """Initialization for anchor labler. - - Args: + def __init__(self, + anchors=None, + anchor_free_level_limits=None, + level_strides=None, + center_radius=None, + max_num_instances=200, + match_threshold=0.25, + best_matches_only=False, + use_tie_breaker=True, + darknet=False, + dtype='float32'): + """Initialization for anchor labler. + + Args: anchors: `Dict[List[Union[int, float]]]` values for each anchor box. - anchor_free_level_limits: `List` the box sizes that will be allowed at - each FPN level as is done in the FCOS and YOLOX paper for anchor free + anchor_free_level_limits: `List` the box sizes that will be allowed at + each FPN level as is done in the FCOS and YOLOX paper for anchor free box assignment. - level_strides: `Dict[int]` for how much the model scales down the + level_strides: `Dict[int]` for how much the model scales down the images at the each level. - center_radius: `Dict[float]` for radius around each box center to search + center_radius: `Dict[float]` for radius around each box center to search for extra centers in each level. max_num_instances: `int` for the number of boxes to compute loss on. - match_threshold: `float` indicating the threshold over which an anchor - will be considered for prediction, at zero, all the anchors will be used - and at 1.0 only the best will be used. for anchor thresholds larger than - 1.0 we stop using the IOU for anchor comparison and resort directly to - comparing the width and height, this is used for the scaled models. - best_matches_only: `boolean` indicating how boxes are selected for - optimization. - use_tie_breaker: `boolean` indicating whether to use the anchor threshold + match_threshold: `float` indicating the threshold over which an anchor + will be considered for prediction, at zero, all the anchors will be used + and at 1.0 only the best will be used. for anchor thresholds larger than + 1.0 we stop using the IOU for anchor comparison and resort directly to + comparing the width and height, this is used for the scaled models. + best_matches_only: `boolean` indicating how boxes are selected for + optimization. + use_tie_breaker: `boolean` indicating whether to use the anchor threshold value. - darknet: `boolean` indicating which data pipeline to use. Setting to True + darknet: `boolean` indicating which data pipeline to use. Setting to True swaps the pipeline to output images realtive to Yolov4 and older. - dtype: `str` indicating the output datatype of the datapipeline selecting + dtype: `str` indicating the output datatype of the datapipeline selecting from {"float32", "float16", "bfloat16"}. """ self.anchors = anchors self.masks = self._get_mask() self.use_tie_breaker = use_tie_breaker self.anchor_free_level_limits = self._get_level_limits( - anchor_free_level_limits) + anchor_free_level_limits) if darknet and self.anchor_free_level_limits is None: center_radius = None - + self.keys = self.anchors.keys() if self.anchor_free_level_limits is not None: maxim = 2000 @@ -168,7 +168,8 @@ def __init__(self, self.num_instances = {key: maxim for key in self.keys} elif not darknet: self.num_instances = { - key: (6 - i) * max_num_instances for i, key in enumerate(self.keys)} + key: (6 - i) * max_num_instances for i, key in enumerate(self.keys) + } else: self.num_instances = {key: max_num_instances for key in self.keys} @@ -190,7 +191,7 @@ def _get_mask(self): masks[str(i)] = list(range(start, per_scale + start)) start += per_scale return masks - + def _get_level_limits(self, level_limits): """For each level receptive feild range for anchor free box placement.""" if level_limits is not None: @@ -200,13 +201,14 @@ def _get_level_limits(self, level_limits): k = 0 for i, key in enumerate(self.anchors.keys()): level_limits_dict[key] = [] - - base = k + + base = k for j, lst in enumerate(self.anchors[key]): # level_limits_dict[key].append(level_limits[k:k + 2]) if self.use_tie_breaker: base = k - level_limits_dict[key].append([level_limits[base], level_limits[k + 1]]) + level_limits_dict[key].append([level_limits[base], + level_limits[k + 1]]) k += 1 level_limits_dict[key] = tf.convert_to_tensor(level_limits_dict[key]) print(level_limits_dict) @@ -218,87 +220,87 @@ def _tie_breaking_search(self, anchors, mask, boxes, classes): """After search, link each anchor ind to the correct map in ground truth.""" mask = tf.cast(tf.reshape(mask, [1, 1, 1, -1]), anchors.dtype) anchors = tf.expand_dims(anchors, axis=-1) - viable = tf.where(tf.squeeze(anchors == mask, axis = 0)) + viable = tf.where(tf.squeeze(anchors == mask, axis=0)) - gather_id, _, anchor_id = tf.split(viable, 3, axis = -1) + gather_id, _, anchor_id = tf.split(viable, 3, axis=-1) boxes = tf.gather_nd(boxes, gather_id) classes = tf.gather_nd(classes, gather_id) - - classes = tf.expand_dims(classes, axis = -1) + + classes = tf.expand_dims(classes, axis=-1) classes = tf.cast(classes, boxes.dtype) anchor_id = tf.cast(anchor_id, boxes.dtype) return boxes, classes, anchor_id - def _get_anchor_id(self, key, boxes, classes, width, height, stride, - iou_index = None): + def _get_anchor_id(self, key, boxes, classes, width, height, stride, + iou_index=None): """Find the object anchor assignments in an anchor based paradigm. """ - + # find the best anchor anchors = self.anchors[key] num_anchors = len(anchors) if self.best_matches_only: # get the best anchor for each box iou_index, _ = get_best_anchor(boxes, anchors, stride, - width=width, height=height, - best_match_only=True, - iou_thresh=self.match_threshold) + width=width, height=height, + best_match_only=True, + iou_thresh=self.match_threshold) mask = range(num_anchors) - else: + else: # search is done across FPN levels, get the mask of anchor indexes - # corralated to this level. + # corralated to this level. mask = self.masks[key] # search for the correct box to use - (boxes, classes, anchors) = self._tie_breaking_search(iou_index, mask, - boxes, classes) + (boxes, classes, anchors) = self._tie_breaking_search(iou_index, mask, + boxes, classes) return boxes, classes, anchors, num_anchors def _get_centers(self, boxes, classes, anchors, width, height, scale_xy): """Find the object center assignments in an anchor based paradigm. """ offset = tf.cast(0.5 * (scale_xy - 1), boxes.dtype) - grid_xy, _ = tf.split(boxes, 2, axis = -1) + grid_xy, _ = tf.split(boxes, 2, axis=-1) wh_scale = tf.cast(tf.convert_to_tensor([width, height]), boxes.dtype) grid_xy = grid_xy * wh_scale centers = tf.math.floor(grid_xy) - if offset != 0.0: + if offset != 0.0: clamp = lambda x, ma: tf.maximum( tf.minimum(x, tf.cast(ma, x.dtype)), tf.zeros_like(x)) grid_xy_index = grid_xy - centers positive_shift = ((grid_xy_index < offset) & (grid_xy > 1.)) negative_shift = ( - (grid_xy_index > (1 - offset)) & (grid_xy < (wh_scale - 1.))) + (grid_xy_index > (1 - offset)) & (grid_xy < (wh_scale - 1.))) - zero , _ = tf.split(tf.ones_like(positive_shift), 2, axis = -1) + zero, _ = tf.split(tf.ones_like(positive_shift), 2, axis=-1) shift_mask = tf.concat( - [zero, positive_shift, negative_shift], axis = -1) - offset = tf.cast([[0, 0], [1, 0], - [0, 1], [-1, 0], + [zero, positive_shift, negative_shift], axis=-1) + offset = tf.cast([[0, 0], [1, 0], + [0, 1], [-1, 0], [0, -1]], offset.dtype) * offset num_shifts = tf.shape(shift_mask) num_shifts = num_shifts[-1] - boxes = tf.tile(tf.expand_dims(boxes, axis = -2), [1, num_shifts, 1]) - classes = tf.tile(tf.expand_dims(classes, axis = -2), [1, num_shifts, 1]) - anchors = tf.tile(tf.expand_dims(anchors, axis = -2), [1, num_shifts, 1]) + boxes = tf.tile(tf.expand_dims(boxes, axis=-2), [1, num_shifts, 1]) + classes = tf.tile(tf.expand_dims(classes, axis=-2), [1, num_shifts, 1]) + anchors = tf.tile(tf.expand_dims(anchors, axis=-2), [1, num_shifts, 1]) shift_mask = tf.cast(shift_mask, boxes.dtype) - shift_ind = shift_mask * tf.range(0, num_shifts, dtype = boxes.dtype) + shift_ind = shift_mask * tf.range(0, num_shifts, dtype=boxes.dtype) shift_ind = shift_ind - (1 - shift_mask) - shift_ind = tf.expand_dims(shift_ind, axis = -1) + shift_ind = tf.expand_dims(shift_ind, axis=-1) boxes_and_centers = tf.concat( - [boxes, classes, anchors, shift_ind], axis = -1) + [boxes, classes, anchors, shift_ind], axis=-1) boxes_and_centers = tf.reshape(boxes_and_centers, [-1, 7]) - _, center_ids = tf.split(boxes_and_centers, [6, 1], axis = -1) + _, center_ids = tf.split(boxes_and_centers, [6, 1], axis=-1) #center_ids = tf.squeeze(center_ids, axis = -1) select = tf.where(center_ids >= 0) - select, _ = tf.split(select, 2, axis = -1) + select, _ = tf.split(select, 2, axis=-1) boxes_and_centers = tf.gather_nd(boxes_and_centers, select) @@ -307,14 +309,14 @@ def _get_centers(self, boxes, classes, anchors, width, height, scale_xy): center_ids = tf.cast(center_ids, tf.int32) shifts = tf.gather_nd(offset, center_ids) - boxes, classes, anchors, _ = tf.split(boxes_and_centers, - [4, 1, 1, 1], axis = -1) - grid_xy, _ = tf.split(boxes, 2, axis = -1) + boxes, classes, anchors, _ = tf.split(boxes_and_centers, + [4, 1, 1, 1], axis=-1) + grid_xy, _ = tf.split(boxes, 2, axis=-1) centers = tf.math.floor(grid_xy * wh_scale - shifts) centers = clamp(centers, wh_scale - 1) - - x, y = tf.split(centers, 2, axis = -1) - centers = tf.cast(tf.concat([y, x, anchors], axis = -1), tf.int32) + + x, y = tf.split(centers, 2, axis=-1) + centers = tf.cast(tf.concat([y, x, anchors], axis=-1), tf.int32) return boxes, classes, centers def _get_anchor_free(self, @@ -343,7 +345,7 @@ def _get_anchor_free(self, boxes = tf.reshape(boxes, [1, 1, -1, 4]) tlbr_boxes = tf.reshape(tlbr_boxes, [1, 1, -1, 4]) if self.use_tie_breaker: - area = tf.reduce_prod(boxes[..., 2:], axis = -1) + area = tf.reduce_prod(boxes[..., 2:], axis=-1) # check if the box is in the receptive feild of the this fpn level b_t = y_centers - tlbr_boxes[..., 0] @@ -357,7 +359,7 @@ def _get_anchor_free(self, level_limits = tf.cast(level_limits, max_reg_targets_per_im.dtype) num_anchors = tf.shape(level_limits)[0] - max_reg_targets_per_im = tf.expand_dims(max_reg_targets_per_im, axis = -1) + max_reg_targets_per_im = tf.expand_dims(max_reg_targets_per_im, axis=-1) gt_min = max_reg_targets_per_im >= level_limits[..., 0] gt_max = max_reg_targets_per_im <= level_limits[..., 1] @@ -380,11 +382,12 @@ def _get_anchor_free(self, is_in_boxes_and_center = tf.logical_and(is_in_index, is_in_boxes_and_center) if level_limits is not None: - is_in_boxes_and_center = tf.expand_dims(is_in_boxes_and_center, axis = -1) - is_in_boxes_and_center = tf.logical_and(is_in_level, is_in_boxes_and_center) - + is_in_boxes_and_center = tf.expand_dims(is_in_boxes_and_center, axis=-1) + is_in_boxes_and_center = tf.logical_and(is_in_level, + is_in_boxes_and_center) + # if self.use_tie_breaker: - # boxes_all = tf.cast(is_in_boxes_and_center, area.dtype) + # boxes_all = tf.cast(is_in_boxes_and_center, area.dtype) # boxes_all = ((boxes_all * area) + ((1 - boxes_all) * INF)) # boxes_min = tf.reduce_min(boxes_all, axis = -1, keepdims = True) # boxes_min = tf.where(boxes_min == INF, -1.0, boxes_min) @@ -407,34 +410,36 @@ def _get_anchor_free(self, indexes = tf.concat([y, x, a], axis=-1) return indexes, samples, num_anchors - def build_label_per_path(self, - key, - boxes, - classes, - width, - height, - iou_index = None): + def build_label_per_path(self, + key, + boxes, + classes, + width, + height, + iou_index=None): """Builds the labels for one path.""" stride = self.level_strides[key] scale_xy = self.center_radius[key] if self.center_radius is not None else 1 width = tf.cast(width//stride, boxes.dtype) height = tf.cast(height//stride, boxes.dtype) - + if self.anchor_free_level_limits is None: - (boxes, classes, - anchors, num_anchors) = self._get_anchor_id(key, boxes, classes, - width, height, stride, - iou_index = iou_index) - boxes, classes, centers = self._get_centers(boxes, classes, anchors, + (boxes, classes, + anchors, num_anchors) = self._get_anchor_id(key, boxes, classes, + width, height, stride, + iou_index=iou_index) + boxes, classes, centers = self._get_centers(boxes, classes, anchors, width, height, scale_xy) ind_mask = tf.ones_like(classes) - updates = tf.concat([boxes, ind_mask, classes], axis = -1) + updates = tf.concat([boxes, ind_mask, classes], axis=-1) else: - (centers, updates, num_anchors) = self._get_anchor_free(key, boxes, classes, height, - width, stride, scale_xy) - boxes, ind_mask, classes = tf.split(updates, [4, 1, 1], axis = -1) - + (centers, updates, num_anchors) = self._get_anchor_free(key, boxes, + classes, height, + width, stride, + scale_xy) + boxes, ind_mask, classes = tf.split(updates, [4, 1, 1], axis=-1) + width = tf.cast(width, tf.int32) height = tf.cast(height, tf.int32) full = tf.zeros([height, width, num_anchors, 1], dtype=classes.dtype) @@ -442,30 +447,30 @@ def build_label_per_path(self, num_instances = int(self.num_instances[key]) centers = preprocessing_ops.pad_max_instances( - centers, num_instances, pad_value=0, pad_axis=0) + centers, num_instances, pad_value=0, pad_axis=0) updates = preprocessing_ops.pad_max_instances( - updates, num_instances, pad_value=0, pad_axis=0) + updates, num_instances, pad_value=0, pad_axis=0) updates = tf.cast(updates, self.dtype) full = tf.cast(full, self.dtype) return centers, updates, full def __call__(self, boxes, classes, width, height): - """Builds the labels for a single image, not functional in batch mode. - - Args: - boxes: `Tensor` of shape [None, 4] indicating the object locations in - an image. + """Builds the labels for a single image, not functional in batch mode. + + Args: + boxes: `Tensor` of shape [None, 4] indicating the object locations in + an image. classes: `Tensor` of shape [None] indicating the each objects classes. - width: `int` for the images width. + width: `int` for the images width. height: `int` for the images height. - num_instances: `int` for the maximum number of expanded boxes to allow. - Returns: - centers: `Tensor` of shape [None, 3] of indexes in the final grid where - boxes are located. + num_instances: `int` for the maximum number of expanded boxes to allow. + Returns: + centers: `Tensor` of shape [None, 3] of indexes in the final grid where + boxes are located. updates: `Tensor` of shape [None, 8] the value to place in the final grid. - full: `Tensor` of [width/stride, height/stride, num_anchors, 1] holding - a mask of where boxes are locates for confidence losses. + full: `Tensor` of [width/stride, height/stride, num_anchors, 2] holding + a mask of where boxes are locates for confidence losses. """ indexes = {} updates = {} @@ -473,7 +478,7 @@ def __call__(self, boxes, classes, width, height): iou_index = None boxes = box_ops.yxyx_to_xcycwh(boxes) - if not self.best_matches_only and self.anchor_free_level_limits is None: + if not self.best_matches_only and self.anchor_free_level_limits is None: # stitch and search boxes across fpn levels anchorsvec = [] for stitch in self.anchors: @@ -482,12 +487,12 @@ def __call__(self, boxes, classes, width, height): stride = tf.cast([width, height], boxes.dtype) # get the best anchor for each box iou_index, _ = get_best_anchor(boxes, anchorsvec, stride, - width=1.0, height=1.0, - best_match_only=False, - use_tie_breaker=self.use_tie_breaker, - iou_thresh=self.match_threshold) + width=1.0, height=1.0, + best_match_only=False, + use_tie_breaker=self.use_tie_breaker, + iou_thresh=self.match_threshold) for key in self.keys: indexes[key], updates[key], true_grids[key] = self.build_label_per_path( - key, boxes, classes, width, height, iou_index = iou_index) - return indexes, updates, true_grids + key, boxes, classes, width, height, iou_index=iou_index) + return indexes, updates, true_grids diff --git a/official/vision/beta/projects/yolo/tasks/yolox.py b/official/vision/beta/projects/yolo/tasks/yolox.py index e718d8cff38..e1e89ddbcfb 100644 --- a/official/vision/beta/projects/yolo/tasks/yolox.py +++ b/official/vision/beta/projects/yolo/tasks/yolox.py @@ -258,13 +258,13 @@ def train_step(self, inputs, model, optimizer, metrics=None): def _reorg_boxes(self, boxes, info, num_detections): """Scale and Clean boxes prior to Evaluation.""" mask = tf.sequence_mask(num_detections, maxlen=tf.shape(boxes)[1]) - mask = tf.cast(tf.expand_dims(mask, axis = -1), boxes.dtype) + mask = tf.cast(tf.expand_dims(mask, axis=-1), boxes.dtype) # Denormalize the boxes by the shape of the image - inshape = tf.expand_dims(info[:, 1, :], axis = 1) - ogshape = tf.expand_dims(info[:, 0, :], axis = 1) - scale = tf.expand_dims(info[:, 2, :], axis = 1) - offset = tf.expand_dims(info[:, 3, :], axis = 1) + inshape = tf.expand_dims(info[:, 1, :], axis=1) + ogshape = tf.expand_dims(info[:, 0, :], axis=1) + scale = tf.expand_dims(info[:, 2, :], axis=1) + offset = tf.expand_dims(info[:, 3, :], axis=1) boxes = box_ops.denormalize_boxes(boxes, inshape) boxes = box_ops.clip_boxes(boxes, inshape) From 3ba2d90f6aeed6d88bf99339a6d922eed917d9d5 Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Fri, 10 Dec 2021 12:51:03 -0500 Subject: [PATCH 45/96] Update for any number of boxes --- .../vision/beta/projects/yolo/modeling/heads/yolox_head.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index fdd27eee775..9f0a686096b 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -172,13 +172,13 @@ def build(self, input_shape): bias_initializer=tf.keras.initializers.constant(self._bias)) self._reg_preds[k] = tf.keras.layers.Conv2D( - filters=4, + filters=4 * self._boxes_per_level, kernel_size=(1, 1), strides=(1, 1), padding='same') self._obj_preds[k] = tf.keras.layers.Conv2D( - filters=self._boxes_per_level * 1, + filters=1 * self._boxes_per_level, kernel_size=(1, 1), strides=(1, 1), padding='same', From b37b18162afbc396665500f6e1dfbc305a4e7b5d Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Fri, 10 Dec 2021 12:51:57 -0500 Subject: [PATCH 46/96] Update yolox_head_test.py --- .../beta/projects/yolo/modeling/heads/yolox_head_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py index 890ff622078..afb5c7793c5 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py @@ -33,7 +33,7 @@ def test_network_creation(self): '5': [1, 13, 13, 1024] } classes = 100 - bps = 1 + bps = 3 head = heads.YOLOXHead(3, 5, classes=classes, boxes_per_level=bps) inputs = {} @@ -57,7 +57,7 @@ def test_serialize_deserialize(self): '5': [1, 13, 13, 1024] } classes = 100 - bps = 1 + bps = 3 head = heads.YOLOXHead(3, 5, classes=classes, boxes_per_level=bps) inputs = {} From 172aa335571cbca469370345f9c9dd159d397caf Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Fri, 10 Dec 2021 13:36:25 -0500 Subject: [PATCH 47/96] Ordering in format --- .../beta/projects/yolo/modeling/heads/yolox_head.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index 9f0a686096b..ee394d9d3f7 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -16,6 +16,7 @@ """Yolox heads.""" from loguru import logger import tensorflow as tf + from tensorflow.keras.models import Sequential from official.vision.beta.projects.yolo.modeling.layers import nn_blocks from official.vision.beta.projects.yolo.ops import box_ops @@ -202,11 +203,19 @@ def build(self, input_shape): def call(self, inputs, *args, **kwargs): outputs = dict() + for k in self._key_list: + ordered_preds = [] cls_output = self._cls_head[k](inputs[k]) reg_output = self._reg_head[k](inputs[k]) obj_output = self._obj_head[k](inputs[k]) - output = tf.concat([reg_output, obj_output, cls_output], axis=-1) + + for b in range(self._boxes_per_level): + ordered_preds.append(reg_output[:,:,:,4 * b: 4 * (b + 1)]) + ordered_preds.append(obj_output[:,:,:,b: b + 1]) + ordered_preds.append(cls_output[:,:,:,self._classes * b: self._classes * (b + 1)]) + + output = tf.concat(ordered_preds, axis=-1) outputs[k] = output #Outputs are not flattened here. return outputs From 2ba8122c035a827a17d028634e605d46d213eff4 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Fri, 10 Dec 2021 17:34:31 -0500 Subject: [PATCH 48/96] Update args documentation --- .../beta/projects/yolo/modeling/heads/yolox_head.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index ee394d9d3f7..ac58d38e64e 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -40,8 +40,6 @@ def __init__( smart_bias=False, use_separable_conv=False, width_scaling=1.0, - strides=[8, 16, 32], - in_channels=[256, 512, 1024], prior_prob=1e-2, **kwargs): @@ -61,12 +59,12 @@ def __init__( kernel_initializer: kernel_initializer for convolutional layers. kernel_regularizer: tf.keras.regularizers.Regularizer object for Conv2D. bias_regularizer: tf.keras.regularizers.Regularizer object for Conv2d. - activation: `str`, the activation function to use typically leaky or mish. + activation: `str`, the activation function to use. Default value: "silu". smart_bias: `bool`, whether to use smart bias. use_separable_conv: `bool` wether to use separable convs. - width_scaling: `float`, factor by which the filters should be scaled - depthwise (bool): whether apply depthwise conv in conv branch. - Default value: False. + width_scaling: `float`, factor by which the filters should be scaled. + prior_prob: 'float', prior probability of custom value between 0.0 and 1. + Defaults to 1e-2. **kwargs: keyword arguments to be passed. """ From 47e6fd56e13499d85b668d0a4af20865827cce2e Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Wed, 12 Jan 2022 00:37:03 -0500 Subject: [PATCH 49/96] Update yolox.yaml --- .../yolo/configs/experiments/yolox/yolox.yaml | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 103358a9486..69ff3713f3d 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -1,9 +1,9 @@ # --experiment_type=yolox # mAP runtime: - distribution_strategy: 'mirrored' + distribution_strategy: 'tpu' mixed_precision_dtype: 'float32' - num_gpus: 1 + tpu_enable_xla_dynamic_padder: false task: model: input_size: [640, 640, 3] @@ -53,12 +53,12 @@ task: level_limits: [64, 128] anchors_per_scale: 1 train_data: - input_path: '/content/train*' + input_path: 'gs://cam2-datasets/coco/train*' shuffle_buffer_size: 10000 parser: mosaic: mosaic_frequency: 1.0 - mixup_frequency: 0.0 + mixup_frequency: 0.2 mosaic_crop_mode: 'scale' mosaic_center: 0.25 aug_scale_min: 0.1 @@ -69,4 +69,23 @@ task: aug_rand_translate: 0.1 area_thresh: 0.1 validation_data: - input_path: '/content/val*' + input_path: 'gs://cam2-datasets/coco/val*' + trainer: + train_steps: 831600 # epoch 300 to 450 + optimizer_config: + learning_rate: + cosine: + decay_steps: 831600 # epoch 300 to 450 + optimizer: + type: sgd_torch + sgd_torch: + momentum: 0.9 + momentum_start: 0.9 + nesterov: true + warmup_steps: 1000 + weight_decay: 0.0005 + name: SGD + warmup: + type: 'linear' + linear: + warmup_steps: 1000 # learning rate rises from 0 to 0.0013 over 1000 steps # 5 epochs From 09d19a894b28b28a4e0d868f7df4dd5f48f269bb Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Wed, 12 Jan 2022 16:34:46 -0500 Subject: [PATCH 50/96] training config --- official/vision/beta/projects/yolo/configs/yolox.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/yolox.py b/official/vision/beta/projects/yolo/configs/yolox.py index 9809dd49624..4c53031a0a6 100644 --- a/official/vision/beta/projects/yolo/configs/yolox.py +++ b/official/vision/beta/projects/yolo/configs/yolox.py @@ -24,8 +24,7 @@ from official.modeling import hyperparams from official.vision.beta.configs import common from official.vision.beta.projects.yolo import optimization -from official.vision.beta.projects.yolo.configs import backbones -from official.vision.beta.projects.yolo.configs import decoders +from official.vision.beta.projects.yolo.configs import backbones, decoders # pytype: disable=annotation-type-mismatch @@ -274,7 +273,7 @@ def yolox() -> cfg.ExperimentConfig: @exp_factory.register_config_factory('yolox_regular') def yolox_regular() -> cfg.ExperimentConfig: """COCO object detection with YOLOvx.""" - train_batch_size = 64 + train_batch_size = 128 eval_batch_size = 8 train_epochs = 300 steps_per_epoch = COCO_TRAIN_EXAMPLES // train_batch_size @@ -379,7 +378,7 @@ def yolox_regular() -> cfg.ExperimentConfig: 'warmup': { 'type': 'linear', 'linear': { - 'warmup_steps': 1000, + 'warmup_steps': 5 * steps_per_epoch, 'warmup_learning_rate': 0 } } From 2031f5edf4f8a34bf2c3e8cf3e8b4f58ec0068f8 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Wed, 12 Jan 2022 17:22:43 -0500 Subject: [PATCH 51/96] training config --- official/vision/beta/projects/yolo/__init__.py | 0 .../beta/projects/yolo/configs/experiments/yolox/yolox.yaml | 5 +++-- official/vision/beta/projects/yolo/configs/yolox.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 official/vision/beta/projects/yolo/__init__.py diff --git a/official/vision/beta/projects/yolo/__init__.py b/official/vision/beta/projects/yolo/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 69ff3713f3d..42cea1bd036 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -75,6 +75,7 @@ task: optimizer_config: learning_rate: cosine: + initial_learning_rate: 0.1 decay_steps: 831600 # epoch 300 to 450 optimizer: type: sgd_torch @@ -82,10 +83,10 @@ task: momentum: 0.9 momentum_start: 0.9 nesterov: true - warmup_steps: 1000 + warmup_steps: 4620 weight_decay: 0.0005 name: SGD warmup: type: 'linear' linear: - warmup_steps: 1000 # learning rate rises from 0 to 0.0013 over 1000 steps # 5 epochs + warmup_steps: 4620 # 5 epochs diff --git a/official/vision/beta/projects/yolo/configs/yolox.py b/official/vision/beta/projects/yolo/configs/yolox.py index 4c53031a0a6..834e8f684d3 100644 --- a/official/vision/beta/projects/yolo/configs/yolox.py +++ b/official/vision/beta/projects/yolo/configs/yolox.py @@ -273,7 +273,7 @@ def yolox() -> cfg.ExperimentConfig: @exp_factory.register_config_factory('yolox_regular') def yolox_regular() -> cfg.ExperimentConfig: """COCO object detection with YOLOvx.""" - train_batch_size = 128 + train_batch_size = 64 eval_batch_size = 8 train_epochs = 300 steps_per_epoch = COCO_TRAIN_EXAMPLES // train_batch_size From 408e5f79e4d97b6f6668c8b7686533ecaeebe271 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Wed, 12 Jan 2022 18:08:23 -0500 Subject: [PATCH 52/96] remove loguru --- .../projects/yolo/common/registry_imports.py | 13 ++++--------- .../beta/projects/yolo/configs/backbones.py | 1 + .../beta/projects/yolo/configs/decoders.py | 1 + .../vision/beta/projects/yolo/configs/head.py | 1 + .../vision/beta/projects/yolo/configs/yolo.py | 6 ++---- .../beta/projects/yolo/configs/yolox.py | 1 - .../projects/yolo/dataloaders/__init__.py | 2 -- .../yolo/dataloaders/classification_input.py | 1 + .../projects/yolo/dataloaders/yolo_input.py | 6 ++---- .../yolo/dataloaders/yolo_input_test.py | 17 ++++++++++------- .../beta/projects/yolo/losses/__init__.py | 1 - .../beta/projects/yolo/losses/yolo_loss.py | 5 ++--- .../projects/yolo/losses/yolo_loss_test.py | 2 +- .../yolo/modeling/backbones/darknet.py | 1 + .../yolo/modeling/backbones/darknet_test.py | 5 ++--- .../yolo/modeling/decoders/__init__.py | 1 - .../yolo/modeling/decoders/yolo_decoder.py | 2 +- .../modeling/decoders/yolo_decoder_test.py | 8 ++++---- .../beta/projects/yolo/modeling/factory.py | 8 ++++---- .../projects/yolo/modeling/heads/__init__.py | 1 - .../projects/yolo/modeling/heads/yolo_head.py | 1 + .../yolo/modeling/heads/yolo_head_test.py | 5 +++-- .../yolo/modeling/heads/yolox_head.py | 5 +++-- .../yolo/modeling/heads/yolox_head_test.py | 5 +++-- .../modeling/layers/detection_generator.py | 3 +-- .../layers/detection_generator_test.py | 5 +++-- .../yolo/modeling/layers/nn_blocks.py | 10 ++++++---- .../yolo/modeling/layers/nn_blocks_test.py | 4 ++-- .../beta/projects/yolo/modeling/yolo_model.py | 2 ++ .../vision/beta/projects/yolo/ops/__init__.py | 2 -- .../vision/beta/projects/yolo/ops/anchor.py | 4 +--- .../vision/beta/projects/yolo/ops/box_ops.py | 2 ++ .../beta/projects/yolo/ops/box_ops_test.py | 2 +- .../beta/projects/yolo/ops/loss_utils.py | 3 +-- .../vision/beta/projects/yolo/ops/mosaic.py | 4 ++-- .../yolo/ops/preprocessing_ops_test.py | 2 +- .../projects/yolo/optimization/__init__.py | 7 ++++--- .../yolo/optimization/configs/__init__.py | 1 - .../configs/optimization_config.py | 6 ++++-- .../yolo/optimization/optimizer_factory.py | 3 +-- .../projects/yolo/optimization/sgd_torch.py | 2 +- .../yolo/tasks/image_classification.py | 9 +++++---- .../vision/beta/projects/yolo/tasks/yolo.py | 19 ++++++++----------- .../vision/beta/projects/yolo/tasks/yolox.py | 19 ++++++++----------- official/vision/beta/projects/yolo/train.py | 6 +++--- 45 files changed, 103 insertions(+), 111 deletions(-) diff --git a/official/vision/beta/projects/yolo/common/registry_imports.py b/official/vision/beta/projects/yolo/common/registry_imports.py index 2879577c236..8bbf153d7eb 100644 --- a/official/vision/beta/projects/yolo/common/registry_imports.py +++ b/official/vision/beta/projects/yolo/common/registry_imports.py @@ -17,23 +17,18 @@ # pylint: disable=unused-import # pylint: disable=g-bad-import-order from official.common import registry_imports - # import configs from official.vision.beta.projects.yolo.configs import darknet_classification from official.vision.beta.projects.yolo.configs import yolo as yolo_config from official.vision.beta.projects.yolo.configs import yolox as yolox_config - # import modeling components from official.vision.beta.projects.yolo.modeling.backbones import darknet from official.vision.beta.projects.yolo.modeling.decoders import yolo_decoder - +# import optimization packages +from official.vision.beta.projects.yolo.optimization import optimizer_factory +from official.vision.beta.projects.yolo.optimization.configs import ( + optimization_config, optimizer_config) # import tasks from official.vision.beta.projects.yolo.tasks import image_classification from official.vision.beta.projects.yolo.tasks import yolo as yolo_task from official.vision.beta.projects.yolo.tasks import yolox as yolox_task - - -# import optimization packages -from official.vision.beta.projects.yolo.optimization import optimizer_factory -from official.vision.beta.projects.yolo.optimization.configs import optimizer_config -from official.vision.beta.projects.yolo.optimization.configs import optimization_config diff --git a/official/vision/beta/projects/yolo/configs/backbones.py b/official/vision/beta/projects/yolo/configs/backbones.py index 071af5bdef7..476e8e6ee4a 100644 --- a/official/vision/beta/projects/yolo/configs/backbones.py +++ b/official/vision/beta/projects/yolo/configs/backbones.py @@ -14,6 +14,7 @@ """Backbones configurations.""" import dataclasses + from official.modeling import hyperparams from official.vision.beta.configs import backbones diff --git a/official/vision/beta/projects/yolo/configs/decoders.py b/official/vision/beta/projects/yolo/configs/decoders.py index 7a4f4a6d997..48a8c542b34 100755 --- a/official/vision/beta/projects/yolo/configs/decoders.py +++ b/official/vision/beta/projects/yolo/configs/decoders.py @@ -15,6 +15,7 @@ """Decoders configurations.""" import dataclasses from typing import Optional + from official.modeling import hyperparams from official.vision.beta.configs import decoders diff --git a/official/vision/beta/projects/yolo/configs/head.py b/official/vision/beta/projects/yolo/configs/head.py index 26a0678f8f1..0014b3e9a09 100644 --- a/official/vision/beta/projects/yolo/configs/head.py +++ b/official/vision/beta/projects/yolo/configs/head.py @@ -15,6 +15,7 @@ """Decoders configurations.""" import dataclasses from typing import Optional + from official.modeling import hyperparams from official.vision.beta.configs import head diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index b9df1afe8f3..6db523a6024 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -24,9 +24,7 @@ from official.modeling import hyperparams from official.vision.beta.configs import common from official.vision.beta.projects.yolo import optimization -from official.vision.beta.projects.yolo.configs import backbones -from official.vision.beta.projects.yolo.configs import decoders - +from official.vision.beta.projects.yolo.configs import backbones, decoders # pytype: disable=annotation-type-mismatch @@ -728,4 +726,4 @@ def yolo_tiny() -> cfg.ExperimentConfig: 'task.validation_data.is_training != None' ]) - return config \ No newline at end of file + return config diff --git a/official/vision/beta/projects/yolo/configs/yolox.py b/official/vision/beta/projects/yolo/configs/yolox.py index 834e8f684d3..b5f39b108e1 100644 --- a/official/vision/beta/projects/yolo/configs/yolox.py +++ b/official/vision/beta/projects/yolo/configs/yolox.py @@ -26,7 +26,6 @@ from official.vision.beta.projects.yolo import optimization from official.vision.beta.projects.yolo.configs import backbones, decoders - # pytype: disable=annotation-type-mismatch MIN_LEVEL = 1 diff --git a/official/vision/beta/projects/yolo/dataloaders/__init__.py b/official/vision/beta/projects/yolo/dataloaders/__init__.py index a25710c222e..e04127d3fc8 100644 --- a/official/vision/beta/projects/yolo/dataloaders/__init__.py +++ b/official/vision/beta/projects/yolo/dataloaders/__init__.py @@ -11,5 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - diff --git a/official/vision/beta/projects/yolo/dataloaders/classification_input.py b/official/vision/beta/projects/yolo/dataloaders/classification_input.py index 57d7ec2382a..07498eb6476 100755 --- a/official/vision/beta/projects/yolo/dataloaders/classification_input.py +++ b/official/vision/beta/projects/yolo/dataloaders/classification_input.py @@ -14,6 +14,7 @@ """Classification decoder and parser.""" import tensorflow as tf + from official.vision.beta.dataloaders import classification_input from official.vision.beta.ops import preprocess_ops diff --git a/official/vision/beta/projects/yolo/dataloaders/yolo_input.py b/official/vision/beta/projects/yolo/dataloaders/yolo_input.py index 86046dffdc8..bda46d15382 100755 --- a/official/vision/beta/projects/yolo/dataloaders/yolo_input.py +++ b/official/vision/beta/projects/yolo/dataloaders/yolo_input.py @@ -15,12 +15,10 @@ """Detection Data parser and processing for YOLO.""" import tensorflow as tf -from official.vision.beta.dataloaders import parser -from official.vision.beta.dataloaders import utils +from official.vision.beta.dataloaders import parser, utils from official.vision.beta.ops import box_ops as bbox_ops from official.vision.beta.ops import preprocess_ops -from official.vision.beta.projects.yolo.ops import anchor -from official.vision.beta.projects.yolo.ops import preprocessing_ops +from official.vision.beta.projects.yolo.ops import anchor, preprocessing_ops class Parser(parser.Parser): diff --git a/official/vision/beta/projects/yolo/dataloaders/yolo_input_test.py b/official/vision/beta/projects/yolo/dataloaders/yolo_input_test.py index 68660a62e56..8c9003a0bfa 100644 --- a/official/vision/beta/projects/yolo/dataloaders/yolo_input_test.py +++ b/official/vision/beta/projects/yolo/dataloaders/yolo_input_test.py @@ -13,16 +13,19 @@ # limitations under the License. """Yolo Dataset Testing functions""" -from official.vision.beta.projects.yolo.common import registry_imports # pylint: disable=unused-import -from official.vision.beta.projects.yolo.tasks import image_classification as imc -from official.vision.beta.projects.yolo.configs import darknet_classification as dcfg - import os + import tensorflow as tf -from official.core import train_utils -from official.core import task_factory from absl.testing import parameterized +from official.core import task_factory, train_utils +from official.vision.beta.projects.yolo.common import \ + registry_imports # pylint: disable=unused-import +from official.vision.beta.projects.yolo.configs import \ + darknet_classification as dcfg +from official.vision.beta.projects.yolo.tasks import \ + image_classification as imc + PATH_TO_COCO = '/media/vbanna/DATA_SHARE/CV/datasets/COCO_raw/records/' def test_yolo_input_task(scaled_pipeline = True, batch_size = 1): @@ -116,4 +119,4 @@ def test_yolo_input(self, scaled_pipeline): if __name__ == '__main__': # tf.test.main() - test_yolo_pipeline_visually(is_training=True, num=20) \ No newline at end of file + test_yolo_pipeline_visually(is_training=True, num=20) diff --git a/official/vision/beta/projects/yolo/losses/__init__.py b/official/vision/beta/projects/yolo/losses/__init__.py index e419af524b5..e04127d3fc8 100644 --- a/official/vision/beta/projects/yolo/losses/__init__.py +++ b/official/vision/beta/projects/yolo/losses/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/official/vision/beta/projects/yolo/losses/yolo_loss.py b/official/vision/beta/projects/yolo/losses/yolo_loss.py index aac117bdf58..d16db20fed0 100755 --- a/official/vision/beta/projects/yolo/losses/yolo_loss.py +++ b/official/vision/beta/projects/yolo/losses/yolo_loss.py @@ -19,9 +19,8 @@ import tensorflow as tf -from official.vision.beta.projects.yolo.ops import box_ops -from official.vision.beta.projects.yolo.ops import loss_utils -from official.vision.beta.projects.yolo.ops import math_ops +from official.vision.beta.projects.yolo.ops import (box_ops, loss_utils, + math_ops) class YoloLossBase(object, metaclass=abc.ABCMeta): diff --git a/official/vision/beta/projects/yolo/losses/yolo_loss_test.py b/official/vision/beta/projects/yolo/losses/yolo_loss_test.py index b9490181269..49af31e94b0 100755 --- a/official/vision/beta/projects/yolo/losses/yolo_loss_test.py +++ b/official/vision/beta/projects/yolo/losses/yolo_loss_test.py @@ -14,8 +14,8 @@ """Tests for yolo heads.""" -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized from official.vision.beta.projects.yolo.losses import yolo_loss diff --git a/official/vision/beta/projects/yolo/modeling/backbones/darknet.py b/official/vision/beta/projects/yolo/modeling/backbones/darknet.py index 0cec8883d56..d1fe0002af9 100644 --- a/official/vision/beta/projects/yolo/modeling/backbones/darknet.py +++ b/official/vision/beta/projects/yolo/modeling/backbones/darknet.py @@ -36,6 +36,7 @@ """ import collections + import tensorflow as tf from official.modeling import hyperparams diff --git a/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py b/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py index 9441b06a311..a045768b9c6 100644 --- a/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py +++ b/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py @@ -15,12 +15,11 @@ # Lint as: python3 """Tests for yolo.""" -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations from official.vision.beta.projects.yolo.modeling.backbones import darknet diff --git a/official/vision/beta/projects/yolo/modeling/decoders/__init__.py b/official/vision/beta/projects/yolo/modeling/decoders/__init__.py index e419af524b5..e04127d3fc8 100644 --- a/official/vision/beta/projects/yolo/modeling/decoders/__init__.py +++ b/official/vision/beta/projects/yolo/modeling/decoders/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py index b19b465a31c..ac53f2e23f3 100644 --- a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py +++ b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder.py @@ -13,7 +13,7 @@ # limitations under the License. """Feature Pyramid Network and Path Aggregation variants used in YOLO.""" -from typing import Mapping, Union, Optional +from typing import Mapping, Optional, Union import tensorflow as tf diff --git a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder_test.py b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder_test.py index 611c4585945..0e2f764ced7 100644 --- a/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder_test.py +++ b/official/vision/beta/projects/yolo/modeling/decoders/yolo_decoder_test.py @@ -15,13 +15,13 @@ # Lint as: python3 """Tests for YOLO.""" +import tensorflow as tf # Import libraries from absl.testing import parameterized -import tensorflow as tf +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations -from official.vision.beta.projects.yolo.modeling.decoders import yolo_decoder as decoders +from official.vision.beta.projects.yolo.modeling.decoders import \ + yolo_decoder as decoders class YoloDecoderTest(parameterized.TestCase, tf.test.TestCase): diff --git a/official/vision/beta/projects/yolo/modeling/factory.py b/official/vision/beta/projects/yolo/modeling/factory.py index d33eda7994c..a116cd98290 100644 --- a/official/vision/beta/projects/yolo/modeling/factory.py +++ b/official/vision/beta/projects/yolo/modeling/factory.py @@ -15,14 +15,14 @@ """Contains common factory functions yolo neural networks.""" from absl import logging -from loguru import logger from official.vision.beta.modeling.backbones import factory as backbone_factory from official.vision.beta.modeling.decoders import factory as decoder_factory - from official.vision.beta.projects.yolo.configs import yolo, yolox from official.vision.beta.projects.yolo.modeling import yolo_model -from official.vision.beta.projects.yolo.modeling.heads import yolo_head, yolox_head -from official.vision.beta.projects.yolo.modeling.layers import detection_generator +from official.vision.beta.projects.yolo.modeling.heads import (yolo_head, + yolox_head) +from official.vision.beta.projects.yolo.modeling.layers import \ + detection_generator def build_yolo_detection_generator(model_config: yolo.Yolo, anchor_boxes): diff --git a/official/vision/beta/projects/yolo/modeling/heads/__init__.py b/official/vision/beta/projects/yolo/modeling/heads/__init__.py index e419af524b5..e04127d3fc8 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/__init__.py +++ b/official/vision/beta/projects/yolo/modeling/heads/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolo_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolo_head.py index 23d41a045e8..7ead787434d 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolo_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolo_head.py @@ -15,6 +15,7 @@ """Yolo heads.""" import tensorflow as tf + from official.vision.beta.projects.yolo.modeling.layers import nn_blocks diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolo_head_test.py b/official/vision/beta/projects/yolo/modeling/heads/yolo_head_test.py index 8c5414e5d84..4d5e4af3efb 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolo_head_test.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolo_head_test.py @@ -15,11 +15,12 @@ # Lint as: python3 """Tests for yolo heads.""" +import tensorflow as tf # Import libraries from absl.testing import parameterized -import tensorflow as tf -from official.vision.beta.projects.yolo.modeling.heads import yolo_head as heads +from official.vision.beta.projects.yolo.modeling.heads import \ + yolo_head as heads class YoloDecoderTest(parameterized.TestCase, tf.test.TestCase): diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index ac58d38e64e..7ad05d198f4 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -14,13 +14,14 @@ # Lint as: python3 """Yolox heads.""" -from loguru import logger import tensorflow as tf - +from loguru import logger from tensorflow.keras.models import Sequential + from official.vision.beta.projects.yolo.modeling.layers import nn_blocks from official.vision.beta.projects.yolo.ops import box_ops + class YOLOXHead(tf.keras.layers.Layer): """YOLOX Prediction Head.""" diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py index afb5c7793c5..0881388b49c 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py @@ -15,11 +15,12 @@ # Lint as: python3 """Tests for yolo heads.""" +import tensorflow as tf # Import libraries from absl.testing import parameterized -import tensorflow as tf -from official.vision.beta.projects.yolo.modeling.heads import yolox_head as heads +from official.vision.beta.projects.yolo.modeling.heads import \ + yolox_head as heads class YoloDecoderTest(parameterized.TestCase, tf.test.TestCase): diff --git a/official/vision/beta/projects/yolo/modeling/layers/detection_generator.py b/official/vision/beta/projects/yolo/modeling/layers/detection_generator.py index 68d70bdb978..ed0e26a6172 100644 --- a/official/vision/beta/projects/yolo/modeling/layers/detection_generator.py +++ b/official/vision/beta/projects/yolo/modeling/layers/detection_generator.py @@ -17,8 +17,7 @@ from official.vision.beta.modeling.layers import detection_generator from official.vision.beta.projects.yolo.losses import yolo_loss -from official.vision.beta.projects.yolo.ops import box_ops -from official.vision.beta.projects.yolo.ops import loss_utils +from official.vision.beta.projects.yolo.ops import box_ops, loss_utils @tf.keras.utils.register_keras_serializable(package='yolo') diff --git a/official/vision/beta/projects/yolo/modeling/layers/detection_generator_test.py b/official/vision/beta/projects/yolo/modeling/layers/detection_generator_test.py index ebe70060427..5d0d2b03a21 100644 --- a/official/vision/beta/projects/yolo/modeling/layers/detection_generator_test.py +++ b/official/vision/beta/projects/yolo/modeling/layers/detection_generator_test.py @@ -13,10 +13,11 @@ # limitations under the License. """Tests for yolo detection generator.""" -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized -from official.vision.beta.projects.yolo.modeling.layers import detection_generator as dg +from official.vision.beta.projects.yolo.modeling.layers import \ + detection_generator as dg class YoloDecoderTest(parameterized.TestCase, tf.test.TestCase): diff --git a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py index 331e9f126a5..d5d0851114b 100644 --- a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py +++ b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py @@ -15,13 +15,15 @@ from typing import Callable, List, Tuple import tensorflow as tf +import tensorflow.keras.backend as K +from tensorflow.keras import Sequential +from tensorflow.keras.layers import (Activation, BatchNormalization, Conv2D, + Layer, LeakyReLU, MaxPool2D, ReLU, + UpSampling2D, concatenate) from official.modeling import tf_utils from official.vision.beta.ops import spatial_transform_ops -import tensorflow.keras.backend as K -from tensorflow.keras import Sequential -from tensorflow.keras.layers import Layer, Conv2D, LeakyReLU, \ - MaxPool2D, UpSampling2D, Activation, ReLU, BatchNormalization, concatenate + @tf.keras.utils.register_keras_serializable(package='yolo') class Identity(tf.keras.layers.Layer): diff --git a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py index b43beefba60..71452792eb9 100644 --- a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py +++ b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Lint as: python3 -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Lint as: python3 +from absl.testing import parameterized from official.vision.beta.projects.yolo.modeling.layers import nn_blocks diff --git a/official/vision/beta/projects/yolo/modeling/yolo_model.py b/official/vision/beta/projects/yolo/modeling/yolo_model.py index 06f79750ea8..44129a2df55 100644 --- a/official/vision/beta/projects/yolo/modeling/yolo_model.py +++ b/official/vision/beta/projects/yolo/modeling/yolo_model.py @@ -15,7 +15,9 @@ """Yolo models.""" from typing import Mapping, Union + import tensorflow as tf + from official.vision.beta.projects.yolo.modeling.layers import nn_blocks diff --git a/official/vision/beta/projects/yolo/ops/__init__.py b/official/vision/beta/projects/yolo/ops/__init__.py index a25710c222e..e04127d3fc8 100644 --- a/official/vision/beta/projects/yolo/ops/__init__.py +++ b/official/vision/beta/projects/yolo/ops/__init__.py @@ -11,5 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - diff --git a/official/vision/beta/projects/yolo/ops/anchor.py b/official/vision/beta/projects/yolo/ops/anchor.py index 0904c7b1069..3aa6b09c738 100644 --- a/official/vision/beta/projects/yolo/ops/anchor.py +++ b/official/vision/beta/projects/yolo/ops/anchor.py @@ -16,9 +16,7 @@ import numpy as np import tensorflow as tf -from yolo.ops import box_ops -from yolo.ops import preprocessing_ops -from yolo.ops import loss_utils +from yolo.ops import box_ops, loss_utils, preprocessing_ops INF = 10000000 diff --git a/official/vision/beta/projects/yolo/ops/box_ops.py b/official/vision/beta/projects/yolo/ops/box_ops.py index 6d15f5d3157..968cb448522 100644 --- a/official/vision/beta/projects/yolo/ops/box_ops.py +++ b/official/vision/beta/projects/yolo/ops/box_ops.py @@ -14,7 +14,9 @@ """Yolo box ops.""" import math + import tensorflow as tf + from official.vision.beta.projects.yolo.ops import math_ops diff --git a/official/vision/beta/projects/yolo/ops/box_ops_test.py b/official/vision/beta/projects/yolo/ops/box_ops_test.py index afba1ee53c1..f0333209759 100644 --- a/official/vision/beta/projects/yolo/ops/box_ops_test.py +++ b/official/vision/beta/projects/yolo/ops/box_ops_test.py @@ -13,9 +13,9 @@ # limitations under the License. """box_ops tests.""" -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized from official.vision.beta.projects.yolo.ops import box_ops diff --git a/official/vision/beta/projects/yolo/ops/loss_utils.py b/official/vision/beta/projects/yolo/ops/loss_utils.py index 5536290199b..83946cd6a19 100755 --- a/official/vision/beta/projects/yolo/ops/loss_utils.py +++ b/official/vision/beta/projects/yolo/ops/loss_utils.py @@ -17,8 +17,7 @@ import numpy as np import tensorflow as tf -from official.vision.beta.projects.yolo.ops import box_ops -from official.vision.beta.projects.yolo.ops import math_ops +from official.vision.beta.projects.yolo.ops import box_ops, math_ops @tf.custom_gradient diff --git a/official/vision/beta/projects/yolo/ops/mosaic.py b/official/vision/beta/projects/yolo/ops/mosaic.py index cf386cd610b..c5be1d2ac45 100755 --- a/official/vision/beta/projects/yolo/ops/mosaic.py +++ b/official/vision/beta/projects/yolo/ops/mosaic.py @@ -14,11 +14,11 @@ """Mosaic op.""" import random + import tensorflow as tf import tensorflow_addons as tfa -from official.vision.beta.ops import box_ops -from official.vision.beta.ops import preprocess_ops +from official.vision.beta.ops import box_ops, preprocess_ops from official.vision.beta.projects.yolo.ops import preprocessing_ops diff --git a/official/vision/beta/projects/yolo/ops/preprocessing_ops_test.py b/official/vision/beta/projects/yolo/ops/preprocessing_ops_test.py index 43cca574b7f..a5dba12fc4e 100755 --- a/official/vision/beta/projects/yolo/ops/preprocessing_ops_test.py +++ b/official/vision/beta/projects/yolo/ops/preprocessing_ops_test.py @@ -13,9 +13,9 @@ # limitations under the License. """Tests for preprocessing_ops.py.""" -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized from official.vision.beta.ops import box_ops as bbox_ops from official.vision.beta.projects.yolo.ops import preprocessing_ops diff --git a/official/vision/beta/projects/yolo/optimization/__init__.py b/official/vision/beta/projects/yolo/optimization/__init__.py index 6ff51c80648..46d5d5003b9 100755 --- a/official/vision/beta/projects/yolo/optimization/__init__.py +++ b/official/vision/beta/projects/yolo/optimization/__init__.py @@ -14,9 +14,10 @@ """Optimization package definition.""" -# pylint: disable=wildcard-import from official.modeling.optimization.configs.learning_rate_config import * -from official.modeling.optimization.ema_optimizer import ExponentialMovingAverage +from official.modeling.optimization.ema_optimizer import \ + ExponentialMovingAverage from official.vision.beta.projects.yolo.optimization.configs.optimization_config import * from official.vision.beta.projects.yolo.optimization.configs.optimizer_config import * -from official.vision.beta.projects.yolo.optimization.optimizer_factory import OptimizerFactory as YoloOptimizerFactory +from official.vision.beta.projects.yolo.optimization.optimizer_factory import \ + OptimizerFactory as YoloOptimizerFactory diff --git a/official/vision/beta/projects/yolo/optimization/configs/__init__.py b/official/vision/beta/projects/yolo/optimization/configs/__init__.py index e419af524b5..e04127d3fc8 100755 --- a/official/vision/beta/projects/yolo/optimization/configs/__init__.py +++ b/official/vision/beta/projects/yolo/optimization/configs/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/official/vision/beta/projects/yolo/optimization/configs/optimization_config.py b/official/vision/beta/projects/yolo/optimization/configs/optimization_config.py index 92b8d1a79b1..64ba9e985c6 100755 --- a/official/vision/beta/projects/yolo/optimization/configs/optimization_config.py +++ b/official/vision/beta/projects/yolo/optimization/configs/optimization_config.py @@ -21,8 +21,10 @@ import dataclasses from typing import Optional -from official.modeling.optimization.configs import optimization_config as optimization_cfg -from official.vision.beta.projects.yolo.optimization.configs import optimizer_config as opt_cfg +from official.modeling.optimization.configs import \ + optimization_config as optimization_cfg +from official.vision.beta.projects.yolo.optimization.configs import \ + optimizer_config as opt_cfg @dataclasses.dataclass diff --git a/official/vision/beta/projects/yolo/optimization/optimizer_factory.py b/official/vision/beta/projects/yolo/optimization/optimizer_factory.py index b2126d16bc2..bd3414215bb 100755 --- a/official/vision/beta/projects/yolo/optimization/optimizer_factory.py +++ b/official/vision/beta/projects/yolo/optimization/optimizer_factory.py @@ -16,8 +16,7 @@ import gin -from official.modeling.optimization import ema_optimizer -from official.modeling.optimization import optimizer_factory +from official.modeling.optimization import ema_optimizer, optimizer_factory from official.vision.beta.projects.yolo.optimization import sgd_torch optimizer_factory.OPTIMIZERS_CLS.update({ diff --git a/official/vision/beta/projects/yolo/optimization/sgd_torch.py b/official/vision/beta/projects/yolo/optimization/sgd_torch.py index 289dc7a6d01..b5ea7e471fc 100644 --- a/official/vision/beta/projects/yolo/optimization/sgd_torch.py +++ b/official/vision/beta/projects/yolo/optimization/sgd_torch.py @@ -15,8 +15,8 @@ """SGD PyTorch optimizer.""" import re -from absl import logging import tensorflow as tf +from absl import logging LearningRateSchedule = tf.keras.optimizers.schedules.LearningRateSchedule diff --git a/official/vision/beta/projects/yolo/tasks/image_classification.py b/official/vision/beta/projects/yolo/tasks/image_classification.py index 4edef631fce..1094afb6586 100644 --- a/official/vision/beta/projects/yolo/tasks/image_classification.py +++ b/official/vision/beta/projects/yolo/tasks/image_classification.py @@ -15,10 +15,11 @@ """Image classification task definition.""" from official.common import dataset_fn from official.core import task_factory -from official.vision.beta.dataloaders import classification_input as classification_input_base -from official.vision.beta.dataloaders import input_reader_factory -from official.vision.beta.dataloaders import tfds_factory -from official.vision.beta.projects.yolo.configs import darknet_classification as exp_cfg +from official.vision.beta.dataloaders import \ + classification_input as classification_input_base +from official.vision.beta.dataloaders import input_reader_factory, tfds_factory +from official.vision.beta.projects.yolo.configs import \ + darknet_classification as exp_cfg from official.vision.beta.projects.yolo.dataloaders import classification_input from official.vision.beta.tasks import image_classification diff --git a/official/vision/beta/projects/yolo/tasks/yolo.py b/official/vision/beta/projects/yolo/tasks/yolo.py index a600a43a8be..cd8641cc83f 100755 --- a/official/vision/beta/projects/yolo/tasks/yolo.py +++ b/official/vision/beta/projects/yolo/tasks/yolo.py @@ -17,25 +17,22 @@ import collections from typing import Optional -from absl import logging import tensorflow as tf +from absl import logging -from official.core import base_task -from official.core import config_definitions -from official.core import input_reader -from official.core import task_factory +from official.core import (base_task, config_definitions, input_reader, + task_factory) from official.modeling import performance -from official.vision.beta.dataloaders import tfds_factory -from official.vision.beta.dataloaders import tf_example_label_map_decoder +from official.vision.beta.dataloaders import (tf_example_label_map_decoder, + tfds_factory) from official.vision.beta.evaluation import coco_evaluator from official.vision.beta.ops import box_ops from official.vision.beta.projects.yolo import optimization from official.vision.beta.projects.yolo.configs import yolo as exp_cfg -from official.vision.beta.projects.yolo.dataloaders import tf_example_decoder -from official.vision.beta.projects.yolo.dataloaders import yolo_input +from official.vision.beta.projects.yolo.dataloaders import (tf_example_decoder, + yolo_input) from official.vision.beta.projects.yolo.modeling import factory -from official.vision.beta.projects.yolo.ops import mosaic -from official.vision.beta.projects.yolo.ops import preprocessing_ops +from official.vision.beta.projects.yolo.ops import mosaic, preprocessing_ops from official.vision.beta.projects.yolo.tasks import task_utils OptimizationConfig = optimization.OptimizationConfig diff --git a/official/vision/beta/projects/yolo/tasks/yolox.py b/official/vision/beta/projects/yolo/tasks/yolox.py index e1e89ddbcfb..63629e8a1d6 100644 --- a/official/vision/beta/projects/yolo/tasks/yolox.py +++ b/official/vision/beta/projects/yolo/tasks/yolox.py @@ -17,25 +17,22 @@ import collections from typing import Optional -from absl import logging import tensorflow as tf +from absl import logging -from official.core import base_task -from official.core import config_definitions -from official.core import input_reader -from official.core import task_factory +from official.core import (base_task, config_definitions, input_reader, + task_factory) from official.modeling import performance -from official.vision.beta.dataloaders import tfds_factory -from official.vision.beta.dataloaders import tf_example_label_map_decoder +from official.vision.beta.dataloaders import (tf_example_label_map_decoder, + tfds_factory) from official.vision.beta.evaluation import coco_evaluator from official.vision.beta.ops import box_ops from official.vision.beta.projects.yolo import optimization from official.vision.beta.projects.yolo.configs import yolox as exp_cfg -from official.vision.beta.projects.yolo.dataloaders import tf_example_decoder -from official.vision.beta.projects.yolo.dataloaders import yolo_input +from official.vision.beta.projects.yolo.dataloaders import (tf_example_decoder, + yolo_input) from official.vision.beta.projects.yolo.modeling import factory -from official.vision.beta.projects.yolo.ops import mosaic -from official.vision.beta.projects.yolo.ops import preprocessing_ops +from official.vision.beta.projects.yolo.ops import mosaic, preprocessing_ops from official.vision.beta.projects.yolo.tasks import task_utils OptimizationConfig = optimization.OptimizationConfig diff --git a/official/vision/beta/projects/yolo/train.py b/official/vision/beta/projects/yolo/train.py index 78ee1ac32ae..09e97ab74ba 100644 --- a/official/vision/beta/projects/yolo/train.py +++ b/official/vision/beta/projects/yolo/train.py @@ -14,12 +14,12 @@ """TensorFlow Model Garden Vision training driver.""" -from absl import app -from absl import flags +from absl import app, flags from official.common import flags as tfm_flags from official.vision.beta import train -from official.vision.beta.projects.yolo.common import registry_imports # pylint: disable=unused-import +from official.vision.beta.projects.yolo.common import \ + registry_imports # pylint: disable=unused-import FLAGS = flags.FLAGS From 3253b382a3d118ddf89a166da13d31e46072d8d5 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Wed, 12 Jan 2022 18:10:29 -0500 Subject: [PATCH 53/96] remove loguru --- official/vision/beta/projects/yolo/modeling/heads/yolox_head.py | 1 - 1 file changed, 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index 7ad05d198f4..0b2ef1bb723 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -15,7 +15,6 @@ # Lint as: python3 """Yolox heads.""" import tensorflow as tf -from loguru import logger from tensorflow.keras.models import Sequential from official.vision.beta.projects.yolo.modeling.layers import nn_blocks From 146375a43865fe6ddf4e86243812948fd8e7eed6 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Wed, 12 Jan 2022 18:13:23 -0500 Subject: [PATCH 54/96] update yaml --- .../yolo/configs/experiments/yolox/yolox.yaml | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 42cea1bd036..593d8a0d49a 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -70,23 +70,23 @@ task: area_thresh: 0.1 validation_data: input_path: 'gs://cam2-datasets/coco/val*' - trainer: - train_steps: 831600 # epoch 300 to 450 - optimizer_config: - learning_rate: - cosine: - initial_learning_rate: 0.1 - decay_steps: 831600 # epoch 300 to 450 - optimizer: - type: sgd_torch - sgd_torch: - momentum: 0.9 - momentum_start: 0.9 - nesterov: true - warmup_steps: 4620 - weight_decay: 0.0005 - name: SGD - warmup: - type: 'linear' - linear: - warmup_steps: 4620 # 5 epochs +trainer: + train_steps: 831600 # epoch 300 to 450 + optimizer_config: + learning_rate: + cosine: + initial_learning_rate: 0.1 + decay_steps: 831600 # epoch 300 to 450 + optimizer: + type: sgd_torch + sgd_torch: + momentum: 0.9 + momentum_start: 0.9 + nesterov: true + warmup_steps: 4620 + weight_decay: 0.0005 + name: SGD + warmup: + type: 'linear' + linear: + warmup_steps: 4620 # 5 epochs From 29e6fb0b893e1dfd68a1ed72f6e4bcea33d0ae4d Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Wed, 12 Jan 2022 18:15:14 -0500 Subject: [PATCH 55/96] update yaml --- .../yolo/configs/experiments/yolox/yolox.yaml | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 593d8a0d49a..2ae1f278ab8 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -77,16 +77,16 @@ trainer: cosine: initial_learning_rate: 0.1 decay_steps: 831600 # epoch 300 to 450 - optimizer: - type: sgd_torch - sgd_torch: - momentum: 0.9 - momentum_start: 0.9 - nesterov: true - warmup_steps: 4620 - weight_decay: 0.0005 - name: SGD - warmup: - type: 'linear' - linear: - warmup_steps: 4620 # 5 epochs + optimizer: + type: sgd_torch + sgd_torch: + momentum: 0.9 + momentum_start: 0.9 + nesterov: true + warmup_steps: 4620 + weight_decay: 0.0005 + name: SGD + warmup: + type: 'linear' + linear: + warmup_steps: 4620 # 5 epochs From df84c2470d6cab0fda64192e25bb678489bb3a9f Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Wed, 12 Jan 2022 18:17:17 -0500 Subject: [PATCH 56/96] remove logger --- official/vision/beta/projects/yolo/modeling/factory.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/factory.py b/official/vision/beta/projects/yolo/modeling/factory.py index a116cd98290..0b3b4cd0fbc 100644 --- a/official/vision/beta/projects/yolo/modeling/factory.py +++ b/official/vision/beta/projects/yolo/modeling/factory.py @@ -142,9 +142,6 @@ def build_yolox(input_specs, model_config, l2_regularization): model.build(input_specs.shape) model.summary(print_fn=logging.info) - logger.info(model.summary) - logger.info(model) - losses = detection_generator_obj.get_losses() return model, losses From 15c8c0184a978d998b97f18230b7952650774602 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Sat, 15 Jan 2022 20:47:29 -0500 Subject: [PATCH 57/96] Update training steps --- .../beta/projects/yolo/configs/experiments/yolox/yolox.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 2ae1f278ab8..1b3ae51979f 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -71,12 +71,12 @@ task: validation_data: input_path: 'gs://cam2-datasets/coco/val*' trainer: - train_steps: 831600 # epoch 300 to 450 + train_steps: 277200 # epoch 300 optimizer_config: learning_rate: cosine: initial_learning_rate: 0.1 - decay_steps: 831600 # epoch 300 to 450 + decay_steps: 277200 # epoch 300 optimizer: type: sgd_torch sgd_torch: From 550ee6fa1e50b277aae92feadde9ca97f8e6aafa Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Sat, 15 Jan 2022 21:02:51 -0500 Subject: [PATCH 58/96] set train_batch_size to 128 --- official/vision/beta/projects/yolo/configs/yolox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/configs/yolox.py b/official/vision/beta/projects/yolo/configs/yolox.py index b5f39b108e1..aa7c9db9b28 100644 --- a/official/vision/beta/projects/yolo/configs/yolox.py +++ b/official/vision/beta/projects/yolo/configs/yolox.py @@ -272,7 +272,7 @@ def yolox() -> cfg.ExperimentConfig: @exp_factory.register_config_factory('yolox_regular') def yolox_regular() -> cfg.ExperimentConfig: """COCO object detection with YOLOvx.""" - train_batch_size = 64 + train_batch_size = 128 eval_batch_size = 8 train_epochs = 300 steps_per_epoch = COCO_TRAIN_EXAMPLES // train_batch_size From 2ec399939227e84ba471d6df04e5b2f7e41e3634 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Sat, 22 Jan 2022 16:52:16 -0500 Subject: [PATCH 59/96] Update yolox.yaml --- .../yolo/configs/experiments/yolox/yolox.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 1b3ae51979f..e69955b2e1e 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -56,13 +56,18 @@ task: input_path: 'gs://cam2-datasets/coco/train*' shuffle_buffer_size: 10000 parser: + aug_rand_saturation: 1.5 + aug_rand_brightness: 1.5 + aug_rand_hue: 0.1 mosaic: mosaic_frequency: 1.0 - mixup_frequency: 0.2 + mixup_frequency: 1.0 mosaic_crop_mode: 'scale' mosaic_center: 0.25 aug_scale_min: 0.1 aug_scale_max: 1.9 + aug_rand_angle: 10 + max_num_instances: 300 letter_box: true random_flip: true @@ -72,10 +77,12 @@ task: input_path: 'gs://cam2-datasets/coco/val*' trainer: train_steps: 277200 # epoch 300 + validation_steps: 625 + validation_interval: 4620 optimizer_config: learning_rate: cosine: - initial_learning_rate: 0.1 + initial_learning_rate: 0.02 decay_steps: 277200 # epoch 300 optimizer: type: sgd_torch @@ -90,3 +97,4 @@ trainer: type: 'linear' linear: warmup_steps: 4620 # 5 epochs + warmup_learning_rate: 0 From d48425e4a2861ee860fa3579d03fb58f5a75f99c Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Sat, 22 Jan 2022 17:04:38 -0500 Subject: [PATCH 60/96] Update yolox.py --- official/vision/beta/projects/yolo/configs/yolox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/configs/yolox.py b/official/vision/beta/projects/yolo/configs/yolox.py index aa7c9db9b28..683b1965822 100644 --- a/official/vision/beta/projects/yolo/configs/yolox.py +++ b/official/vision/beta/projects/yolo/configs/yolox.py @@ -348,7 +348,7 @@ def yolox_regular() -> cfg.ExperimentConfig: checkpoint_interval=steps_per_epoch, optimizer_config=optimization.OptimizationConfig({ 'ema': { - 'average_decay': 0.9998, + 'average_decay': 0.9999, 'trainable_weights_only': False, 'dynamic_decay': True, }, From 683cd35b793145aa1ec10dbffbe9517d8e751068 Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Sat, 22 Jan 2022 17:20:31 -0500 Subject: [PATCH 61/96] Update yolox.py --- official/vision/beta/projects/yolo/configs/yolox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/configs/yolox.py b/official/vision/beta/projects/yolo/configs/yolox.py index 683b1965822..aa7c9db9b28 100644 --- a/official/vision/beta/projects/yolo/configs/yolox.py +++ b/official/vision/beta/projects/yolo/configs/yolox.py @@ -348,7 +348,7 @@ def yolox_regular() -> cfg.ExperimentConfig: checkpoint_interval=steps_per_epoch, optimizer_config=optimization.OptimizationConfig({ 'ema': { - 'average_decay': 0.9999, + 'average_decay': 0.9998, 'trainable_weights_only': False, 'dynamic_decay': True, }, From aebd5df80add33c70c2f1432bf47e143f61f3e9f Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Mon, 24 Jan 2022 15:33:16 -0500 Subject: [PATCH 62/96] solve conflicts --- .../vision/beta/projects/yolo/configs/yolo.py | 132 +++++- .../beta/projects/yolo/configs/yolox.py | 390 ----------------- .../beta/projects/yolo/modeling/factory.py | 66 +-- .../vision/beta/projects/yolo/tasks/yolox.py | 405 ------------------ 4 files changed, 145 insertions(+), 848 deletions(-) delete mode 100644 official/vision/beta/projects/yolo/configs/yolox.py delete mode 100644 official/vision/beta/projects/yolo/tasks/yolox.py diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index 6db523a6024..a2bae1c3474 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -135,6 +135,15 @@ class YoloHead(hyperparams.Config): smart_bias: bool = True +@dataclasses.dataclass +class YOLOXHead(hyperparams.Config): + """Parameterization for the YOLOX Head.""" + width: float = 1.0 + depthwise: bool = False + activation: str = 'silu' + smart_bias: bool = True + + @dataclasses.dataclass class YoloDetectionGenerator(hyperparams.Config): box_type: FPNConfig = dataclasses.field( @@ -261,7 +270,7 @@ def yolo() -> cfg.ExperimentConfig: 'task.train_data.is_training != None', 'task.validation_data.is_training != None' ]) - + @exp_factory.register_config_factory('yolo_darknet') def yolo_darknet() -> cfg.ExperimentConfig: @@ -727,3 +736,124 @@ def yolo_tiny() -> cfg.ExperimentConfig: ]) return config + + +@exp_factory.register_config_factory('yolox_regular') +def yolox_regular() -> cfg.ExperimentConfig: + """COCO object detection with YOLOvx.""" + train_batch_size = 128 + eval_batch_size = 8 + train_epochs = 300 + steps_per_epoch = COCO_TRAIN_EXAMPLES // train_batch_size + validation_interval = 5 + + max_num_instances = 200 + config = cfg.ExperimentConfig( + runtime=cfg.RuntimeConfig(mixed_precision_dtype='bfloat16'), + task=YoloTask( + smart_bias_lr=0.1, + init_checkpoint='', + init_checkpoint_modules='backbone', + annotation_file=None, + weight_decay=0.0, + model=Yolo( + darknet_based_model=True, + norm_activation=common.NormActivation(use_sync_bn=True), + head=YOLOXHead(smart_bias=True), + loss=YoloLoss(use_scaled_loss=False, update_on_repeat=True), + anchor_boxes=AnchorBoxes( + anchors_per_scale=3, + boxes=[ + Box(box=[12, 16]), + Box(box=[19, 36]), + Box(box=[40, 28]), + Box(box=[36, 75]), + Box(box=[76, 55]), + Box(box=[72, 146]), + Box(box=[142, 110]), + Box(box=[192, 243]), + Box(box=[459, 401]) + ])), + train_data=DataConfig( + input_path=os.path.join(COCO_INPUT_PATH_BASE, 'train*'), + is_training=True, + global_batch_size=train_batch_size, + dtype='float32', + parser=Parser( + letter_box=False, + aug_rand_saturation=1.5, + aug_rand_brightness=1.5, + aug_rand_hue=0.1, + use_tie_breaker=True, + best_match_only=False, + anchor_thresh=0.4, + area_thresh=0.1, + max_num_instances=max_num_instances, + mosaic=Mosaic( + mosaic_frequency=0.75, + mixup_frequency=0.0, + mosaic_crop_mode='crop', + mosaic_center=0.2))), + validation_data=DataConfig( + input_path=os.path.join(COCO_INPUT_PATH_BASE, 'val*'), + is_training=False, + global_batch_size=eval_batch_size, + drop_remainder=True, + dtype='float32', + parser=Parser( + letter_box=False, + use_tie_breaker=True, + best_match_only=False, + anchor_thresh=0.4, + area_thresh=0.1, + max_num_instances=max_num_instances, + ))), + trainer=cfg.TrainerConfig( + train_steps=train_epochs * steps_per_epoch, + validation_steps=COCO_VAL_EXAMPLES // eval_batch_size, + validation_interval=validation_interval * steps_per_epoch, + steps_per_loop=steps_per_epoch, + summary_interval=steps_per_epoch, + checkpoint_interval=steps_per_epoch, + optimizer_config=optimization.OptimizationConfig({ + 'ema': { + 'average_decay': 0.9998, + 'trainable_weights_only': False, + 'dynamic_decay': True, + }, + 'optimizer': { + 'type': 'sgd_torch', + 'sgd_torch': { + 'momentum': 0.949, + 'momentum_start': 0.949, + 'nesterov': True, + 'warmup_steps': 1000, + 'weight_decay': 0.0005, + } + }, + 'learning_rate': { + 'type': 'stepwise', + 'stepwise': { + 'boundaries': [ + 240 * steps_per_epoch + ], + 'values': [ + 0.00131 * train_batch_size / 64.0, + 0.000131 * train_batch_size / 64.0, + ] + } + }, + 'warmup': { + 'type': 'linear', + 'linear': { + 'warmup_steps': 5 * steps_per_epoch, + 'warmup_learning_rate': 0 + } + } + })), + restrictions=[ + 'task.train_data.is_training != None', + 'task.validation_data.is_training != None' + ]) + + return config diff --git a/official/vision/beta/projects/yolo/configs/yolox.py b/official/vision/beta/projects/yolo/configs/yolox.py deleted file mode 100644 index aa7c9db9b28..00000000000 --- a/official/vision/beta/projects/yolo/configs/yolox.py +++ /dev/null @@ -1,390 +0,0 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""YOLO configuration definition.""" -import dataclasses -import os -from typing import Any, List, Optional, Union - -import numpy as np - -from official.core import config_definitions as cfg -from official.core import exp_factory -from official.modeling import hyperparams -from official.vision.beta.configs import common -from official.vision.beta.projects.yolo import optimization -from official.vision.beta.projects.yolo.configs import backbones, decoders - -# pytype: disable=annotation-type-mismatch - -MIN_LEVEL = 1 -MAX_LEVEL = 7 -GLOBAL_SEED = 1000 - - -def _build_dict(min_level, max_level, value): - vals = {str(key): value for key in range(min_level, max_level + 1)} - vals['all'] = None - return lambda: vals - - -def _build_path_scales(min_level, max_level): - return lambda: {str(key): 2**key for key in range(min_level, max_level + 1)} - - -@dataclasses.dataclass -class FPNConfig(hyperparams.Config): - """FPN config.""" - all: Optional[Any] = None - - def get(self): - """Allow for a key for each level or a single key for all the levels.""" - values = self.as_dict() - if 'all' in values and values['all'] is not None: - for key in values: - if key != 'all': - values[key] = values['all'] - return values - - -# pylint: disable=missing-class-docstring -@dataclasses.dataclass -class TfExampleDecoder(hyperparams.Config): - regenerate_source_id: bool = False - coco91_to_80: bool = True - - -@dataclasses.dataclass -class TfExampleDecoderLabelMap(hyperparams.Config): - regenerate_source_id: bool = False - label_map: str = '' - - -@dataclasses.dataclass -class DataDecoder(hyperparams.OneOfConfig): - type: Optional[str] = 'simple_decoder' - simple_decoder: TfExampleDecoder = TfExampleDecoder() - label_map_decoder: TfExampleDecoderLabelMap = TfExampleDecoderLabelMap() - - -@dataclasses.dataclass -class Mosaic(hyperparams.Config): - mosaic_frequency: float = 0.0 - mixup_frequency: float = 0.0 - mosaic_center: float = 0.2 - mosaic_crop_mode: Optional[str] = None - aug_scale_min: float = 1.0 - aug_scale_max: float = 1.0 - jitter: float = 0.0 - - -@dataclasses.dataclass -class Parser(hyperparams.Config): - max_num_instances: int = 200 - letter_box: Optional[bool] = True - random_flip: bool = True - random_pad: float = False - jitter: float = 0.0 - aug_scale_min: float = 1.0 - aug_scale_max: float = 1.0 - aug_rand_saturation: float = 0.0 - aug_rand_brightness: float = 0.0 - aug_rand_hue: float = 0.0 - aug_rand_angle: float = 0.0 - aug_rand_translate: float = 0.0 - aug_rand_perspective: float = 0.0 - use_tie_breaker: bool = True - best_match_only: bool = False - anchor_thresh: float = -0.01 - area_thresh: float = 0.1 - mosaic: Mosaic = Mosaic() - - -@dataclasses.dataclass -class DataConfig(cfg.DataConfig): - """Input config for training.""" - global_batch_size: int = 64 - input_path: str = '' - tfds_name: str = '' - tfds_split: str = '' - global_batch_size: int = 1 - is_training: bool = True - dtype: str = 'float16' - decoder: DataDecoder = DataDecoder() - parser: Parser = Parser() - shuffle_buffer_size: int = 10000 - tfds_download: bool = True - cache: bool = False - drop_remainder: bool = True - - -@dataclasses.dataclass -class YOLOXHead(hyperparams.Config): - """Parameterization for the YOLOX Head.""" - #classes: int = None - width: float = 1.0 - # strides: List[int] = [8, 16, 32] - # in_channels: List[int] = [256, 512, 1024] - depthwise: bool = False - activation: str = 'silu' - smart_bias: bool = True - - -@dataclasses.dataclass -class YoloDetectionGenerator(hyperparams.Config): - box_type: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 'original')) - scale_xy: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) - path_scales: FPNConfig = dataclasses.field( - default_factory=_build_path_scales(MIN_LEVEL, MAX_LEVEL)) - nms_type: str = 'greedy' - iou_thresh: float = 0.001 - nms_thresh: float = 0.6 - max_boxes: int = 200 - pre_nms_points: int = 5000 - - -@dataclasses.dataclass -class YoloLoss(hyperparams.Config): - ignore_thresh: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 0.0)) - truth_thresh: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) - box_loss_type: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 'ciou')) - iou_normalizer: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) - cls_normalizer: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) - object_normalizer: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) - max_delta: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, np.inf)) - objectness_smooth: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 0.0)) - label_smoothing: float = 0.0 - use_scaled_loss: bool = True - update_on_repeat: bool = True - - -@dataclasses.dataclass -class Box(hyperparams.Config): - box: List[int] = dataclasses.field(default=list) - - -@dataclasses.dataclass -class AnchorBoxes(hyperparams.Config): - boxes: Optional[List[Box]] = None - level_limits: Optional[List[int]] = None - anchors_per_scale: int = 3 - - def get(self, min_level, max_level): - """Distribute them in order to each level. - - Args: - min_level: `int` the lowest output level. - max_level: `int` the heighest output level. - Returns: - anchors_per_level: A `Dict[List[int]]` of the anchor boxes for each level. - self.level_limits: A `List[int]` of the box size limits to link to each - level under anchor free conditions. - """ - if self.level_limits is None: - boxes = [box.box for box in self.boxes] - else: - boxes = [[1.0, 1.0]] * ((max_level - min_level) + 1) - self.anchors_per_scale = 1 - - anchors_per_level = dict() - start = 0 - for i in range(min_level, max_level + 1): - anchors_per_level[str(i)] = boxes[start:start + self.anchors_per_scale] - start += self.anchors_per_scale - return anchors_per_level, self.level_limits - - -@dataclasses.dataclass -class YOLOX(hyperparams.Config): - input_size: Optional[List[int]] = dataclasses.field( - default_factory=lambda: [640, 640, 3]) - backbone: backbones.Backbone = backbones.Backbone( - type='darknet', darknet=backbones.Darknet(model_id='darknet53')) - decoder: decoders.Decoder = decoders.Decoder( - type='yolo_decoder', - yolo_decoder=decoders.YoloDecoder(version='vx', type='regular')) - head: YOLOXHead = YOLOXHead() - detection_generator: YoloDetectionGenerator = YoloDetectionGenerator() - loss: YoloLoss = YoloLoss() - norm_activation: common.NormActivation = common.NormActivation( - activation='mish', - use_sync_bn=True, - norm_momentum=0.99, - norm_epsilon=0.001) - num_classes: int = 80 - anchor_boxes: AnchorBoxes = AnchorBoxes() - darknet_based_model: bool = False - - -@dataclasses.dataclass -class YOLOXTask(cfg.TaskConfig): - per_category_metrics: bool = False - smart_bias_lr: float = 0.0 - model: YOLOX = YOLOX() - train_data: DataConfig = DataConfig(is_training=True) - validation_data: DataConfig = DataConfig(is_training=False) - weight_decay: float = 0.0 - annotation_file: Optional[str] = None - init_checkpoint: Optional[str] = None - init_checkpoint_modules: Union[ - str, List[str]] = 'all' # all, backbone, and/or decoder - gradient_clip_norm: float = 0.0 - seed = GLOBAL_SEED - - -COCO_INPUT_PATH_BASE = 'coco' -COCO_TRAIN_EXAMPLES = 118287 -COCO_VAL_EXAMPLES = 5000 - - -@exp_factory.register_config_factory('yolox') -def yolox() -> cfg.ExperimentConfig: - """Yolox general config.""" - return cfg.ExperimentConfig( - task=YOLOXTask(), - restrictions=[ - 'task.train_data.is_training != None', - 'task.validation_data.is_training != None' - ]) - - -@exp_factory.register_config_factory('yolox_regular') -def yolox_regular() -> cfg.ExperimentConfig: - """COCO object detection with YOLOvx.""" - train_batch_size = 128 - eval_batch_size = 8 - train_epochs = 300 - steps_per_epoch = COCO_TRAIN_EXAMPLES // train_batch_size - validation_interval = 5 - - max_num_instances = 200 - config = cfg.ExperimentConfig( - runtime=cfg.RuntimeConfig(mixed_precision_dtype='bfloat16'), - task=YOLOXTask( - smart_bias_lr=0.1, - init_checkpoint='', - init_checkpoint_modules='backbone', - annotation_file=None, - weight_decay=0.0, - model=YOLOX( - darknet_based_model=True, - norm_activation=common.NormActivation(use_sync_bn=True), - head=YOLOXHead(smart_bias=True), - loss=YoloLoss(use_scaled_loss=False, update_on_repeat=True), - anchor_boxes=AnchorBoxes( - anchors_per_scale=3, - boxes=[ - Box(box=[12, 16]), - Box(box=[19, 36]), - Box(box=[40, 28]), - Box(box=[36, 75]), - Box(box=[76, 55]), - Box(box=[72, 146]), - Box(box=[142, 110]), - Box(box=[192, 243]), - Box(box=[459, 401]) - ])), - train_data=DataConfig( - input_path=os.path.join(COCO_INPUT_PATH_BASE, 'train*'), - is_training=True, - global_batch_size=train_batch_size, - dtype='float32', - parser=Parser( - letter_box=False, - aug_rand_saturation=1.5, - aug_rand_brightness=1.5, - aug_rand_hue=0.1, - use_tie_breaker=True, - best_match_only=False, - anchor_thresh=0.4, - area_thresh=0.1, - max_num_instances=max_num_instances, - mosaic=Mosaic( - mosaic_frequency=0.75, - mixup_frequency=0.0, - mosaic_crop_mode='crop', - mosaic_center=0.2))), - validation_data=DataConfig( - input_path=os.path.join(COCO_INPUT_PATH_BASE, 'val*'), - is_training=False, - global_batch_size=eval_batch_size, - drop_remainder=True, - dtype='float32', - parser=Parser( - letter_box=False, - use_tie_breaker=True, - best_match_only=False, - anchor_thresh=0.4, - area_thresh=0.1, - max_num_instances=max_num_instances, - ))), - trainer=cfg.TrainerConfig( - train_steps=train_epochs * steps_per_epoch, - validation_steps=COCO_VAL_EXAMPLES // eval_batch_size, - validation_interval=validation_interval * steps_per_epoch, - steps_per_loop=steps_per_epoch, - summary_interval=steps_per_epoch, - checkpoint_interval=steps_per_epoch, - optimizer_config=optimization.OptimizationConfig({ - 'ema': { - 'average_decay': 0.9998, - 'trainable_weights_only': False, - 'dynamic_decay': True, - }, - 'optimizer': { - 'type': 'sgd_torch', - 'sgd_torch': { - 'momentum': 0.949, - 'momentum_start': 0.949, - 'nesterov': True, - 'warmup_steps': 1000, - 'weight_decay': 0.0005, - } - }, - 'learning_rate': { - 'type': 'stepwise', - 'stepwise': { - 'boundaries': [ - 240 * steps_per_epoch - ], - 'values': [ - 0.00131 * train_batch_size / 64.0, - 0.000131 * train_batch_size / 64.0, - ] - } - }, - 'warmup': { - 'type': 'linear', - 'linear': { - 'warmup_steps': 5 * steps_per_epoch, - 'warmup_learning_rate': 0 - } - } - })), - restrictions=[ - 'task.train_data.is_training != None', - 'task.validation_data.is_training != None' - ]) - - return config diff --git a/official/vision/beta/projects/yolo/modeling/factory.py b/official/vision/beta/projects/yolo/modeling/factory.py index 0b3b4cd0fbc..947df65f28c 100644 --- a/official/vision/beta/projects/yolo/modeling/factory.py +++ b/official/vision/beta/projects/yolo/modeling/factory.py @@ -15,9 +15,10 @@ """Contains common factory functions yolo neural networks.""" from absl import logging + from official.vision.beta.modeling.backbones import factory as backbone_factory from official.vision.beta.modeling.decoders import factory as decoder_factory -from official.vision.beta.projects.yolo.configs import yolo, yolox +from official.vision.beta.projects.yolo.configs import yolo from official.vision.beta.projects.yolo.modeling import yolo_model from official.vision.beta.projects.yolo.modeling.heads import (yolo_head, yolox_head) @@ -56,22 +57,9 @@ def build_yolo_head(input_specs, model_config: yolo.Yolo, l2_regularization): """Builds yolo head.""" min_level = min(map(int, input_specs.keys())) max_level = max(map(int, input_specs.keys())) - head = yolo_head.YoloHead( - min_level=min_level, - max_level=max_level, - classes=model_config.num_classes, - boxes_per_level=model_config.anchor_boxes.anchors_per_scale, - norm_momentum=model_config.norm_activation.norm_momentum, - norm_epsilon=model_config.norm_activation.norm_epsilon, - kernel_regularizer=l2_regularization, - smart_bias=model_config.head.smart_bias) - return head -def build_yolox_head(input_specs, model_config: yolox.yolox, l2_regularization): - """Builds yolo head.""" - min_level = min(map(int, input_specs.keys())) - max_level = max(map(int, input_specs.keys())) - head = yolox_head.YOLOXHead( + if model_config == yolo.yolox: + head = yolox_head.YOLOXHead( min_level=min_level, max_level=max_level, classes=model_config.num_classes, @@ -80,6 +68,16 @@ def build_yolox_head(input_specs, model_config: yolox.yolox, l2_regularization): norm_epsilon=model_config.norm_activation.norm_epsilon, kernel_regularizer=l2_regularization, smart_bias=model_config.head.smart_bias) + else: + head = yolo_head.YoloHead( + min_level=min_level, + max_level=max_level, + classes=model_config.num_classes, + boxes_per_level=model_config.anchor_boxes.anchors_per_scale, + norm_momentum=model_config.norm_activation.norm_momentum, + norm_epsilon=model_config.norm_activation.norm_epsilon, + kernel_regularizer=l2_regularization, + smart_bias=model_config.head.smart_bias) return head @@ -109,39 +107,3 @@ def build_yolo(input_specs, model_config, l2_regularization): losses = detection_generator_obj.get_losses() return model, losses - -def build_yolox(input_specs, model_config, l2_regularization): - """Builds yolox model.""" - backbone = model_config.backbone.get() - anchor_dict, _ = model_config.anchor_boxes.get( - backbone.min_level, backbone.max_level) - backbone = backbone_factory.build_backbone(input_specs, model_config.backbone, - model_config.norm_activation, - l2_regularization) - decoder = decoder_factory.build_decoder(backbone.output_specs, model_config, - l2_regularization) - - head = build_yolox_head(decoder.output_specs, model_config, l2_regularization) - detection_generator_obj = build_yolo_detection_generator(model_config, - anchor_dict) - - model = yolo_model.Yolo( - backbone=backbone, - decoder=decoder, - head=head, - detection_generator=detection_generator_obj) - - # from tensorflow.keras import Input - # from tensorflow.keras import Model - # h, w = (640,640) - # inputs = Input(shape=(h, w,3)) - # outputs = model(inputs) - # model=Model(inputs,outputs) - # model.summary() - - model.build(input_specs.shape) - - model.summary(print_fn=logging.info) - - losses = detection_generator_obj.get_losses() - return model, losses diff --git a/official/vision/beta/projects/yolo/tasks/yolox.py b/official/vision/beta/projects/yolo/tasks/yolox.py deleted file mode 100644 index 63629e8a1d6..00000000000 --- a/official/vision/beta/projects/yolo/tasks/yolox.py +++ /dev/null @@ -1,405 +0,0 @@ -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Contains classes used to train Yolox.""" - -import collections -from typing import Optional - -import tensorflow as tf -from absl import logging - -from official.core import (base_task, config_definitions, input_reader, - task_factory) -from official.modeling import performance -from official.vision.beta.dataloaders import (tf_example_label_map_decoder, - tfds_factory) -from official.vision.beta.evaluation import coco_evaluator -from official.vision.beta.ops import box_ops -from official.vision.beta.projects.yolo import optimization -from official.vision.beta.projects.yolo.configs import yolox as exp_cfg -from official.vision.beta.projects.yolo.dataloaders import (tf_example_decoder, - yolo_input) -from official.vision.beta.projects.yolo.modeling import factory -from official.vision.beta.projects.yolo.ops import mosaic, preprocessing_ops -from official.vision.beta.projects.yolo.tasks import task_utils - -OptimizationConfig = optimization.OptimizationConfig -RuntimeConfig = config_definitions.RuntimeConfig - - -@task_factory.register_task_cls(exp_cfg.YOLOXTask) -class YOLOXTask(base_task.Task): - """A single-replica view of training procedure. - - YOLOX task provides artifacts for training/evalution procedures, including - loading/iterating over Datasets, initializing the model, calculating the loss, - post-processing, and customized metrics with reduction. - """ - - def __init__(self, params, logging_dir: Optional[str] = None): - super().__init__(params, logging_dir) - self.coco_metric = None - self._loss_fn = None - self._model = None - self._coco_91_to_80 = False - self._metrics = [] - - # globally set the random seed - preprocessing_ops.set_random_seeds(seed=params.seed) - return - - def build_model(self): - """Build an instance of Yolox.""" - - model_base_cfg = self.task_config.model - l2_weight_decay = self.task_config.weight_decay / 2.0 - - input_size = model_base_cfg.input_size.copy() - input_specs = tf.keras.layers.InputSpec(shape=[None] + input_size) - l2_regularizer = ( - tf.keras.regularizers.l2(l2_weight_decay) if l2_weight_decay else None) - model, losses = factory.build_yolox( - input_specs, model_base_cfg, l2_regularizer) - - # save for later usage within the task. - self._loss_fn = losses - self._model = model - return model - - def _get_data_decoder(self, params): - """Get a decoder object to decode the dataset.""" - if params.tfds_name: - decoder = tfds_factory.get_detection_decoder(params.tfds_name) - else: - decoder_cfg = params.decoder.get() - if params.decoder.type == 'simple_decoder': - self._coco_91_to_80 = decoder_cfg.coco91_to_80 - decoder = tf_example_decoder.TfExampleDecoder( - coco91_to_80=decoder_cfg.coco91_to_80, - regenerate_source_id=decoder_cfg.regenerate_source_id) - elif params.decoder.type == 'label_map_decoder': - decoder = tf_example_label_map_decoder.TfExampleDecoderLabelMap( - label_map=decoder_cfg.label_map, - regenerate_source_id=decoder_cfg.regenerate_source_id) - else: - raise ValueError('Unknown decoder type: {}!'.format( - params.decoder.type)) - return decoder - - def build_inputs(self, params, input_context=None): - """Build input dataset.""" - model = self.task_config.model - - # get anchor boxes dict based on models min and max level - backbone = model.backbone.get() - anchor_dict, level_limits = model.anchor_boxes.get(backbone.min_level, - backbone.max_level) - - params.seed = self.task_config.seed - # set shared patamters between mosaic and yolo_input - base_config = dict( - letter_box=params.parser.letter_box, - aug_rand_translate=params.parser.aug_rand_translate, - aug_rand_angle=params.parser.aug_rand_angle, - aug_rand_perspective=params.parser.aug_rand_perspective, - area_thresh=params.parser.area_thresh, - random_flip=params.parser.random_flip, - seed=params.seed, - ) - - # get the decoder - decoder = self._get_data_decoder(params) - - # init Mosaic - sample_fn = mosaic.Mosaic( - output_size=model.input_size, - mosaic_frequency=params.parser.mosaic.mosaic_frequency, - mixup_frequency=params.parser.mosaic.mixup_frequency, - jitter=params.parser.mosaic.jitter, - mosaic_center=params.parser.mosaic.mosaic_center, - mosaic_crop_mode=params.parser.mosaic.mosaic_crop_mode, - aug_scale_min=params.parser.mosaic.aug_scale_min, - aug_scale_max=params.parser.mosaic.aug_scale_max, - **base_config) - - # init Parser - parser = yolo_input.Parser( - output_size=model.input_size, - anchors=anchor_dict, - use_tie_breaker=params.parser.use_tie_breaker, - jitter=params.parser.jitter, - aug_scale_min=params.parser.aug_scale_min, - aug_scale_max=params.parser.aug_scale_max, - aug_rand_hue=params.parser.aug_rand_hue, - aug_rand_saturation=params.parser.aug_rand_saturation, - aug_rand_brightness=params.parser.aug_rand_brightness, - max_num_instances=params.parser.max_num_instances, - scale_xy=model.detection_generator.scale_xy.get(), - expanded_strides=model.detection_generator.path_scales.get(), - darknet=model.darknet_based_model, - best_match_only=params.parser.best_match_only, - anchor_t=params.parser.anchor_thresh, - random_pad=params.parser.random_pad, - level_limits=level_limits, - dtype=params.dtype, - **base_config) - - # init the dataset reader - reader = input_reader.InputReader( - params, - dataset_fn=tf.data.TFRecordDataset, - decoder_fn=decoder.decode, - sample_fn=sample_fn.mosaic_fn(is_training=params.is_training), - parser_fn=parser.parse_fn(params.is_training)) - dataset = reader.read(input_context=input_context) - return dataset - - def build_metrics(self, training=True): - """Build detection metrics.""" - metrics = [] - - backbone = self.task_config.model.backbone.get() - metric_names = collections.defaultdict(list) - for key in range(backbone.min_level, backbone.max_level + 1): - key = str(key) - metric_names[key].append('loss') - metric_names[key].append('avg_iou') - metric_names[key].append('avg_obj') - - metric_names['net'].append('box') - metric_names['net'].append('class') - metric_names['net'].append('conf') - - for _, key in enumerate(metric_names.keys()): - metrics.append(task_utils.ListMetrics(metric_names[key], name=key)) - - self._metrics = metrics - if not training: - annotation_file = self.task_config.annotation_file - if self._coco_91_to_80: - annotation_file = None - self.coco_metric = coco_evaluator.COCOEvaluator( - annotation_file=annotation_file, - include_mask=False, - need_rescale_bboxes=False, - per_category_metrics=self._task_config.per_category_metrics) - - return metrics - - def build_losses(self, outputs, labels, aux_losses=None): - """Build YOLOX losses.""" - return self._loss_fn(labels, outputs) - - def train_step(self, inputs, model, optimizer, metrics=None): - """Train Step. - - Forward step and backwards propagate the model. - - Args: - inputs: a dictionary of input tensors. - model: the model, forward pass definition. - optimizer: the optimizer for this training step. - metrics: a nested structure of metrics objects. - - Returns: - A dictionary of logs. - """ - image, label = inputs - - with tf.GradientTape(persistent=False) as tape: - # Compute a prediction - y_pred = model(image, training=True) - - # Cast to float32 for gradietn computation - y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred) - - # Get the total loss - (scaled_loss, metric_loss, - loss_metrics) = self.build_losses(y_pred['raw_output'], label) - - # Scale the loss for numerical stability - if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer): - scaled_loss = optimizer.get_scaled_loss(scaled_loss) - - # Compute the gradient - train_vars = model.trainable_variables - gradients = tape.gradient(scaled_loss, train_vars) - - # Get unscaled loss if we are using the loss scale optimizer on fp16 - if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer): - gradients = optimizer.get_unscaled_gradients(gradients) - - # Apply gradients to the model - optimizer.apply_gradients(zip(gradients, train_vars)) - logs = {self.loss: metric_loss} - - # Compute all metrics - if metrics: - for m in metrics: - m.update_state(loss_metrics[m.name]) - logs.update({m.name: m.result()}) - return logs - - def _reorg_boxes(self, boxes, info, num_detections): - """Scale and Clean boxes prior to Evaluation.""" - mask = tf.sequence_mask(num_detections, maxlen=tf.shape(boxes)[1]) - mask = tf.cast(tf.expand_dims(mask, axis=-1), boxes.dtype) - - # Denormalize the boxes by the shape of the image - inshape = tf.expand_dims(info[:, 1, :], axis=1) - ogshape = tf.expand_dims(info[:, 0, :], axis=1) - scale = tf.expand_dims(info[:, 2, :], axis=1) - offset = tf.expand_dims(info[:, 3, :], axis=1) - - boxes = box_ops.denormalize_boxes(boxes, inshape) - boxes = box_ops.clip_boxes(boxes, inshape) - boxes += tf.tile(offset, [1, 1, 2]) - boxes /= tf.tile(scale, [1, 1, 2]) - boxes = box_ops.clip_boxes(boxes, ogshape) - - # Mask the boxes for usage - boxes *= mask - boxes += (mask - 1) - return boxes - - def validation_step(self, inputs, model, metrics=None): - """Validatation step. - - Args: - inputs: a dictionary of input tensors. - model: the keras.Model. - metrics: a nested structure of metrics objects. - - Returns: - A dictionary of logs. - """ - image, label = inputs - - # Step the model once - y_pred = model(image, training=False) - y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred) - (_, metric_loss, loss_metrics) = self.build_losses(y_pred['raw_output'], - label) - logs = {self.loss: metric_loss} - - # Reorganize and rescale the boxes - info = label['groundtruths']['image_info'] - boxes = self._reorg_boxes(y_pred['bbox'], info, y_pred["num_detections"]) - - # Build the input for the coc evaluation metric - coco_model_outputs = { - 'detection_boxes': boxes, - 'detection_scores': y_pred['confidence'], - 'detection_classes': y_pred['classes'], - 'num_detections': y_pred['num_detections'], - 'source_id': label['groundtruths']['source_id'], - 'image_info': label['groundtruths']['image_info'] - } - - # Compute all metrics - if metrics: - logs.update( - {self.coco_metric.name: (label['groundtruths'], coco_model_outputs)}) - for m in metrics: - m.update_state(loss_metrics[m.name]) - logs.update({m.name: m.result()}) - return logs - - def aggregate_logs(self, state=None, step_outputs=None): - """Get Metric Results.""" - if not state: - self.coco_metric.reset_states() - state = self.coco_metric - self.coco_metric.update_state(step_outputs[self.coco_metric.name][0], - step_outputs[self.coco_metric.name][1]) - return state - - def reduce_aggregated_logs(self, aggregated_logs, global_step=None): - """Reduce logs and remove unneeded items. Update with COCO results.""" - res = self.coco_metric.result() - return res - - def initialize(self, model: tf.keras.Model): - """Loading pretrained checkpoint.""" - - if not self.task_config.init_checkpoint: - logging.info('Training from Scratch.') - return - - ckpt_dir_or_file = self.task_config.init_checkpoint - if tf.io.gfile.isdir(ckpt_dir_or_file): - ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file) - - # Restoring checkpoint. - if self.task_config.init_checkpoint_modules == 'all': - ckpt = tf.train.Checkpoint(**model.checkpoint_items) - status = ckpt.read(ckpt_dir_or_file) - status.expect_partial().assert_existing_objects_matched() - else: - ckpt_items = {} - if 'backbone' in self.task_config.init_checkpoint_modules: - ckpt_items.update(backbone=model.backbone) - if 'decoder' in self.task_config.init_checkpoint_modules: - ckpt_items.update(decoder=model.decoder) - - ckpt = tf.train.Checkpoint(**ckpt_items) - status = ckpt.read(ckpt_dir_or_file) - status.expect_partial().assert_existing_objects_matched() - - logging.info('Finished loading pretrained checkpoint from %s', - ckpt_dir_or_file) - - def create_optimizer(self, - optimizer_config: OptimizationConfig, - runtime_config: Optional[RuntimeConfig] = None): - """Creates an TF optimizer from configurations. - - Args: - optimizer_config: the parameters of the Optimization settings. - runtime_config: the parameters of the runtime. - - Returns: - A tf.optimizers.Optimizer object. - """ - opt_factory = optimization.YoloOptimizerFactory(optimizer_config) - # pylint: disable=protected-access - ema = opt_factory._use_ema - opt_factory._use_ema = False - - opt_type = opt_factory._optimizer_type - if opt_type == 'sgd_torch': - optimizer = opt_factory.build_optimizer(opt_factory.build_learning_rate()) - optimizer.set_bias_lr( - opt_factory.get_bias_lr_schedule(self._task_config.smart_bias_lr)) - optimizer.search_and_set_variable_groups(self._model.trainable_variables) - else: - optimizer = opt_factory.build_optimizer(opt_factory.build_learning_rate()) - opt_factory._use_ema = ema - - if ema: - logging.info('EMA is enabled.') - optimizer = opt_factory.add_ema(optimizer) - - # pylint: enable=protected-access - - if runtime_config and runtime_config.loss_scale: - use_float16 = runtime_config.mixed_precision_dtype == 'float16' - optimizer = performance.configure_optimizer( - optimizer, - use_graph_rewrite=False, - use_float16=use_float16, - loss_scale=runtime_config.loss_scale) - - return optimizer From 9a53f00648f258e843b83ecb577665387b123ef6 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Mon, 24 Jan 2022 15:40:35 -0500 Subject: [PATCH 63/96] solve conflicts --- official/vision/beta/projects/yolo/common/registry_imports.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/common/registry_imports.py b/official/vision/beta/projects/yolo/common/registry_imports.py index 8bbf153d7eb..56349093624 100644 --- a/official/vision/beta/projects/yolo/common/registry_imports.py +++ b/official/vision/beta/projects/yolo/common/registry_imports.py @@ -20,7 +20,6 @@ # import configs from official.vision.beta.projects.yolo.configs import darknet_classification from official.vision.beta.projects.yolo.configs import yolo as yolo_config -from official.vision.beta.projects.yolo.configs import yolox as yolox_config # import modeling components from official.vision.beta.projects.yolo.modeling.backbones import darknet from official.vision.beta.projects.yolo.modeling.decoders import yolo_decoder @@ -31,4 +30,3 @@ # import tasks from official.vision.beta.projects.yolo.tasks import image_classification from official.vision.beta.projects.yolo.tasks import yolo as yolo_task -from official.vision.beta.projects.yolo.tasks import yolox as yolox_task From 40fa116a6c9ba28ee83441610ecdc4b5f085fe34 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Mon, 24 Jan 2022 15:45:47 -0500 Subject: [PATCH 64/96] Update yolox.yaml --- .../beta/projects/yolo/configs/experiments/yolox/yolox.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index e69955b2e1e..ba6b9e1df91 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -64,9 +64,9 @@ task: mixup_frequency: 1.0 mosaic_crop_mode: 'scale' mosaic_center: 0.25 - aug_scale_min: 0.1 - aug_scale_max: 1.9 - aug_rand_angle: 10 +# aug_scale_min: 0.1 +# aug_scale_max: 1.9 +# aug_rand_angle: 10 max_num_instances: 300 letter_box: true From d191638e76f0c4fc26c9470e4329082b0b000a5c Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Mon, 24 Jan 2022 15:53:29 -0500 Subject: [PATCH 65/96] solve conflicts --- official/vision/beta/projects/yolo/configs/yolo.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index a2bae1c3474..41a5213bef6 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -270,7 +270,18 @@ def yolo() -> cfg.ExperimentConfig: 'task.train_data.is_training != None', 'task.validation_data.is_training != None' ]) + +@exp_factory.register_config_factory('yolox') +def yolox() -> cfg.ExperimentConfig: + """Yolox general config.""" + return cfg.ExperimentConfig( + task=YoloTask(), + restrictions=[ + 'task.train_data.is_training != None', + 'task.validation_data.is_training != None' + ]) + @exp_factory.register_config_factory('yolo_darknet') def yolo_darknet() -> cfg.ExperimentConfig: From 5bb6271bb4e58d51fcd4dd7610c0bcc37cd4ec44 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Mon, 24 Jan 2022 17:07:16 -0500 Subject: [PATCH 66/96] yolov4 test --- .../experiments/yolox/yolov4_512_tpu.yaml | 138 ++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 official/vision/beta/projects/yolo/configs/experiments/yolox/yolov4_512_tpu.yaml diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolov4_512_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolov4_512_tpu.yaml new file mode 100644 index 00000000000..606ac73f8ba --- /dev/null +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolov4_512_tpu.yaml @@ -0,0 +1,138 @@ +# --experiment_type=yolo_darknet +# mAP 43.0 +runtime: + distribution_strategy: 'tpu' + mixed_precision_dtype: 'bfloat16' +task: + smart_bias_lr: 0.0 + model: + darknet_based_model: true + input_size: [512, 512, 3] + backbone: + type: 'darknet' + darknet: + model_id: 'cspdarknet53' + max_level: 5 + min_level: 3 + decoder: + type: yolo_decoder + yolo_decoder: + version: v4 + type: regular + activation: leaky + head: + smart_bias: true + detection_generator: + box_type: + 'all': original + scale_xy: + '5': 1.05 + '4': 1.1 + '3': 1.2 + max_boxes: 200 + nms_type: iou + iou_thresh: 0.001 + nms_thresh: 0.60 + loss: + use_scaled_loss: false + box_loss_type: + 'all': ciou + ignore_thresh: + 'all': 0.7 + iou_normalizer: + 'all': 0.07 + cls_normalizer: + 'all': 1.0 + object_normalizer: + 'all': 1.0 + objectness_smooth: + 'all': 0.0 + max_delta: + 'all': 5.0 + norm_activation: + activation: mish + norm_epsilon: 0.0001 + norm_momentum: 0.99 + use_sync_bn: true + num_classes: 80 + anchor_boxes: + anchors_per_scale: 3 + boxes: [box: [12, 16], box: [19, 36], box: [40, 28], + box: [36, 75], box: [76, 55], box: [72, 146], + box: [142, 110], box: [192, 243], box: [459, 401]] + train_data: + global_batch_size: 64 + dtype: float32 + input_path: 'gs://cam2-datasets/coco/train*' + is_training: true + drop_remainder: true + seed: 1000 + parser: + mosaic: + mosaic_frequency: 0.75 + mixup_frequency: 0.0 + mosaic_crop_mode: 'crop' + mosaic_center: 0.2 + aug_scale_min: 0.2 + aug_scale_max: 1.6 + jitter: 0.3 + max_num_instances: 200 + letter_box: false + random_flip: true + aug_rand_saturation: 1.5 + aug_rand_brightness: 1.5 + aug_rand_hue: 0.1 + aug_scale_min: 0.1 + aug_scale_max: 1.9 + aug_rand_translate: 0.0 + jitter: 0.3 + area_thresh: 0.1 + random_pad: true + use_tie_breaker: true + anchor_thresh: 0.4 + validation_data: + global_batch_size: 8 + dtype: float32 + input_path: 'gs://cam2-datasets/coco/val*' + is_training: false + drop_remainder: true + parser: + max_num_instances: 200 + letter_box: false + use_tie_breaker: true + anchor_thresh: 0.4 + weight_decay: 0.000 + init_checkpoint: 'gs://tf_model_garden/vision/yolo/ckpt-15000' + init_checkpoint_modules: 'backbone' + annotation_file: null +trainer: + train_steps: 555000 + validation_steps: 625 + steps_per_loop: 1850 + summary_interval: 1850 + validation_interval: 9250 + checkpoint_interval: 1850 + optimizer_config: + ema: + average_decay: 0.9998 + trainable_weights_only: false + dynamic_decay: true + learning_rate: + type: stepwise + stepwise: + boundaries: [400000] + name: PiecewiseConstantDecay + values: [0.00131, 0.000131] + optimizer: + type: sgd_torch + sgd_torch: + momentum: 0.949 + momentum_start: 0.949 + nesterov: true + warmup_steps: 1000 + weight_decay: 0.0005 + name: SGD + warmup: + type: 'linear' + linear: + warmup_steps: 1000 # learning rate rises from 0 to 0.0013 over 1000 steps From 62a9b682e57876cf6973be2798ab3c6360bc511a Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Mon, 24 Jan 2022 17:33:36 -0500 Subject: [PATCH 67/96] solve conflicts --- .../vision/beta/projects/yolo/configs/yolo.py | 52 ++++++++++++++----- .../beta/projects/yolo/modeling/factory.py | 4 +- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index 41a5213bef6..aecb533bff9 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -240,6 +240,28 @@ class Yolo(hyperparams.Config): darknet_based_model: bool = False +@dataclasses.dataclass +class YoloX(hyperparams.Config): + input_size: Optional[List[int]] = dataclasses.field( + default_factory=lambda: [640, 640, 3]) + backbone: backbones.Backbone = backbones.Backbone( + type='darknet', darknet=backbones.Darknet(model_id='darknet53')) + decoder: decoders.Decoder = decoders.Decoder( + type='yolo_decoder', + yolo_decoder=decoders.YoloDecoder(version='vx, type='regular')) + head: YoloHead = YoloXHead() + detection_generator: YoloDetectionGenerator = YoloDetectionGenerator() + loss: YoloLoss = YoloLoss() + norm_activation: common.NormActivation = common.NormActivation( + activation='mish', + use_sync_bn=True, + norm_momentum=0.99, + norm_epsilon=0.001) + num_classes: int = 80 + anchor_boxes: AnchorBoxes = AnchorBoxes() + darknet_based_model: bool = False + + @dataclasses.dataclass class YoloTask(cfg.TaskConfig): per_category_metrics: bool = False @@ -255,6 +277,21 @@ class YoloTask(cfg.TaskConfig): gradient_clip_norm: float = 0.0 seed = GLOBAL_SEED +@dataclasses.dataclass +class YoloXTask(cfg.TaskConfig): + per_category_metrics: bool = False + smart_bias_lr: float = 0.0 + model: YoloX = YoloX() + train_data: DataConfig = DataConfig(is_training=True) + validation_data: DataConfig = DataConfig(is_training=False) + weight_decay: float = 0.0 + annotation_file: Optional[str] = None + init_checkpoint: Optional[str] = None + init_checkpoint_modules: Union[ + str, List[str]] = 'all' # all, backbone, and/or decoder + gradient_clip_norm: float = 0.0 + seed = GLOBAL_SEED + COCO_INPUT_PATH_BASE = 'coco' COCO_TRAIN_EXAMPLES = 118287 @@ -271,17 +308,6 @@ def yolo() -> cfg.ExperimentConfig: 'task.validation_data.is_training != None' ]) - -@exp_factory.register_config_factory('yolox') -def yolox() -> cfg.ExperimentConfig: - """Yolox general config.""" - return cfg.ExperimentConfig( - task=YoloTask(), - restrictions=[ - 'task.train_data.is_training != None', - 'task.validation_data.is_training != None' - ]) - @exp_factory.register_config_factory('yolo_darknet') def yolo_darknet() -> cfg.ExperimentConfig: @@ -761,13 +787,13 @@ def yolox_regular() -> cfg.ExperimentConfig: max_num_instances = 200 config = cfg.ExperimentConfig( runtime=cfg.RuntimeConfig(mixed_precision_dtype='bfloat16'), - task=YoloTask( + task=YoloXTask( smart_bias_lr=0.1, init_checkpoint='', init_checkpoint_modules='backbone', annotation_file=None, weight_decay=0.0, - model=Yolo( + model=YoloX( darknet_based_model=True, norm_activation=common.NormActivation(use_sync_bn=True), head=YOLOXHead(smart_bias=True), diff --git a/official/vision/beta/projects/yolo/modeling/factory.py b/official/vision/beta/projects/yolo/modeling/factory.py index 947df65f28c..d8a0420f000 100644 --- a/official/vision/beta/projects/yolo/modeling/factory.py +++ b/official/vision/beta/projects/yolo/modeling/factory.py @@ -58,7 +58,9 @@ def build_yolo_head(input_specs, model_config: yolo.Yolo, l2_regularization): min_level = min(map(int, input_specs.keys())) max_level = max(map(int, input_specs.keys())) - if model_config == yolo.yolox: + print(type(model_config)) + + if model_config is yolo.Yolox: head = yolox_head.YOLOXHead( min_level=min_level, max_level=max_level, From d75000d9253ecc5b562457ff7e986b785f2f2dff Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Mon, 24 Jan 2022 17:35:48 -0500 Subject: [PATCH 68/96] solve conflicts --- official/vision/beta/projects/yolo/configs/yolo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index aecb533bff9..9f9b690f116 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -248,7 +248,7 @@ class YoloX(hyperparams.Config): type='darknet', darknet=backbones.Darknet(model_id='darknet53')) decoder: decoders.Decoder = decoders.Decoder( type='yolo_decoder', - yolo_decoder=decoders.YoloDecoder(version='vx, type='regular')) + yolo_decoder=decoders.YoloDecoder(version='vx', type='regular')) head: YoloHead = YoloXHead() detection_generator: YoloDetectionGenerator = YoloDetectionGenerator() loss: YoloLoss = YoloLoss() From 1d1585bd5508927cf71fa62558a3210cc5c0bc0a Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Mon, 24 Jan 2022 17:36:58 -0500 Subject: [PATCH 69/96] solve conflicts --- official/vision/beta/projects/yolo/configs/yolo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index 9f9b690f116..c9eeb850908 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -136,7 +136,7 @@ class YoloHead(hyperparams.Config): @dataclasses.dataclass -class YOLOXHead(hyperparams.Config): +class YoloXHead(hyperparams.Config): """Parameterization for the YOLOX Head.""" width: float = 1.0 depthwise: bool = False From 842b27ce2a32d8aafe36e5a066e2c53246d6c0d7 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Mon, 24 Jan 2022 17:37:23 -0500 Subject: [PATCH 70/96] solve conflicts --- official/vision/beta/projects/yolo/configs/yolo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index c9eeb850908..7272c971e72 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -796,7 +796,7 @@ def yolox_regular() -> cfg.ExperimentConfig: model=YoloX( darknet_based_model=True, norm_activation=common.NormActivation(use_sync_bn=True), - head=YOLOXHead(smart_bias=True), + head=YoloXHead(smart_bias=True), loss=YoloLoss(use_scaled_loss=False, update_on_repeat=True), anchor_boxes=AnchorBoxes( anchors_per_scale=3, From ee9939a7e83f0f2590b663eab12a949d44850d02 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Mon, 24 Jan 2022 17:40:04 -0500 Subject: [PATCH 71/96] solve conflicts --- .../vision/beta/projects/yolo/configs/yolo.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index 7272c971e72..fe1ba48d215 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -277,21 +277,6 @@ class YoloTask(cfg.TaskConfig): gradient_clip_norm: float = 0.0 seed = GLOBAL_SEED -@dataclasses.dataclass -class YoloXTask(cfg.TaskConfig): - per_category_metrics: bool = False - smart_bias_lr: float = 0.0 - model: YoloX = YoloX() - train_data: DataConfig = DataConfig(is_training=True) - validation_data: DataConfig = DataConfig(is_training=False) - weight_decay: float = 0.0 - annotation_file: Optional[str] = None - init_checkpoint: Optional[str] = None - init_checkpoint_modules: Union[ - str, List[str]] = 'all' # all, backbone, and/or decoder - gradient_clip_norm: float = 0.0 - seed = GLOBAL_SEED - COCO_INPUT_PATH_BASE = 'coco' COCO_TRAIN_EXAMPLES = 118287 @@ -787,7 +772,7 @@ def yolox_regular() -> cfg.ExperimentConfig: max_num_instances = 200 config = cfg.ExperimentConfig( runtime=cfg.RuntimeConfig(mixed_precision_dtype='bfloat16'), - task=YoloXTask( + task=YoloTask( smart_bias_lr=0.1, init_checkpoint='', init_checkpoint_modules='backbone', From 1519bf3945facdbcd39a069eb5a5a7dd6a148594 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Mon, 24 Jan 2022 17:42:35 -0500 Subject: [PATCH 72/96] solve conflicts --- official/vision/beta/projects/yolo/modeling/factory.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/factory.py b/official/vision/beta/projects/yolo/modeling/factory.py index d8a0420f000..af0be138e57 100644 --- a/official/vision/beta/projects/yolo/modeling/factory.py +++ b/official/vision/beta/projects/yolo/modeling/factory.py @@ -58,9 +58,9 @@ def build_yolo_head(input_specs, model_config: yolo.Yolo, l2_regularization): min_level = min(map(int, input_specs.keys())) max_level = max(map(int, input_specs.keys())) - print(type(model_config)) - - if model_config is yolo.Yolox: + print(type(model_config.as_dict())) + + if isinstance(model_config, yolo.Yolox): head = yolox_head.YOLOXHead( min_level=min_level, max_level=max_level, From 69cf6c0348ea015acbbc3261bac08e96edbf1117 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Mon, 24 Jan 2022 17:43:50 -0500 Subject: [PATCH 73/96] solve conflicts --- official/vision/beta/projects/yolo/modeling/factory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/factory.py b/official/vision/beta/projects/yolo/modeling/factory.py index af0be138e57..093c51eb0d8 100644 --- a/official/vision/beta/projects/yolo/modeling/factory.py +++ b/official/vision/beta/projects/yolo/modeling/factory.py @@ -58,9 +58,9 @@ def build_yolo_head(input_specs, model_config: yolo.Yolo, l2_regularization): min_level = min(map(int, input_specs.keys())) max_level = max(map(int, input_specs.keys())) - print(type(model_config.as_dict())) + print(model_config.as_dict()) - if isinstance(model_config, yolo.Yolox): + if isinstance(model_config, yolo.YoloX): head = yolox_head.YOLOXHead( min_level=min_level, max_level=max_level, From e64372c08f5caa8522b7fccf818afed01865d073 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang Date: Mon, 24 Jan 2022 17:49:54 -0500 Subject: [PATCH 74/96] solve conflicts --- official/vision/beta/projects/yolo/configs/yolo.py | 10 +++++----- official/vision/beta/projects/yolo/modeling/factory.py | 6 ++---- .../beta/projects/yolo/modeling/heads/yolox_head.py | 2 +- .../projects/yolo/modeling/heads/yolox_head_test.py | 6 +++--- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index fe1ba48d215..cb999093cb1 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -136,7 +136,7 @@ class YoloHead(hyperparams.Config): @dataclasses.dataclass -class YoloXHead(hyperparams.Config): +class YoloxHead(hyperparams.Config): """Parameterization for the YOLOX Head.""" width: float = 1.0 depthwise: bool = False @@ -241,7 +241,7 @@ class Yolo(hyperparams.Config): @dataclasses.dataclass -class YoloX(hyperparams.Config): +class Yolox(hyperparams.Config): input_size: Optional[List[int]] = dataclasses.field( default_factory=lambda: [640, 640, 3]) backbone: backbones.Backbone = backbones.Backbone( @@ -249,7 +249,7 @@ class YoloX(hyperparams.Config): decoder: decoders.Decoder = decoders.Decoder( type='yolo_decoder', yolo_decoder=decoders.YoloDecoder(version='vx', type='regular')) - head: YoloHead = YoloXHead() + head: YoloHead = YoloxHead() detection_generator: YoloDetectionGenerator = YoloDetectionGenerator() loss: YoloLoss = YoloLoss() norm_activation: common.NormActivation = common.NormActivation( @@ -778,10 +778,10 @@ def yolox_regular() -> cfg.ExperimentConfig: init_checkpoint_modules='backbone', annotation_file=None, weight_decay=0.0, - model=YoloX( + model=Yolox( darknet_based_model=True, norm_activation=common.NormActivation(use_sync_bn=True), - head=YoloXHead(smart_bias=True), + head=YoloxHead(smart_bias=True), loss=YoloLoss(use_scaled_loss=False, update_on_repeat=True), anchor_boxes=AnchorBoxes( anchors_per_scale=3, diff --git a/official/vision/beta/projects/yolo/modeling/factory.py b/official/vision/beta/projects/yolo/modeling/factory.py index 093c51eb0d8..e6e81719dd1 100644 --- a/official/vision/beta/projects/yolo/modeling/factory.py +++ b/official/vision/beta/projects/yolo/modeling/factory.py @@ -58,10 +58,8 @@ def build_yolo_head(input_specs, model_config: yolo.Yolo, l2_regularization): min_level = min(map(int, input_specs.keys())) max_level = max(map(int, input_specs.keys())) - print(model_config.as_dict()) - - if isinstance(model_config, yolo.YoloX): - head = yolox_head.YOLOXHead( + if isinstance(model_config, yolo.Yolox): + head = yolox_head.YoloxHead( min_level=min_level, max_level=max_level, classes=model_config.num_classes, diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index 0b2ef1bb723..367563c3663 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -21,7 +21,7 @@ from official.vision.beta.projects.yolo.ops import box_ops -class YOLOXHead(tf.keras.layers.Layer): +class YoloxHead(tf.keras.layers.Layer): """YOLOX Prediction Head.""" def __init__( diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py index 0881388b49c..6f2068e7141 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head_test.py @@ -35,7 +35,7 @@ def test_network_creation(self): } classes = 100 bps = 3 - head = heads.YOLOXHead(3, 5, classes=classes, boxes_per_level=bps) + head = heads.YoloxHead(3, 5, classes=classes, boxes_per_level=bps) inputs = {} for key in input_shape: @@ -59,7 +59,7 @@ def test_serialize_deserialize(self): } classes = 100 bps = 3 - head = heads.YOLOXHead(3, 5, classes=classes, boxes_per_level=bps) + head = heads.YoloxHead(3, 5, classes=classes, boxes_per_level=bps) inputs = {} for key in input_shape: @@ -67,7 +67,7 @@ def test_serialize_deserialize(self): _ = head(inputs) configs = head.get_config() - head_from_config = heads.YOLOXHead.from_config(configs) + head_from_config = heads.YoloxHead.from_config(configs) self.assertAllEqual(head.get_config(), head_from_config.get_config()) From 985c34433c07e6e3f25a7b208c7f60a0a7263898 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Mon, 24 Jan 2022 18:01:57 -0500 Subject: [PATCH 75/96] Update yolov4_512_tpu.yaml --- .../yolo/configs/experiments/yolox/yolov4_512_tpu.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolov4_512_tpu.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolov4_512_tpu.yaml index 606ac73f8ba..e918d34581b 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolov4_512_tpu.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolov4_512_tpu.yaml @@ -102,9 +102,6 @@ task: use_tie_breaker: true anchor_thresh: 0.4 weight_decay: 0.000 - init_checkpoint: 'gs://tf_model_garden/vision/yolo/ckpt-15000' - init_checkpoint_modules: 'backbone' - annotation_file: null trainer: train_steps: 555000 validation_steps: 625 From 56d08e90c9c63c4e62cb9fb63ba481ed5d6bf7fe Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Wed, 26 Jan 2022 17:51:03 -0500 Subject: [PATCH 76/96] Update yolox_head.py --- .../beta/projects/yolo/modeling/heads/yolox_head.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index 367563c3663..577dcf50cae 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -208,12 +208,13 @@ def call(self, inputs, *args, **kwargs): reg_output = self._reg_head[k](inputs[k]) obj_output = self._obj_head[k](inputs[k]) - for b in range(self._boxes_per_level): - ordered_preds.append(reg_output[:,:,:,4 * b: 4 * (b + 1)]) - ordered_preds.append(obj_output[:,:,:,b: b + 1]) - ordered_preds.append(cls_output[:,:,:,self._classes * b: self._classes * (b + 1)]) + #for b in range(self._boxes_per_level): + # ordered_preds.append(reg_output[:,:,:,4 * b: 4 * (b + 1)]) + # ordered_preds.append(obj_output[:,:,:,b: b + 1]) + # ordered_preds.append(cls_output[:,:,:,self._classes * b: self._classes * (b + 1)]) - output = tf.concat(ordered_preds, axis=-1) + #output = tf.concat(ordered_preds, axis=-1) + output = tf.concat([cls_output, reg_output, obj_output], axis = -1) outputs[k] = output #Outputs are not flattened here. return outputs From 1d7372299791633ed0770fe55353b3a2217c641c Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Wed, 26 Jan 2022 18:16:21 -0500 Subject: [PATCH 77/96] Update yolox_head.py --- .../beta/projects/yolo/modeling/heads/yolox_head.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index 577dcf50cae..367563c3663 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -208,13 +208,12 @@ def call(self, inputs, *args, **kwargs): reg_output = self._reg_head[k](inputs[k]) obj_output = self._obj_head[k](inputs[k]) - #for b in range(self._boxes_per_level): - # ordered_preds.append(reg_output[:,:,:,4 * b: 4 * (b + 1)]) - # ordered_preds.append(obj_output[:,:,:,b: b + 1]) - # ordered_preds.append(cls_output[:,:,:,self._classes * b: self._classes * (b + 1)]) + for b in range(self._boxes_per_level): + ordered_preds.append(reg_output[:,:,:,4 * b: 4 * (b + 1)]) + ordered_preds.append(obj_output[:,:,:,b: b + 1]) + ordered_preds.append(cls_output[:,:,:,self._classes * b: self._classes * (b + 1)]) - #output = tf.concat(ordered_preds, axis=-1) - output = tf.concat([cls_output, reg_output, obj_output], axis = -1) + output = tf.concat(ordered_preds, axis=-1) outputs[k] = output #Outputs are not flattened here. return outputs From b14404f790ed7a6be652ae539e2ad8387a177b53 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Wed, 26 Jan 2022 18:21:59 -0500 Subject: [PATCH 78/96] Update yolo.py --- .../vision/beta/projects/yolo/configs/yolo.py | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index cb999093cb1..f283242a544 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -157,6 +157,21 @@ class YoloDetectionGenerator(hyperparams.Config): nms_thresh: float = 0.6 max_boxes: int = 200 pre_nms_points: int = 5000 + + +@dataclasses.dataclass +class YoloxDetectionGenerator(hyperparams.Config): + box_type: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 'original')) + scale_xy: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) + path_scales: FPNConfig = dataclasses.field( + default_factory=_build_path_scales(MIN_LEVEL, MAX_LEVEL)) + nms_type: str = 'greedy' + iou_thresh: float = 0.001 + nms_thresh: float = 0.65 + max_boxes: int = 200 + pre_nms_points: int = 5000 @dataclasses.dataclass @@ -180,7 +195,30 @@ class YoloLoss(hyperparams.Config): label_smoothing: float = 0.0 use_scaled_loss: bool = True update_on_repeat: bool = True + +@dataclasses.dataclass +class YoloxLoss(hyperparams.Config): + ignore_thresh: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 0.0)) + truth_thresh: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) + box_loss_type: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 'iou')) + iou_normalizer: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 5.0)) + cls_normalizer: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) + object_normalizer: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) + max_delta: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, np.inf)) + objectness_smooth: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 0.0)) + label_smoothing: float = 0.0 + use_scaled_loss: bool = True + update_on_repeat: bool = True + @dataclasses.dataclass class Box(hyperparams.Config): @@ -249,9 +287,9 @@ class Yolox(hyperparams.Config): decoder: decoders.Decoder = decoders.Decoder( type='yolo_decoder', yolo_decoder=decoders.YoloDecoder(version='vx', type='regular')) - head: YoloHead = YoloxHead() - detection_generator: YoloDetectionGenerator = YoloDetectionGenerator() - loss: YoloLoss = YoloLoss() + head: YoloxHead = YoloxHead() + detection_generator: YoloxDetectionGenerator = YoloxDetectionGenerator() + loss: YoloxLoss = YoloxLoss() norm_activation: common.NormActivation = common.NormActivation( activation='mish', use_sync_bn=True, @@ -782,7 +820,9 @@ def yolox_regular() -> cfg.ExperimentConfig: darknet_based_model=True, norm_activation=common.NormActivation(use_sync_bn=True), head=YoloxHead(smart_bias=True), - loss=YoloLoss(use_scaled_loss=False, update_on_repeat=True), + loss=YoloxLoss( + use_scaled_loss=True, + update_on_repeat=True), anchor_boxes=AnchorBoxes( anchors_per_scale=3, boxes=[ From c9e3606468f396e149dc39f6445d7caa28558690 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Wed, 26 Jan 2022 18:24:54 -0500 Subject: [PATCH 79/96] Update yolox.yaml --- .../projects/yolo/configs/experiments/yolox/yolox.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index ba6b9e1df91..6a167b394f2 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -33,17 +33,15 @@ task: use_scaled_loss: true update_on_repeat: true box_loss_type: - 'all': ciou + 'all': iou ignore_thresh: 'all': 0.0 iou_normalizer: - 'all': 0.05 + 'all': 5.0 cls_normalizer: - 'all': 0.3 + 'all': 1.0 object_normalizer: - '5': 0.28 - '4': 0.70 - '3': 2.80 + 'all': 1.0 objectness_smooth: 'all': 1.0 norm_activation: From 251d3af440f126532934c5aec4e08f0a93e3d88b Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Wed, 26 Jan 2022 18:25:04 -0500 Subject: [PATCH 80/96] Create yolox.yaml From 09a066956d8a8c000d4738b74dfe1befb968dd7d Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Wed, 26 Jan 2022 18:27:17 -0500 Subject: [PATCH 81/96] Update yolo.py --- .../vision/beta/projects/yolo/configs/yolo.py | 48 ++----------------- 1 file changed, 4 insertions(+), 44 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index f283242a544..5f731b648cf 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -157,21 +157,6 @@ class YoloDetectionGenerator(hyperparams.Config): nms_thresh: float = 0.6 max_boxes: int = 200 pre_nms_points: int = 5000 - - -@dataclasses.dataclass -class YoloxDetectionGenerator(hyperparams.Config): - box_type: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 'original')) - scale_xy: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) - path_scales: FPNConfig = dataclasses.field( - default_factory=_build_path_scales(MIN_LEVEL, MAX_LEVEL)) - nms_type: str = 'greedy' - iou_thresh: float = 0.001 - nms_thresh: float = 0.65 - max_boxes: int = 200 - pre_nms_points: int = 5000 @dataclasses.dataclass @@ -195,30 +180,7 @@ class YoloLoss(hyperparams.Config): label_smoothing: float = 0.0 use_scaled_loss: bool = True update_on_repeat: bool = True - - -@dataclasses.dataclass -class YoloxLoss(hyperparams.Config): - ignore_thresh: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 0.0)) - truth_thresh: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) - box_loss_type: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 'iou')) - iou_normalizer: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 5.0)) - cls_normalizer: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) - object_normalizer: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) - max_delta: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, np.inf)) - objectness_smooth: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 0.0)) - label_smoothing: float = 0.0 - use_scaled_loss: bool = True - update_on_repeat: bool = True - + @dataclasses.dataclass class Box(hyperparams.Config): @@ -288,8 +250,8 @@ class Yolox(hyperparams.Config): type='yolo_decoder', yolo_decoder=decoders.YoloDecoder(version='vx', type='regular')) head: YoloxHead = YoloxHead() - detection_generator: YoloxDetectionGenerator = YoloxDetectionGenerator() - loss: YoloxLoss = YoloxLoss() + detection_generator: YoloDetectionGenerator = YoloDetectionGenerator() + loss: YoloLoss = YoloLoss() norm_activation: common.NormActivation = common.NormActivation( activation='mish', use_sync_bn=True, @@ -820,9 +782,7 @@ def yolox_regular() -> cfg.ExperimentConfig: darknet_based_model=True, norm_activation=common.NormActivation(use_sync_bn=True), head=YoloxHead(smart_bias=True), - loss=YoloxLoss( - use_scaled_loss=True, - update_on_repeat=True), + loss=YoloLoss(use_scaled_loss=True, update_on_repeat=True), anchor_boxes=AnchorBoxes( anchors_per_scale=3, boxes=[ From 09f9961c615a8c285f0b3315bd70b7028455b385 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Thu, 27 Jan 2022 12:03:49 -0500 Subject: [PATCH 82/96] Update loss --- .../vision/beta/projects/yolo/configs/yolo.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index 5f731b648cf..94129ee72fe 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -182,6 +182,29 @@ class YoloLoss(hyperparams.Config): update_on_repeat: bool = True +@dataclasses.dataclass +class YoloxLoss(hyperparams.Config): + ignore_thresh: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 0.0)) + truth_thresh: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) + box_loss_type: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 'iou')) + iou_normalizer: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) + cls_normalizer: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) + object_normalizer: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) + max_delta: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, np.inf)) + objectness_smooth: FPNConfig = dataclasses.field( + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 0.0)) + label_smoothing: float = 0.0 + use_scaled_loss: bool = True + update_on_repeat: bool = True + + @dataclasses.dataclass class Box(hyperparams.Config): box: List[int] = dataclasses.field(default=list) @@ -251,7 +274,7 @@ class Yolox(hyperparams.Config): yolo_decoder=decoders.YoloDecoder(version='vx', type='regular')) head: YoloxHead = YoloxHead() detection_generator: YoloDetectionGenerator = YoloDetectionGenerator() - loss: YoloLoss = YoloLoss() + loss: YoloxLoss = YoloxLoss() norm_activation: common.NormActivation = common.NormActivation( activation='mish', use_sync_bn=True, @@ -782,7 +805,7 @@ def yolox_regular() -> cfg.ExperimentConfig: darknet_based_model=True, norm_activation=common.NormActivation(use_sync_bn=True), head=YoloxHead(smart_bias=True), - loss=YoloLoss(use_scaled_loss=True, update_on_repeat=True), + loss=YoloxLoss(use_scaled_loss=True, update_on_repeat=True), anchor_boxes=AnchorBoxes( anchors_per_scale=3, boxes=[ From 4bc3768cffde05b173f6e50b26bb1a200bc86b1a Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Thu, 27 Jan 2022 12:24:06 -0500 Subject: [PATCH 83/96] Update IOU normalizer --- official/vision/beta/projects/yolo/configs/yolo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index 94129ee72fe..b8901c8df1b 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -191,7 +191,7 @@ class YoloxLoss(hyperparams.Config): box_loss_type: FPNConfig = dataclasses.field( default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 'iou')) iou_normalizer: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 5.0)) cls_normalizer: FPNConfig = dataclasses.field( default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) object_normalizer: FPNConfig = dataclasses.field( From 191e19f2c92757e6f732411f98136972dbd3f446 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Thu, 27 Jan 2022 16:44:58 -0500 Subject: [PATCH 84/96] Update yolo.py --- .../vision/beta/projects/yolo/configs/yolo.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/yolo.py b/official/vision/beta/projects/yolo/configs/yolo.py index b8901c8df1b..6435d09f708 100755 --- a/official/vision/beta/projects/yolo/configs/yolo.py +++ b/official/vision/beta/projects/yolo/configs/yolo.py @@ -191,7 +191,7 @@ class YoloxLoss(hyperparams.Config): box_loss_type: FPNConfig = dataclasses.field( default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 'iou')) iou_normalizer: FPNConfig = dataclasses.field( - default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 5.0)) + default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) cls_normalizer: FPNConfig = dataclasses.field( default_factory=_build_dict(MIN_LEVEL, MAX_LEVEL, 1.0)) object_normalizer: FPNConfig = dataclasses.field( @@ -274,7 +274,7 @@ class Yolox(hyperparams.Config): yolo_decoder=decoders.YoloDecoder(version='vx', type='regular')) head: YoloxHead = YoloxHead() detection_generator: YoloDetectionGenerator = YoloDetectionGenerator() - loss: YoloxLoss = YoloxLoss() + loss: YoloLoss = YoloLoss() norm_activation: common.NormActivation = common.NormActivation( activation='mish', use_sync_bn=True, @@ -805,7 +805,7 @@ def yolox_regular() -> cfg.ExperimentConfig: darknet_based_model=True, norm_activation=common.NormActivation(use_sync_bn=True), head=YoloxHead(smart_bias=True), - loss=YoloxLoss(use_scaled_loss=True, update_on_repeat=True), + loss=YoloLoss(use_scaled_loss=True, update_on_repeat=True), anchor_boxes=AnchorBoxes( anchors_per_scale=3, boxes=[ @@ -877,15 +877,11 @@ def yolox_regular() -> cfg.ExperimentConfig: } }, 'learning_rate': { - 'type': 'stepwise', - 'stepwise': { - 'boundaries': [ - 240 * steps_per_epoch - ], - 'values': [ - 0.00131 * train_batch_size / 64.0, - 0.000131 * train_batch_size / 64.0, - ] + 'type': 'cosine', + 'cosine': { + 'initial_learning_rate': 0.02, + 'alpha': 0.2, + 'decay_steps': train_epochs * steps_per_epoch, } }, 'warmup': { From 5f0125b3ed6a966f916e144fca69c21229831f85 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Thu, 27 Jan 2022 18:00:05 -0500 Subject: [PATCH 85/96] Update loss in yolox.yaml --- .../projects/yolo/configs/experiments/yolox/yolox.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 6a167b394f2..c7998bd6548 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -37,11 +37,13 @@ task: ignore_thresh: 'all': 0.0 iou_normalizer: - 'all': 5.0 + 'all': 0.05 cls_normalizer: - 'all': 1.0 + 'all': 0.3 object_normalizer: - 'all': 1.0 + '5': 0.28 + '4': 0.70 + '3': 2.80 objectness_smooth: 'all': 1.0 norm_activation: From 46c8947ed0dc825d02d91bb6b34211e5a35460c6 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Fri, 28 Jan 2022 01:24:45 -0500 Subject: [PATCH 86/96] Update yolox.yaml --- .../yolo/configs/experiments/yolox/yolox.yaml | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index c7998bd6548..42cea1bd036 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -33,7 +33,7 @@ task: use_scaled_loss: true update_on_repeat: true box_loss_type: - 'all': iou + 'all': ciou ignore_thresh: 'all': 0.0 iou_normalizer: @@ -56,18 +56,13 @@ task: input_path: 'gs://cam2-datasets/coco/train*' shuffle_buffer_size: 10000 parser: - aug_rand_saturation: 1.5 - aug_rand_brightness: 1.5 - aug_rand_hue: 0.1 mosaic: mosaic_frequency: 1.0 - mixup_frequency: 1.0 + mixup_frequency: 0.2 mosaic_crop_mode: 'scale' mosaic_center: 0.25 -# aug_scale_min: 0.1 -# aug_scale_max: 1.9 -# aug_rand_angle: 10 - + aug_scale_min: 0.1 + aug_scale_max: 1.9 max_num_instances: 300 letter_box: true random_flip: true @@ -75,15 +70,13 @@ task: area_thresh: 0.1 validation_data: input_path: 'gs://cam2-datasets/coco/val*' -trainer: - train_steps: 277200 # epoch 300 - validation_steps: 625 - validation_interval: 4620 - optimizer_config: - learning_rate: - cosine: - initial_learning_rate: 0.02 - decay_steps: 277200 # epoch 300 + trainer: + train_steps: 831600 # epoch 300 to 450 + optimizer_config: + learning_rate: + cosine: + initial_learning_rate: 0.1 + decay_steps: 831600 # epoch 300 to 450 optimizer: type: sgd_torch sgd_torch: @@ -97,4 +90,3 @@ trainer: type: 'linear' linear: warmup_steps: 4620 # 5 epochs - warmup_learning_rate: 0 From 60dd2c8cb355cec6bd6b919f8642454f33f33e89 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Fri, 28 Jan 2022 01:32:19 -0500 Subject: [PATCH 87/96] Update yolox.yaml --- .../yolo/configs/experiments/yolox/yolox.yaml | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 42cea1bd036..593d8a0d49a 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -70,23 +70,23 @@ task: area_thresh: 0.1 validation_data: input_path: 'gs://cam2-datasets/coco/val*' - trainer: - train_steps: 831600 # epoch 300 to 450 - optimizer_config: - learning_rate: - cosine: - initial_learning_rate: 0.1 - decay_steps: 831600 # epoch 300 to 450 - optimizer: - type: sgd_torch - sgd_torch: - momentum: 0.9 - momentum_start: 0.9 - nesterov: true - warmup_steps: 4620 - weight_decay: 0.0005 - name: SGD - warmup: - type: 'linear' - linear: - warmup_steps: 4620 # 5 epochs +trainer: + train_steps: 831600 # epoch 300 to 450 + optimizer_config: + learning_rate: + cosine: + initial_learning_rate: 0.1 + decay_steps: 831600 # epoch 300 to 450 + optimizer: + type: sgd_torch + sgd_torch: + momentum: 0.9 + momentum_start: 0.9 + nesterov: true + warmup_steps: 4620 + weight_decay: 0.0005 + name: SGD + warmup: + type: 'linear' + linear: + warmup_steps: 4620 # 5 epochs From f889199b6425eff83b02af43c394826d0446a46f Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Fri, 28 Jan 2022 01:33:17 -0500 Subject: [PATCH 88/96] Update yolox.yaml --- .../yolo/configs/experiments/yolox/yolox.yaml | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 593d8a0d49a..7d660fa4318 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -77,16 +77,16 @@ trainer: cosine: initial_learning_rate: 0.1 decay_steps: 831600 # epoch 300 to 450 - optimizer: - type: sgd_torch - sgd_torch: - momentum: 0.9 - momentum_start: 0.9 - nesterov: true - warmup_steps: 4620 - weight_decay: 0.0005 - name: SGD - warmup: - type: 'linear' - linear: - warmup_steps: 4620 # 5 epochs + optimizer: + type: sgd_torch + sgd_torch: + momentum: 0.9 + momentum_start: 0.9 + nesterov: true + warmup_steps: 4620 + weight_decay: 0.0005 + name: SGD + warmup: + type: 'linear' + linear: + warmup_steps: 4620 # 5 epochs From 4ea7a295c93a3c2f6b53970909eb53254db8ab31 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Fri, 28 Jan 2022 01:33:49 -0500 Subject: [PATCH 89/96] Update yolox.yaml --- .../projects/yolo/configs/experiments/yolox/yolox.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 7d660fa4318..2ae1f278ab8 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -86,7 +86,7 @@ trainer: warmup_steps: 4620 weight_decay: 0.0005 name: SGD - warmup: - type: 'linear' - linear: - warmup_steps: 4620 # 5 epochs + warmup: + type: 'linear' + linear: + warmup_steps: 4620 # 5 epochs From eb4e60de1262ae0dd19acb2f3c9a1801760373af Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Fri, 28 Jan 2022 02:11:34 -0500 Subject: [PATCH 90/96] Update yolox.yaml --- .../yolo/configs/experiments/yolox/yolox.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 2ae1f278ab8..5159e79c789 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -33,17 +33,18 @@ task: use_scaled_loss: true update_on_repeat: true box_loss_type: - 'all': ciou + 'all': iou ignore_thresh: 'all': 0.0 iou_normalizer: - 'all': 0.05 + 'all': 5.00 cls_normalizer: - 'all': 0.3 + 'all': 1.00 object_normalizer: '5': 0.28 '4': 0.70 '3': 2.80 + ‘all': 1.00 objectness_smooth: 'all': 1.0 norm_activation: @@ -71,12 +72,14 @@ task: validation_data: input_path: 'gs://cam2-datasets/coco/val*' trainer: - train_steps: 831600 # epoch 300 to 450 + train_steps: 277200 # epoch 300 + validation_steps: 625 + validation_interval: 4620 optimizer_config: learning_rate: cosine: - initial_learning_rate: 0.1 - decay_steps: 831600 # epoch 300 to 450 + initial_learning_rate: 0.02 + decay_steps: 277200 # epoch 300 optimizer: type: sgd_torch sgd_torch: @@ -90,3 +93,4 @@ trainer: type: 'linear' linear: warmup_steps: 4620 # 5 epochs + warmup_learning_rate: 0 From 03c038fbaaa6373e3497ae828e0a09978fa94e3f Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Fri, 28 Jan 2022 02:14:52 -0500 Subject: [PATCH 91/96] Update yolox.yaml --- .../beta/projects/yolo/configs/experiments/yolox/yolox.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 5159e79c789..2a37748afb5 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -41,9 +41,9 @@ task: cls_normalizer: 'all': 1.00 object_normalizer: - '5': 0.28 - '4': 0.70 - '3': 2.80 +# '5': 0.28 +# '4': 0.70 +# '3': 2.80 ‘all': 1.00 objectness_smooth: 'all': 1.0 From 220f33e5a2a33db160503d0f541d915536348028 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Fri, 28 Jan 2022 02:16:46 -0500 Subject: [PATCH 92/96] Create yolox.yaml From 36b8a994b64461ed0aef331dd82cac99e0d6c90e Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Fri, 28 Jan 2022 02:18:52 -0500 Subject: [PATCH 93/96] Update yolox.yaml --- .../beta/projects/yolo/configs/experiments/yolox/yolox.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 2a37748afb5..b39c445e8a9 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -41,9 +41,6 @@ task: cls_normalizer: 'all': 1.00 object_normalizer: -# '5': 0.28 -# '4': 0.70 -# '3': 2.80 ‘all': 1.00 objectness_smooth: 'all': 1.0 From bfabacee2b53395da42b1d0350c0253dfe4eac11 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Fri, 28 Jan 2022 02:20:22 -0500 Subject: [PATCH 94/96] Update yolox.yaml --- .../beta/projects/yolo/configs/experiments/yolox/yolox.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index b39c445e8a9..2d444022bcd 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -41,7 +41,7 @@ task: cls_normalizer: 'all': 1.00 object_normalizer: - ‘all': 1.00 + 'all': 1.00 objectness_smooth: 'all': 1.0 norm_activation: From ad0ff85a44fc902581f0c2bd345986c925960501 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Fri, 28 Jan 2022 12:26:39 -0500 Subject: [PATCH 95/96] Update yolox.yaml --- .../yolo/configs/experiments/yolox/yolox.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml index 2d444022bcd..1b3ae51979f 100644 --- a/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml +++ b/official/vision/beta/projects/yolo/configs/experiments/yolox/yolox.yaml @@ -33,15 +33,17 @@ task: use_scaled_loss: true update_on_repeat: true box_loss_type: - 'all': iou + 'all': ciou ignore_thresh: 'all': 0.0 iou_normalizer: - 'all': 5.00 + 'all': 0.05 cls_normalizer: - 'all': 1.00 + 'all': 0.3 object_normalizer: - 'all': 1.00 + '5': 0.28 + '4': 0.70 + '3': 2.80 objectness_smooth: 'all': 1.0 norm_activation: @@ -70,12 +72,10 @@ task: input_path: 'gs://cam2-datasets/coco/val*' trainer: train_steps: 277200 # epoch 300 - validation_steps: 625 - validation_interval: 4620 optimizer_config: learning_rate: cosine: - initial_learning_rate: 0.02 + initial_learning_rate: 0.1 decay_steps: 277200 # epoch 300 optimizer: type: sgd_torch @@ -90,4 +90,3 @@ trainer: type: 'linear' linear: warmup_steps: 4620 # 5 epochs - warmup_learning_rate: 0 From b4ae7e6563c1faa37ec06527293c1e94b2a9b1d6 Mon Sep 17 00:00:00 2001 From: Wenxin Jiang <71545489+Wenxin-Jiang@users.noreply.github.com> Date: Sun, 30 Jan 2022 15:37:01 -0500 Subject: [PATCH 96/96] Update yolox_head.py --- official/vision/beta/projects/yolo/modeling/heads/yolox_head.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py index 367563c3663..c580372f352 100644 --- a/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py +++ b/official/vision/beta/projects/yolo/modeling/heads/yolox_head.py @@ -171,7 +171,7 @@ def build(self, input_shape): bias_initializer=tf.keras.initializers.constant(self._bias)) self._reg_preds[k] = tf.keras.layers.Conv2D( - filters=4 * self._boxes_per_level, + filters=4, kernel_size=(1, 1), strides=(1, 1), padding='same')