[Refactor] migrate File I/O to the newest mmengine (open-mmlab#9709)

imclab · Mar 17, 2023 · 621b8c8 · 621b8c8
1 parent 0f5cd10
commit 621b8c8
Show file tree

Hide file tree

Showing 158 changed files with 1,255 additions and 842 deletions.
diff --git a/configs/_base_/datasets/cityscapes_detection.py b/configs/_base_/datasets/cityscapes_detection.py
@@ -2,8 +2,23 @@
 dataset_type = 'CityscapesDataset'
 data_root = 'data/cityscapes/'
 
+# Example to use different file client
+# Method 1: simply set the data root and let the file I/O module
+# automatically infer from prefix (not support LMDB and Memcache yet)
+
+# data_root = 's3://openmmlab/datasets/segmentation/cityscapes/'
+
+# Method 2: Use `backend_args`, `file_client_args` in versions before 3.0.0rc6
+# backend_args = dict(
+#     backend='petrel',
+#     path_mapping=dict({
+#         './data/': 's3://openmmlab/datasets/segmentation/',
+#          'data/': 's3://openmmlab/datasets/segmentation/'
+#      }))
+backend_args = None
+
 train_pipeline = [
-    dict(type='LoadImageFromFile'),
+    dict(type='LoadImageFromFile', backend_args=backend_args),
     dict(type='LoadAnnotations', with_bbox=True),
     dict(
         type='RandomResize',
@@ -14,7 +29,7 @@
 ]
 
 test_pipeline = [
-    dict(type='LoadImageFromFile'),
+    dict(type='LoadImageFromFile', backend_args=backend_args),
     dict(type='Resize', scale=(2048, 1024), keep_ratio=True),
     # If you don't have a gt annotation, delete the pipeline
     dict(type='LoadAnnotations', with_bbox=True),
@@ -39,7 +54,8 @@
             ann_file='annotations/instancesonly_filtered_gtFine_train.json',
             data_prefix=dict(img='leftImg8bit/train/'),
             filter_cfg=dict(filter_empty_gt=True, min_size=32),
-            pipeline=train_pipeline)))
+            pipeline=train_pipeline,
+            backend_args=backend_args)))
 
 val_dataloader = dict(
     batch_size=1,
@@ -54,13 +70,15 @@
         data_prefix=dict(img='leftImg8bit/val/'),
         test_mode=True,
         filter_cfg=dict(filter_empty_gt=True, min_size=32),
-        pipeline=test_pipeline))
+        pipeline=test_pipeline,
+        backend_args=backend_args))
 
 test_dataloader = val_dataloader
 
 val_evaluator = dict(
     type='CocoMetric',
     ann_file=data_root + 'annotations/instancesonly_filtered_gtFine_val.json',
-    metric='bbox')
+    metric='bbox',
+    backend_args=backend_args)
 
 test_evaluator = val_evaluator
diff --git a/configs/_base_/datasets/cityscapes_instance.py b/configs/_base_/datasets/cityscapes_instance.py
@@ -2,8 +2,23 @@
 dataset_type = 'CityscapesDataset'
 data_root = 'data/cityscapes/'
 
+# Example to use different file client
+# Method 1: simply set the data root and let the file I/O module
+# automatically infer from prefix (not support LMDB and Memcache yet)
+
+# data_root = 's3://openmmlab/datasets/segmentation/cityscapes/'
+
+# Method 2: Use backend_args, file_client_args in versions before 3.0.0rc6
+# backend_args = dict(
+#     backend='petrel',
+#     path_mapping=dict({
+#         './data/': 's3://openmmlab/datasets/segmentation/',
+#          'data/': 's3://openmmlab/datasets/segmentation/'
+#      }))
+backend_args = None
+
 train_pipeline = [
-    dict(type='LoadImageFromFile'),
+    dict(type='LoadImageFromFile', backend_args=backend_args),
     dict(type='LoadAnnotations', with_bbox=True, with_mask=True),
     dict(
         type='RandomResize',
@@ -14,7 +29,7 @@
 ]
 
 test_pipeline = [
-    dict(type='LoadImageFromFile'),
+    dict(type='LoadImageFromFile', backend_args=backend_args),
     dict(type='Resize', scale=(2048, 1024), keep_ratio=True),
     # If you don't have a gt annotation, delete the pipeline
     dict(type='LoadAnnotations', with_bbox=True, with_mask=True),
@@ -39,7 +54,8 @@
             ann_file='annotations/instancesonly_filtered_gtFine_train.json',
             data_prefix=dict(img='leftImg8bit/train/'),
             filter_cfg=dict(filter_empty_gt=True, min_size=32),
-            pipeline=train_pipeline)))
+            pipeline=train_pipeline,
+            backend_args=backend_args)))
 
 val_dataloader = dict(
     batch_size=1,
@@ -54,7 +70,8 @@
         data_prefix=dict(img='leftImg8bit/val/'),
         test_mode=True,
         filter_cfg=dict(filter_empty_gt=True, min_size=32),
-        pipeline=test_pipeline))
+        pipeline=test_pipeline,
+        backend_args=backend_args))
 
 test_dataloader = val_dataloader
 
@@ -63,13 +80,13 @@
         type='CocoMetric',
         ann_file=data_root +
         'annotations/instancesonly_filtered_gtFine_val.json',
-        metric=['bbox', 'segm']),
+        metric=['bbox', 'segm'],
+        backend_args=backend_args),
     dict(
         type='CityScapesMetric',
-        ann_file=data_root +
-        'annotations/instancesonly_filtered_gtFine_val.json',
-        seg_prefix=data_root + '/gtFine/val',
-        outfile_prefix='./work_dirs/cityscapes_metric/instance')
+        seg_prefix=data_root + 'gtFine/val',
+        outfile_prefix='./work_dirs/cityscapes_metric/instance',
+        backend_args=backend_args)
 ]
 
 test_evaluator = val_evaluator

diff --git a/configs/_base_/datasets/coco_detection.py b/configs/_base_/datasets/coco_detection.py
@@ -2,23 +2,30 @@
 dataset_type = 'CocoDataset'
 data_root = 'data/coco/'
 
-# file_client_args = dict(
+# Example to use different file client
+# Method 1: simply set the data root and let the file I/O module
+# automatically infer from prefix (not support LMDB and Memcache yet)
+
+# data_root = 's3://openmmlab/datasets/detection/coco/'
+
+# Method 2: Use `backend_args`, `file_client_args` in versions before 3.0.0rc6
+# backend_args = dict(
 #     backend='petrel',
 #     path_mapping=dict({
 #         './data/': 's3://openmmlab/datasets/detection/',
 #         'data/': 's3://openmmlab/datasets/detection/'
 #     }))
-file_client_args = dict(backend='disk')
+backend_args = None
 
 train_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=file_client_args),
+    dict(type='LoadImageFromFile', backend_args=backend_args),
     dict(type='LoadAnnotations', with_bbox=True),
     dict(type='Resize', scale=(1333, 800), keep_ratio=True),
     dict(type='RandomFlip', prob=0.5),
     dict(type='PackDetInputs')
 ]
 test_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=file_client_args),
+    dict(type='LoadImageFromFile', backend_args=backend_args),
     dict(type='Resize', scale=(1333, 800), keep_ratio=True),
     # If you don't have a gt annotation, delete the pipeline
     dict(type='LoadAnnotations', with_bbox=True),
@@ -39,7 +46,8 @@
         ann_file='annotations/instances_train2017.json',
         data_prefix=dict(img='train2017/'),
         filter_cfg=dict(filter_empty_gt=True, min_size=32),
-        pipeline=train_pipeline))
+        pipeline=train_pipeline,
+        backend_args=backend_args))
 val_dataloader = dict(
     batch_size=1,
     num_workers=2,
@@ -52,14 +60,16 @@
         ann_file='annotations/instances_val2017.json',
         data_prefix=dict(img='val2017/'),
         test_mode=True,
-        pipeline=test_pipeline))
+        pipeline=test_pipeline,
+        backend_args=backend_args))
 test_dataloader = val_dataloader
 
 val_evaluator = dict(
     type='CocoMetric',
     ann_file=data_root + 'annotations/instances_val2017.json',
     metric='bbox',
-    format_only=False)
+    format_only=False,
+    backend_args=backend_args)
 test_evaluator = val_evaluator
 
 # inference on test dataset and

diff --git a/configs/_base_/datasets/coco_instance.py b/configs/_base_/datasets/coco_instance.py
@@ -2,23 +2,30 @@
 dataset_type = 'CocoDataset'
 data_root = 'data/coco/'
 
-# file_client_args = dict(
+# Example to use different file client
+# Method 1: simply set the data root and let the file I/O module
+# automatically infer from prefix (not support LMDB and Memcache yet)
+
+# data_root = 's3://openmmlab/datasets/detection/coco/'
+
+# Method 2: Use `backend_args`, `file_client_args` in versions before 3.0.0rc6
+# backend_args = dict(
 #     backend='petrel',
 #     path_mapping=dict({
 #         './data/': 's3://openmmlab/datasets/detection/',
 #         'data/': 's3://openmmlab/datasets/detection/'
 #     }))
-file_client_args = dict(backend='disk')
+backend_args = None
 
 train_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=file_client_args),
+    dict(type='LoadImageFromFile', backend_args=backend_args),
     dict(type='LoadAnnotations', with_bbox=True, with_mask=True),
     dict(type='Resize', scale=(1333, 800), keep_ratio=True),
     dict(type='RandomFlip', prob=0.5),
     dict(type='PackDetInputs')
 ]
 test_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=file_client_args),
+    dict(type='LoadImageFromFile', backend_args=backend_args),
     dict(type='Resize', scale=(1333, 800), keep_ratio=True),
     # If you don't have a gt annotation, delete the pipeline
     dict(type='LoadAnnotations', with_bbox=True, with_mask=True),
@@ -39,7 +46,8 @@
         ann_file='annotations/instances_train2017.json',
         data_prefix=dict(img='train2017/'),
         filter_cfg=dict(filter_empty_gt=True, min_size=32),
-        pipeline=train_pipeline))
+        pipeline=train_pipeline,
+        backend_args=backend_args))
 val_dataloader = dict(
     batch_size=1,
     num_workers=2,
@@ -52,14 +60,16 @@
         ann_file='annotations/instances_val2017.json',
         data_prefix=dict(img='val2017/'),
         test_mode=True,
-        pipeline=test_pipeline))
+        pipeline=test_pipeline,
+        backend_args=backend_args))
 test_dataloader = val_dataloader
 
 val_evaluator = dict(
     type='CocoMetric',
     ann_file=data_root + 'annotations/instances_val2017.json',
     metric=['bbox', 'segm'],
-    format_only=False)
+    format_only=False,
+    backend_args=backend_args)
 test_evaluator = val_evaluator
 
 # inference on test dataset and

diff --git a/configs/_base_/datasets/coco_instance_semantic.py b/configs/_base_/datasets/coco_instance_semantic.py
@@ -2,24 +2,31 @@
 dataset_type = 'CocoDataset'
 data_root = 'data/coco/'
 
-# file_client_args = dict(
+# Example to use different file client
+# Method 1: simply set the data root and let the file I/O module
+# automatically infer from prefix (not support LMDB and Memcache yet)
+
+# data_root = 's3://openmmlab/datasets/detection/coco/'
+
+# Method 2: Use `backend_args`, `file_client_args` in versions before 3.0.0rc6
+# backend_args = dict(
 #     backend='petrel',
 #     path_mapping=dict({
 #         './data/': 's3://openmmlab/datasets/detection/',
 #         'data/': 's3://openmmlab/datasets/detection/'
 #     }))
-file_client_args = dict(backend='disk')
+backend_args = None
 
 train_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=file_client_args),
+    dict(type='LoadImageFromFile', backend_args=backend_args),
     dict(
         type='LoadAnnotations', with_bbox=True, with_mask=True, with_seg=True),
     dict(type='Resize', scale=(1333, 800), keep_ratio=True),
     dict(type='RandomFlip', prob=0.5),
     dict(type='PackDetInputs')
 ]
 test_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=file_client_args),
+    dict(type='LoadImageFromFile', backend_args=backend_args),
     dict(type='Resize', scale=(1333, 800), keep_ratio=True),
     # If you don't have a gt annotation, delete the pipeline
     dict(
@@ -42,7 +49,8 @@
         ann_file='annotations/instances_train2017.json',
         data_prefix=dict(img='train2017/', seg='stuffthingmaps/train2017/'),
         filter_cfg=dict(filter_empty_gt=True, min_size=32),
-        pipeline=train_pipeline))
+        pipeline=train_pipeline,
+        backend_args=backend_args))
 
 val_dataloader = dict(
     batch_size=1,
@@ -56,13 +64,15 @@
         ann_file='annotations/instances_val2017.json',
         data_prefix=dict(img='val2017/'),
         test_mode=True,
-        pipeline=test_pipeline))
+        pipeline=test_pipeline,
+        backend_args=backend_args))
 
 test_dataloader = val_dataloader
 
 val_evaluator = dict(
     type='CocoMetric',
     ann_file=data_root + 'annotations/instances_val2017.json',
     metric=['bbox', 'segm'],
-    format_only=False)
+    format_only=False,
+    backend_args=backend_args)
 test_evaluator = val_evaluator
diff --git a/configs/_base_/datasets/coco_panoptic.py b/configs/_base_/datasets/coco_panoptic.py
@@ -1,26 +1,33 @@
 # dataset settings
 dataset_type = 'CocoPanopticDataset'
-data_root = 'data/coco/'
+# data_root = 'data/coco/'
 
-# file_client_args = dict(
+# Example to use different file client
+# Method 1: simply set the data root and let the file I/O module
+# automatically infer from prefix (not support LMDB and Memcache yet)
+
+data_root = 's3://openmmlab/datasets/detection/coco/'
+
+# Method 2: Use `backend_args`, `file_client_args` in versions before 3.0.0rc6
+# backend_args = dict(
 #     backend='petrel',
 #     path_mapping=dict({
 #         './data/': 's3://openmmlab/datasets/detection/',
 #         'data/': 's3://openmmlab/datasets/detection/'
 #     }))
-file_client_args = dict(backend='disk')
+backend_args = None
 
 train_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=file_client_args),
-    dict(type='LoadPanopticAnnotations', file_client_args=file_client_args),
+    dict(type='LoadImageFromFile', backend_args=backend_args),
+    dict(type='LoadPanopticAnnotations', backend_args=backend_args),
     dict(type='Resize', scale=(1333, 800), keep_ratio=True),
     dict(type='RandomFlip', prob=0.5),
     dict(type='PackDetInputs')
 ]
 test_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=file_client_args),
+    dict(type='LoadImageFromFile', backend_args=backend_args),
     dict(type='Resize', scale=(1333, 800), keep_ratio=True),
-    dict(type='LoadPanopticAnnotations', file_client_args=file_client_args),
+    dict(type='LoadPanopticAnnotations', backend_args=backend_args),
     dict(
         type='PackDetInputs',
         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
@@ -40,7 +47,8 @@
         data_prefix=dict(
             img='train2017/', seg='annotations/panoptic_train2017/'),
         filter_cfg=dict(filter_empty_gt=True, min_size=32),
-        pipeline=train_pipeline))
+        pipeline=train_pipeline,
+        backend_args=backend_args))
 val_dataloader = dict(
     batch_size=1,
     num_workers=2,
@@ -53,15 +61,15 @@
         ann_file='annotations/panoptic_val2017.json',
         data_prefix=dict(img='val2017/', seg='annotations/panoptic_val2017/'),
         test_mode=True,
-        pipeline=test_pipeline))
+        pipeline=test_pipeline,
+        backend_args=backend_args))
 test_dataloader = val_dataloader
 
 val_evaluator = dict(
     type='CocoPanopticMetric',
     ann_file=data_root + 'annotations/panoptic_val2017.json',
     seg_prefix=data_root + 'annotations/panoptic_val2017/',
-    file_client_args=file_client_args,
-)
+    backend_args=backend_args)
 test_evaluator = val_evaluator
 
 # inference on test dataset and