[Dev] update benchmark script (#3116)

open-mmlab · Jun 19, 2023 · 20adcfa · 20adcfa
1 parent bb93b48
commit 20adcfa
Show file tree

Hide file tree

Showing 22 changed files with 95 additions and 3 deletions.
diff --git a/.dev/batch_test_list.py → .dev_scripts/batch_test_list.py b/.dev/batch_test_list.py → .dev_scripts/batch_test_list.py
diff --git a/.dev/batch_train_list.txt → .dev_scripts/batch_train_list.txt b/.dev/batch_train_list.txt → .dev_scripts/batch_train_list.txt
diff --git a/.dev/benchmark_evaluation.sh → .dev_scripts/benchmark_evaluation.sh b/.dev/benchmark_evaluation.sh → .dev_scripts/benchmark_evaluation.sh
diff --git a/.dev_scripts/benchmark_full_models.txt b/.dev_scripts/benchmark_full_models.txt
@@ -0,0 +1,56 @@
+ann/ann_r50-d8_4xb2-40k_cityscapes-512x1024.py
+apcnet/apcnet_r50-d8_4xb4-80k_ade20k-512x512.py
+beit/beit-large_upernet_8xb1-amp-160k_ade20k-640x640.py
+bisenetv1/bisenetv1_r18-d32_4xb4-160k_coco-stuff164k-512x512.py
+bisenetv2/bisenetv2_fcn_4xb4-ohem-160k_cityscapes-1024x1024.py
+ccnet/ccnet_r50-d8_4xb2-40k_cityscapes-512x1024.py
+cgnet/cgnet_fcn_4xb4-60k_cityscapes-680x680.py
+convnext/convnext-tiny_upernet_8xb2-amp-160k_ade20k-512x512.py
+danet/danet_r50-d8_4xb2-40k_cityscapes-512x1024.py
+deeplabv3/deeplabv3_r50-d8_4xb2-40k_cityscapes-512x1024.py
+deeplabv3plus/deeplabv3plus_r101-d8_4xb2-40k_cityscapes-769x769.py
+dmnet/dmnet_r50-d8_4xb2-40k_cityscapes-512x1024.py
+dnlnet/dnl_r50-d8_4xb2-40k_cityscapes-512x1024.py
+dpt/dpt_vit-b16_8xb2-160k_ade20k-512x512.py
+emanet/eemanet_r50-d8_4xb2-80k_cityscapes-512x1024.py
+encnet/encnet_r50-d8_4xb4-80k_ade20k-512x512.py
+erfnet/erfnet_fcn_4xb4-160k_cityscapes-512x1024.py
+fastfcn/fastfcn_r50-d32_jpu_aspp_4xb2-80k_cityscapes-512x1024.py
+fastscnn/fast_scnn_8xb4-160k_cityscapes-512x1024.py
+fcn/fcn_r50-d8_4xb2-40k_cityscapes-512x1024.py
+gcnet/gcnet_r50-d8_4xb2-40k_cityscapes-769x769.py
+hrnet/fcn_hr18s_4xb4-80k_ade20k-512x512.py
+icnet/icnet_r50-d8_4xb2-80k_cityscapes-832x832.py
+isanet/isanet_r50-d8_4xb2-80k_cityscapes-512x1024.py
+knet/knet-s3_swin-l_upernet_8xb2-adamw-80k_ade20k-640x640.py
+mae/mae-base_upernet_8xb2-amp-160k_ade20k-512x512.py
+mask2former/mask2former_r50_8xb2-90k_cityscapes-512x1024.py
+mask2former/mask2former_swin-t_8xb2-90k_cityscapes-512x1024.py
+mask2former/mask2former_swin-s_8xb2-160k_ade20k-512x512.py
+mask2former/mask2former_swin-l-in22k-384x384-pre_8xb2-160k_ade20k-640x640.py
+maskformer/maskformer_r50-d32_8xb2-160k_ade20k-512x512.py
+maskformer/maskformer_r101-d32_8xb2-160k_ade20k-512x512.py
+maskformer/maskformer_swin-t_upernet_8xb2-160k_ade20k-512x512.py
+maskformer/maskformer_swin-s_upernet_8xb2-160k_ade20k-512x512.py
+mobilenet_v2/mobilenet-v2-d8_pspnet_4xb2-80k_cityscapes-512x1024.py
+mobilenet_v3/mobilenet-v3-d8-scratch_lraspp_4xb4-320k_cityscapes-512x1024.py
+nonlocal_net/nonlocal_r50-d8_4xb2-40k_cityscapes-512x1024.py
+ocrnet/ocrnet_hr18_4xb4-80k_ade20k-512x512.py
+pidnet/pidnet-l_2xb6-120k_1024x1024-cityscapes.py
+point_rend/pointrend_r50_4xb2-80k_cityscapes-512x1024.py
+poolformer/fpn_poolformer_m48_8xb4-40k_ade20k-512x512.py
+psanet/psanet_r101-d8_4xb4-80k_ade20k-512x512.py
+pspnet/pspnet_r50-d8-rsb_4xb2-adamw-80k_cityscapes-512x1024.py
+resnest/resnest_s101-d8_deeplabv3_4xb4-160k_ade20k-512x512.py
+segformer/segformer_mit-b5_8xb1-160k_cityscapes-1024x1024.py
+segformer/segformer_mit-b5_8xb2-160k_ade20k-512x512.py
+segmenter/segmenter_vit-t_mask_8xb1-160k_ade20k-512x512.py
+segnext/segnext_mscan-t_1xb16-adamw-160k_ade20k-512x512.py
+sem_fpn/fpn_r101_4xb2-80k_cityscapes-512x1024.py
+setr/setr_vit-l-mla_8xb1-160k_ade20k-512x512.py
+stdc/stdc1_in1k-pre_4xb12-80k_cityscapes-512x1024.py
+swin/swin-small-patch4-window7-in1k-pre_upernet_8xb2-160k_ade20k-512x512.py
+twins/twins_pcpvt-s_uperhead_8xb4-160k_ade20k-512x512.py
+unet/unet-s5-d16_fcn_4xb4-160k_cityscapes-512x1024.py
+upernet/upernet_r50_4xb2-40k_cityscapes-769x769.py
+vit/vit_deit-s16_upernet_8xb2-80k_ade20k-512x512.py
diff --git a/.dev/benchmark_inference.py → .dev_scripts/benchmark_inference.py b/.dev/benchmark_inference.py → .dev_scripts/benchmark_inference.py
diff --git a/.dev_scripts/benchmark_options.py b/.dev_scripts/benchmark_options.py
@@ -0,0 +1,10 @@
+third_part_libs = [
+    'pip install mmengine',
+    'pip install mmcv>=2.0.0',
+    'pip install mmcls==1.0.0rc6',
+    'pip install mmdet==3.0.0',
+    'pip install -r requirements.txt',
+    'pip install timm',
+]
+
+default_floating_range = 0.5
diff --git a/.dev/benchmark_train.sh → .dev_scripts/benchmark_train.sh b/.dev/benchmark_train.sh → .dev_scripts/benchmark_train.sh
diff --git a/.dev_scripts/benchmark_train_models.txt b/.dev_scripts/benchmark_train_models.txt
@@ -0,0 +1,25 @@
+bisenetv1/bisenetv1_r18-d32_4xb4-160k_coco-stuff164k-512x512.py
+bisenetv2/bisenetv2_fcn_4xb4-ohem-160k_cityscapes-1024x1024.py
+deeplabv3/deeplabv3_r50-d8_4xb2-40k_cityscapes-512x1024.py
+deeplabv3/deeplabv3_r101-d8_4xb4-160k_ade20k-512x512.py
+deeplabv3plus/deeplabv3plus_r101-d8_4xb2-40k_cityscapes-769x769.py
+deeplabv3plus/deeplabv3plus_r50-d8_4xb4-80k_ade20k-512x512.py
+hrnet/fcn_hr18s_4xb4-80k_ade20k-512x512.py
+hrnet/fcn_hr18_4xb2-160k_cityscapes-512x1024.py
+knet/knet-s3_swin-l_upernet_8xb2-adamw-80k_ade20k-512x512.py
+knet/knet-s3_r50-d8_pspnet_8xb2-adamw-80k_ade20k-512x512.py
+mae/mae-base_upernet_8xb2-amp-160k_ade20k-512x512.py
+mask2former/mask2former_swin-l-in22k-384x384-pre_8xb2-160k_ade20k-640x640.py
+maskformer/maskformer_swin-s_upernet_8xb2-160k_ade20k-512x512.py
+mobilenet_v2/mobilenet-v2-d8_deeplabv3_4xb2-80k_cityscapes-512x1024.py
+ocrnet/ocrnet_hr48_4xb4-80k_ade20k-512x512.py
+pidnet/pidnet-m_2xb6-120k_1024x1024-cityscapes.py
+pspnet/pspnet_r50-d8-rsb_4xb2-adamw-80k_cityscapes-512x1024.py
+pspnet/pspnet_r101-d8_4xb4-80k_ade20k-512x512.py
+segformer/segformer_mit-b5_8xb2-160k_ade20k-512x512.py
+segmenter/segmenter_vit-t_mask_8xb1-160k_ade20k-512x512.py
+segnext/segnext_mscan-t_1xb16-adamw-160k_ade20k-512x512.py
+swin/swin-base-patch4-window12-in22k-384x384-pre_upernet_8xb2-160k_ade20k-512x512.py
+twins/twins_pcpvt-l_uperhead_8xb2-160k_ade20k-512x512.py
+unet/unet-s5-d16_fcn_4xb4-160k_cityscapes-512x1024.py
+upernet/upernet_r101_4xb2-40k_cityscapes-512x1024.py
diff --git a/.dev/check_urls.py → .dev_scripts/check_urls.py b/.dev/check_urls.py → .dev_scripts/check_urls.py
diff --git a/.dev/gather_benchmark_evaluation_results.py → ...ts/gather_benchmark_evaluation_results.py b/.dev/gather_benchmark_evaluation_results.py → ...ts/gather_benchmark_evaluation_results.py
diff --git a/.dev/gather_benchmark_train_results.py → ...scripts/gather_benchmark_train_results.py b/.dev/gather_benchmark_train_results.py → ...scripts/gather_benchmark_train_results.py
diff --git a/.dev/gather_models.py → .dev_scripts/gather_models.py b/.dev/gather_models.py → .dev_scripts/gather_models.py
diff --git a/.dev/generate_benchmark_evaluation_script.py → ...s/generate_benchmark_evaluation_script.py b/.dev/generate_benchmark_evaluation_script.py → ...s/generate_benchmark_evaluation_script.py
@@ -17,7 +17,7 @@ def parse_args():
     parser.add_argument(
         '--out',
         type=str,
-        default='.dev/benchmark_evaluation.sh',
+        default='.dev_scripts/benchmark_evaluation.sh',
         help='path to save model benchmark script')
 
     args = parser.parse_args()

diff --git a/.dev/generate_benchmark_train_script.py → ...cripts/generate_benchmark_train_script.py b/.dev/generate_benchmark_train_script.py → ...cripts/generate_benchmark_train_script.py
@@ -19,7 +19,7 @@ def parse_args():
     parser.add_argument(
         '--out',
         type=str,
-        default='.dev/benchmark_train.sh',
+        default='.dev_scripts/benchmark_train.sh',
         help='path to save model benchmark script')
 
     args = parser.parse_args()

diff --git a/.dev/log_collector/example_config.py → .dev_scripts/log_collector/example_config.py b/.dev/log_collector/example_config.py → .dev_scripts/log_collector/example_config.py
diff --git a/.dev/log_collector/log_collector.py → .dev_scripts/log_collector/log_collector.py b/.dev/log_collector/log_collector.py → .dev_scripts/log_collector/log_collector.py
diff --git a/.dev/log_collector/readme.md → .dev_scripts/log_collector/readme.md b/.dev/log_collector/readme.md → .dev_scripts/log_collector/readme.md
diff --git a/.dev/log_collector/utils.py → .dev_scripts/log_collector/utils.py b/.dev/log_collector/utils.py → .dev_scripts/log_collector/utils.py
diff --git a/.dev/update_model_index.py → .dev_scripts/update_model_index.py b/.dev/update_model_index.py → .dev_scripts/update_model_index.py
diff --git a/.dev/upload_modelzoo.py → .dev_scripts/upload_modelzoo.py b/.dev/upload_modelzoo.py → .dev_scripts/upload_modelzoo.py
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -47,7 +47,7 @@ repos:
       - id: update-model-index
         name: update-model-index
         description: Collect model information and update model-index.yml
-        entry: .dev/update_model_index.py
+        entry: .dev_scripts/update_model_index.py
         additional_dependencies: [pyyaml]
         language: python
         require_serial: true

diff --git a/requirements/albu.txt b/requirements/albu.txt
@@ -0,0 +1 @@
+albumentations>=0.3.2 --no-binary qudida,albumentations