diff --git a/.gitmodules b/.gitmodules index 32e1b67d4b..572e3c0367 100644 --- a/.gitmodules +++ b/.gitmodules @@ -141,3 +141,7 @@ path = frame_benchmark/pytorch/dynamic/PaddleDetection/models/yolov7 url = https://github.com/PaddleBenchmark/yolov7.git branch = main +[submodule "frame_benchmark/pytorch/dynamic/Paddle3D/models/OpenPCDet"] + path = frame_benchmark/pytorch/dynamic/Paddle3D/models/OpenPCDet + url = git@github.com:PaddleBenchmark/OpenPCDet.git + breanch = master diff --git a/frame_benchmark/docker_images.yaml b/frame_benchmark/docker_images.yaml index 6bd31b76aa..362526038d 100755 --- a/frame_benchmark/docker_images.yaml +++ b/frame_benchmark/docker_images.yaml @@ -99,3 +99,4 @@ pytorch: petrv2: iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.7-cudnn8.4.1-nccl2.12.12 VideoSwin: iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.7-cudnn8.4.1-nccl2.12.12 vit_adapter: iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.7-cudnn8.4.1-nccl2.12.12 + pointpillars: iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.7-cudnn8.4.1-nccl2.12.12 diff --git a/frame_benchmark/models_path.yaml b/frame_benchmark/models_path.yaml index 0b711b1f1d..afbe4cde25 100644 --- a/frame_benchmark/models_path.yaml +++ b/frame_benchmark/models_path.yaml @@ -99,3 +99,4 @@ pytorch: petrv2: benchmark/frame_benchmark/pytorch/dynamic/Paddle3D/models/PETR VideoSwin: benchmark/frame_benchmark/pytorch/dynamic/PaddleVideo/models/Video-Swin-Transformer vit_adapter: benchmark/frame_benchmark/pytorch/dynamic/PaddleSeg/models/ViT-Adapter + pointpillars: benchmark/frame_benchmark/pytorch/dynamic/Paddle3D/models/OpenPCDet diff --git a/frame_benchmark/pytorch/dynamic/Paddle3D/models/OpenPCDet b/frame_benchmark/pytorch/dynamic/Paddle3D/models/OpenPCDet new file mode 160000 index 0000000000..0d6d9eb75f --- /dev/null +++ b/frame_benchmark/pytorch/dynamic/Paddle3D/models/OpenPCDet @@ -0,0 +1 @@ +Subproject commit 0d6d9eb75fbacb693915e7673ecba40f22859386 diff --git a/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/petrv2/benchmark_common/PrepareEnv.sh b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/petrv2/benchmark_common/PrepareEnv.sh index d07cca5b2b..2a6923b46b 100644 --- a/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/petrv2/benchmark_common/PrepareEnv.sh +++ b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/petrv2/benchmark_common/PrepareEnv.sh @@ -14,7 +14,7 @@ pip install setuptools==59.5.0 pip install mmcv-full==1.4.0 -f https://download.openmmlab.com/mmcv/dist/cu111/torch1.9.1/index.html wget https://paddle-wheel.bj.bcebos.com/benchmark/mmdetection-2.24.1.zip -unzip mmdetection-2.24.1.zip +unzip -o mmdetection-2.24.1.zip rm -rf mmdetection-2.24.1.zip mv mmdetection-2.24.1 mmdetection cd mmdetection diff --git a/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/N1C1/pointpillars_bs2_fp16_DP.sh b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/N1C1/pointpillars_bs2_fp16_DP.sh new file mode 100644 index 0000000000..829e7b765a --- /dev/null +++ b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/N1C1/pointpillars_bs2_fp16_DP.sh @@ -0,0 +1,10 @@ +model_item="pointpillars" +bs_item=2 +fp_item=fp16 +run_mode=DP +device_num=N1C1 +max_iter=400 +num_workers=8 + +bash prepare.sh; +bash run_benchmark.sh ${model_item} ${bs_item} ${fp_item} ${run_mode} ${device_num} ${max_iter} ${num_workers} 2>&1; diff --git a/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/N1C1/pointpillars_bs2_fp32_DP.sh b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/N1C1/pointpillars_bs2_fp32_DP.sh new file mode 100644 index 0000000000..e282087c7d --- /dev/null +++ b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/N1C1/pointpillars_bs2_fp32_DP.sh @@ -0,0 +1,10 @@ +model_item="pointpillars" +bs_item=2 +fp_item=fp32 +run_mode=DP +device_num=N1C1 +max_iter=400 +num_workers=8 + +bash prepare.sh; +bash run_benchmark.sh ${model_item} ${bs_item} ${fp_item} ${run_mode} ${device_num} ${max_iter} ${num_workers} 2>&1; diff --git a/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/N1C4/pointpillars_bs2_fp16_DP.sh b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/N1C4/pointpillars_bs2_fp16_DP.sh new file mode 100644 index 0000000000..1fc00b60a4 --- /dev/null +++ b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/N1C4/pointpillars_bs2_fp16_DP.sh @@ -0,0 +1,10 @@ +model_item="pointpillars" +bs_item=2 +fp_item=fp16 +run_mode=DP +device_num=N1C4 +max_iter=400 +num_workers=8 + +bash prepare.sh; +bash run_benchmark.sh ${model_item} ${bs_item} ${fp_item} ${run_mode} ${device_num} ${max_iter} ${num_workers} 2>&1; diff --git a/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/N1C4/pointpillars_bs2_fp32_DP.sh b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/N1C4/pointpillars_bs2_fp32_DP.sh new file mode 100644 index 0000000000..55d04203e0 --- /dev/null +++ b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/N1C4/pointpillars_bs2_fp32_DP.sh @@ -0,0 +1,10 @@ +model_item="pointpillars" +bs_item=2 +fp_item=fp32 +run_mode=DP +device_num=N1C4 +max_iter=400 +num_workers=8 + +bash prepare.sh; +bash run_benchmark.sh ${model_item} ${bs_item} ${fp_item} ${run_mode} ${device_num} ${max_iter} ${num_workers} 2>&1; diff --git a/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/analysis_log.py b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/analysis_log.py new file mode 100644 index 0000000000..adc6ff90b7 --- /dev/null +++ b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/analysis_log.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# encoding=utf-8 vi:ts=4:sw=4:expandtab:ft=python + +import re +import sys +import json +import os + +def analyze(model_name, batch_size, log_file, res_log_file, device_num): + gpu_ids_pat = re.compile(r"GPU (.*):") + time_pat = re.compile(r"Batch time: (.*)\(.*\)") + + logs = open(log_file).readlines() + logs = ";".join(logs) + gpu_ids_res = gpu_ids_pat.findall(logs) + time_res = time_pat.findall(logs) + print(time_res, "***********************") + + print("---device_num:-", device_num) + index_c = device_num.index('C') + print("---index_c:-", index_c) + gpu_num = int(device_num[index_c + 1:len(device_num)]) + print("-----gpu_num:", gpu_num) + + fail_flag = 0 + run_mode = "" + fp_item = "fp32" + ips = 0 + + run_mode = "DP" + skip_num = 10 + total_time = 0 + for i in range(skip_num, len(time_res)): + total_time += float(time_res[i]) + avg_time = total_time / (len(time_res) - skip_num) + ips = float(batch_size) * round(1 / avg_time, 3) + + info = { "model_branch": os.getenv('model_branch'), + "model_commit": os.getenv('model_commit'), + "model_name": model_name, + "batch_size": batch_size, + "fp_item": fp_item, + "run_mode": run_mode, + "convergence_value": 0, + "convergence_key": "", + "ips": ips * int(gpu_num), + "speed_unit":"images/s", + "device_num": device_num, + "model_run_time": os.getenv('model_run_time'), + "frame_commit": "", + "frame_version": os.getenv('frame_version'), + } + print(info) + json_info = json.dumps(info) + with open(res_log_file, "w") as of: + of.write(json_info) + +if __name__ == "__main__": + if len(sys.argv) != 6: + print("Usage:" + sys.argv[0] + " model_name path/to/log/file path/to/res/log/file") + sys.exit() + + model_name = sys.argv[1] + batch_size = sys.argv[2] + log_file = sys.argv[3] + res_log_file = sys.argv[4] + device_num = sys.argv[5] + + analyze(model_name, batch_size, log_file, res_log_file, device_num) diff --git a/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/kitti_dataset.yaml b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/kitti_dataset.yaml new file mode 100644 index 0000000000..a69acfff8e --- /dev/null +++ b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/kitti_dataset.yaml @@ -0,0 +1,70 @@ +DATASET: 'KittiDataset' +DATA_PATH: '../data/KITTI_800' + +POINT_CLOUD_RANGE: [0, -40, -3, 70.4, 40, 1] + +DATA_SPLIT: { + 'train': train, + 'test': val +} + +INFO_PATH: { + 'train': [kitti_infos_train.pkl], + 'test': [kitti_infos_val.pkl], +} + +GET_ITEM_LIST: ["points"] +FOV_POINTS_ONLY: True + +DATA_AUGMENTOR: + DISABLE_AUG_LIST: ['placeholder'] + AUG_CONFIG_LIST: + - NAME: gt_sampling + USE_ROAD_PLANE: True + DB_INFO_PATH: + - kitti_dbinfos_train.pkl + PREPARE: { + filter_by_min_points: ['Car:5', 'Pedestrian:5', 'Cyclist:5'], + filter_by_difficulty: [-1], + } + + SAMPLE_GROUPS: ['Car:20','Pedestrian:15', 'Cyclist:15'] + NUM_POINT_FEATURES: 4 + DATABASE_WITH_FAKELIDAR: False + REMOVE_EXTRA_WIDTH: [0.0, 0.0, 0.0] + LIMIT_WHOLE_SCENE: True + + - NAME: random_world_flip + ALONG_AXIS_LIST: ['x'] + + - NAME: random_world_rotation + WORLD_ROT_ANGLE: [-0.78539816, 0.78539816] + + - NAME: random_world_scaling + WORLD_SCALE_RANGE: [0.95, 1.05] + + +POINT_FEATURE_ENCODING: { + encoding_type: absolute_coordinates_encoding, + used_feature_list: ['x', 'y', 'z', 'intensity'], + src_feature_list: ['x', 'y', 'z', 'intensity'], +} + + +DATA_PROCESSOR: + - NAME: mask_points_and_boxes_outside_range + REMOVE_OUTSIDE_BOXES: True + + - NAME: shuffle_points + SHUFFLE_ENABLED: { + 'train': True, + 'test': False + } + + - NAME: transform_points_to_voxels + VOXEL_SIZE: [0.05, 0.05, 0.1] + MAX_POINTS_PER_VOXEL: 5 + MAX_NUMBER_OF_VOXELS: { + 'train': 16000, + 'test': 40000 + } diff --git a/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/pointpillars.yaml b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/pointpillars.yaml new file mode 100644 index 0000000000..d0225644bf --- /dev/null +++ b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/pointpillars.yaml @@ -0,0 +1,144 @@ +CLASS_NAMES: ['Car'] + +DATA_CONFIG: + _BASE_CONFIG_: kitti_dataset.yaml + POINT_CLOUD_RANGE: [0, -39.68, -3, 69.12, 39.68, 1] + DATA_PROCESSOR: + - NAME: mask_points_and_boxes_outside_range + REMOVE_OUTSIDE_BOXES: True + + - NAME: shuffle_points + SHUFFLE_ENABLED: { + 'train': True, + 'test': False + } + + - NAME: transform_points_to_voxels + VOXEL_SIZE: [0.16, 0.16, 4] + MAX_POINTS_PER_VOXEL: 32 + MAX_NUMBER_OF_VOXELS: { + 'train': 16000, + 'test': 40000 + } + DATA_AUGMENTOR: + DISABLE_AUG_LIST: ['placeholder'] + AUG_CONFIG_LIST: + - NAME: gt_sampling + USE_ROAD_PLANE: True + DB_INFO_PATH: + - kitti_dbinfos_train.pkl + PREPARE: { + filter_by_min_points: ['Car:5'], + filter_by_difficulty: [-1], + } + + SAMPLE_GROUPS: ['Car:15'] + NUM_POINT_FEATURES: 4 + DATABASE_WITH_FAKELIDAR: False + REMOVE_EXTRA_WIDTH: [0.0, 0.0, 0.0] + LIMIT_WHOLE_SCENE: False + + - NAME: random_world_flip + ALONG_AXIS_LIST: ['x'] + + - NAME: random_world_rotation + WORLD_ROT_ANGLE: [-0.78539816, 0.78539816] + + - NAME: random_world_scaling + WORLD_SCALE_RANGE: [0.95, 1.05] + +MODEL: + NAME: PointPillar + + VFE: + NAME: PillarVFE + WITH_DISTANCE: False + USE_ABSLOTE_XYZ: True + USE_NORM: True + NUM_FILTERS: [64] + + MAP_TO_BEV: + NAME: PointPillarScatter + NUM_BEV_FEATURES: 64 + + BACKBONE_2D: + NAME: BaseBEVBackbone + LAYER_NUMS: [3, 5, 5] + LAYER_STRIDES: [2, 2, 2] + NUM_FILTERS: [64, 128, 256] + UPSAMPLE_STRIDES: [1, 2, 4] + NUM_UPSAMPLE_FILTERS: [128, 128, 128] + + DENSE_HEAD: + NAME: AnchorHeadSingle + CLASS_AGNOSTIC: False + + USE_DIRECTION_CLASSIFIER: True + DIR_OFFSET: 0.78539 + DIR_LIMIT_OFFSET: 0.0 + NUM_DIR_BINS: 2 + + ANCHOR_GENERATOR_CONFIG: [ + { + 'class_name': 'Car', + 'anchor_sizes': [[3.9, 1.6, 1.56]], + 'anchor_rotations': [0, 1.57], + 'anchor_bottom_heights': [-1.78], + 'align_center': False, + 'feature_map_stride': 2, + 'matched_threshold': 0.6, + 'unmatched_threshold': 0.45 + } + ] + + TARGET_ASSIGNER_CONFIG: + NAME: AxisAlignedTargetAssigner + POS_FRACTION: -1.0 + SAMPLE_SIZE: 512 + NORM_BY_NUM_EXAMPLES: False + MATCH_HEIGHT: False + BOX_CODER: ResidualCoder + + LOSS_CONFIG: + LOSS_WEIGHTS: { + 'cls_weight': 1.0, + 'loc_weight': 2.0, + 'dir_weight': 0.2, + 'code_weights': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + } + + POST_PROCESSING: + RECALL_THRESH_LIST: [0.3, 0.5, 0.7] + SCORE_THRESH: 0.1 + OUTPUT_RAW_SCORE: False + + EVAL_METRIC: kitti + + NMS_CONFIG: + MULTI_CLASSES_NMS: False + NMS_TYPE: nms_gpu + NMS_THRESH: 0.01 + NMS_PRE_MAXSIZE: 4096 + NMS_POST_MAXSIZE: 500 + + +OPTIMIZATION: + BATCH_SIZE_PER_GPU: 2 + NUM_EPOCHS: 1 + + OPTIMIZER: adam_onecycle + LR: 0.003 + WEIGHT_DECAY: 0.01 + MOMENTUM: 0.9 + + MOMS: [0.95, 0.85] + PCT_START: 0.4 + DIV_FACTOR: 10 + DECAY_STEP_LIST: [35, 45] + LR_DECAY: 0.1 + LR_CLIP: 0.0000001 + + LR_WARMUP: False + WARMUP_EPOCH: 1 + + GRAD_NORM_CLIP: 10 diff --git a/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/prepare.sh b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/prepare.sh new file mode 100644 index 0000000000..1d33ceaa19 --- /dev/null +++ b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/prepare.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# install env +echo "*******prepare benchmark start ***********" +pip install -U pip +echo `pip --version` + + +cd .. +unset https_proxy && unset http_proxy +if [ ! -f "torch_dev_whls.tar" ];then + wget ${FLAG_TORCH_WHL_URL} +fi +export https_proxy=${HTTP_PRO} && export http_proxy=${HTTPS_PRO} +tar -xf torch_dev_whls.tar +for whl_file in torch_dev_whls/* +do + pip install ${whl_file} +done +pip install kornia +unset https_proxy && unset http_proxy +pip install spconv-cu117 +python setup.py develop + + +# 由于kitti数据集太大，为避免每次下载过于耗时，请将kitti数据集下载后，软链到/data/Dataset/kitti +mkdir -p /data/Dataset +if [ ! -d "/data/Dataset/KITTI_800" ]; then + cd /data/Dataset + wget https://paddle3d.bj.bcebos.com/TIPC/dataset/KITTI_800.tar --no-check-certificate + tar -xf KITTI_800.tar + cd - +fi +rm -rf data/KITTI_800 +ln -s /data/Dataset/KITTI_800 data/KITTI_800 +cd - +echo "*******prepare benchmark end***********" diff --git a/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/run_benchmark.sh b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/run_benchmark.sh new file mode 100644 index 0000000000..5c0d35f3f5 --- /dev/null +++ b/frame_benchmark/pytorch/dynamic/Paddle3D/scripts/pointpillars/benchmark_common/run_benchmark.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +# Test training benchmark for a model. + +# Usage: CUDA_VISIBLE_DEVICES=xxx bash run_benchmark.sh ${model_name} ${run_mode} ${fp_item} ${bs_item} ${max_iter} ${num_workers} + +function _set_params(){ + model_item=${1:-"pointpillars"} # (必选) 模型 item |fastscnn|segformer_b0| ocrnet_hrnetw48 + base_batch_size=${2:-"2"} # (必选) 每张卡上的batch_size + fp_item=${3:-"fp32"} # (必选) fp32|fp16 + run_mode=${4:-"DP"} # (必选) MP模型并行|DP数据并行|PP流水线并行|混合并行DP1-MP1-PP1|DP1-MP4-PP1 + device_num=${5:-"N1C4"} # (必选) 使用的卡数量，N1C1|N1C8|N4C8 （4机32卡） + profiling=${PROFILING:-"false"} # (必选) Profiling 开关，默认关闭，通过全局变量传递 + model_repo="OpenPCDet" # (必选) 模型套件的名字 + ips_unit="samples/sec" # (必选)速度指标单位 + skip_steps=10 # (必选)解析日志，跳过模型前几个性能不稳定的step + keyword="ips:" # (必选)解析日志，筛选出性能数据所在行的关键字 + + convergence_key="loss:" # (可选)解析日志，筛选出收敛数据所在行的关键字 如：convergence_key="loss:" + max_iter=${6:-"400"} # (可选) + num_workers=${7:-"16"} # (可选) + + # 以下为通用拼接log路径，无特殊可不用修改 + model_name=${model_item}_bs${base_batch_size}_${fp_item}_${run_mode} # (必填) 切格式不要改动,与平台页面展示对齐 + device=${CUDA_VISIBLE_DEVICES//,/ } + arr=(${device}) + num_gpu_devices=${#arr[*]} + run_log_path=${TRAIN_LOG_DIR:-$(pwd)} # （必填） TRAIN_LOG_DIR benchmark框架设置该参数为全局变量 + profiling_log_path=${PROFILING_LOG_DIR:-$(pwd)} # （必填） PROFILING_LOG_DIR benchmark框架设置该参数为全局变量 + speed_log_path=${LOG_PATH_INDEX_DIR:-$(pwd)} + train_log_file=${run_log_path}/${model_repo}_${model_name}_${device_num}_log + profiling_log_file=${profiling_log_path}/${model_repo}_${model_name}_${device_num}_profiling + speed_log_file=${speed_log_path}/${model_repo}_${model_name}_${device_num}_speed + if [ ${profiling} = "true" ];then + add_options="profiler_options=\"batch_range=[50, 60]; profile_path=model.profile\"" + log_file=${profiling_log_file} + else + add_options="" + log_file=${train_log_file} + fi +} + +function _analysis_log(){ + python analysis_log.py ${model_name} ${base_batch_size} ${log_file} ${speed_log_file} ${device_num} +} + +function _train(){ + batch_size=${base_batch_size} # 如果模型跑多卡但进程时,请在_train函数中计算出多卡需要的bs + + echo "current ${model_name} CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES, gpus=${device_num}, batch_size=${batch_size}" + + train_config="${model_item}.yaml" + + use_com_args="" + if [ ${FLAG_TORCH_COMPILE} = "True" ];then + use_com_args="--torchcompile" + fi + use_amp="" + if [ ${fp_item} = "fp16" ];then + use_amp="--use_amp" + fi + + gpus=${device_num:3:1} + train_options="--cfg_file=${train_config} + --epochs=${gpus} + --workers=${num_workers} + --logger_iter_interval=10 + --num_epochs_to_eval=10000 + --ckpt_save_interval=10000 + ${use_amp} + --set OPTIMIZATION.BATCH_SIZE_PER_GPU ${batch_size}" + + if [ ${device_num} = "N1C1" ]; then + train_cmd="python train.py ${train_options}" + else + train_cmd="bash scripts/dist_train.sh ${gpus} ${train_options}" + fi + +# 以下为通用执行命令，无特殊可不用修改 + echo ${train_cmd} + timeout 15m ${train_cmd} > ${log_file} 2>&1 + if [ $? -ne 0 ];then + echo -e "${model_name}, FAIL" + else + echo -e "${model_name}, SUCCESS" + fi + kill -9 `ps -ef|grep 'python'|awk '{print $2}'` + if [ ${device_num} != "N1C1" -a -d mylog ]; then + rm ${log_file} + cp mylog/workerlog.0 ${log_file} + fi +} + +_set_params $@ +export model_branch=`git symbolic-ref HEAD 2>/dev/null | cut -d"/" -f 3` +export model_commit=$(git log|head -n1|awk '{print $2}') + +# clear share memory +rm -rf /dev/shm/* + +export frame_version=`python -c "import torch;print(torch.__version__)"` +echo "---------frame_version is torch ${frame_version}" +echo "---------model_branch is ${model_branch}" +echo "---------model_commit is ${model_commit}" + +job_bt=`date '+%Y%m%d%H%M%S'` +_train +job_et=`date '+%Y%m%d%H%M%S'` +export model_run_time=$((${job_et}-${job_bt})) +_analysis_log + +# kill all python processes +ps -ef | grep python | awk '{ print $2 }' | xargs kill -9