Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Bug Fix] release task scripts #411

Merged
merged 8 commits into from
Oct 22, 2022
Merged

[Bug Fix] release task scripts #411

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
db536d8
Update py_run.bat
felixhjh Oct 14, 2022
7f6948b
Update cpp_run.bat
felixhjh Oct 14, 2022
aa1c2e2
Update compare_with_gt.py
felixhjh Oct 14, 2022
9d62aef
Update cpp_run.bat
felixhjh Oct 14, 2022
d625c9d
Merge branch 'PaddlePaddle:develop' into publish_task
felixhjh Oct 20, 2022
dddd29c
Merge branch 'PaddlePaddle:develop' into publish_task
felixhjh Oct 21, 2022
e9bd4fa
Update release task scripts according to diffrent platforms
felixhjh Oct 21, 2022
053970c
Delete CMAKE_CXX_COMPILER in cpp_run.bat
felixhjh Oct 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 39 additions & 3 deletions tests/release_task/compare_with_gt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,39 @@
import numpy as np
import re

diff_score_threshold = {
"linux-x64": {
"label_diff": 0,
"score_diff": 1e-4,
"boxes_diff_ratio": 1e-4,
"boxes_diff": 1e-3
},
"linux-aarch64": {
"label_diff": 0,
"score_diff": 1e-4,
"boxes_diff_ratio": 1e-4,
"boxes_diff": 1e-3
},
"osx-x86_64": {
"label_diff": 0,
"score_diff": 1e-4,
"boxes_diff_ratio": 2e-4,
"boxes_diff": 1e-3
},
"osx-arm64": {
"label_diff": 0,
"score_diff": 1e-4,
"boxes_diff_ratio": 2e-4,
"boxes_diff": 1e-3
},
"win-x64": {
"label_diff": 0,
"score_diff": 5e-4,
"boxes_diff_ratio": 1e-3,
"boxes_diff": 1e-3
}
}


def parse_arguments():
import argparse
Expand Down Expand Up @@ -59,6 +92,7 @@ def save_numpy_result(file_path, error_msg):


def check_result(gt_result, infer_result, args):
platform = args.platform
if len(gt_result) != len(infer_result):
infer_result = infer_result[-len(gt_result):]
diff = np.abs(gt_result - infer_result)
Expand All @@ -68,17 +102,19 @@ def check_result(gt_result, infer_result, args):
boxes_diff_ratio = boxes_diff / (infer_result[:, :-2] + 1e-6)
is_diff = False
backend = args.result_path.split(".")[0]
if (label_diff > 0).any():
if (label_diff > diff_score_threshold[platform]["label_diff"]).any():
print(args.platform, args.device, "label diff ", label_diff)
is_diff = True
label_diff_bool_file = args.platform + "_" + backend + "_" + "label_diff_bool.txt"
save_numpy_result(label_diff_bool_file, label_diff > 0)
if (score_diff > 2e-4).any():
if (score_diff > diff_score_threshold[platform]["score_diff"]).any():
print(args.platform, args.device, "score diff ", score_diff)
is_diff = True
score_diff_bool_file = args.platform + "_" + backend + "_" + "score_diff_bool.txt"
save_numpy_result(score_diff_bool_file, score_diff > 1e-4)
if (boxes_diff_ratio > 1e-4).any() and (boxes_diff > 1e-3).any():
if (boxes_diff_ratio > diff_score_threshold[platform]["boxes_diff_ratio"]
).any() and (
boxes_diff > diff_score_threshold[platform]["boxes_diff"]).any():
print(args.platform, args.device, "boxes diff ", boxes_diff_ratio)
is_diff = True
boxes_diff_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_bool.txt"
Expand Down
15 changes: 6 additions & 9 deletions tests/release_task/cpp_run.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ if "%DEVICE%" == "gpu" (
set CPP_FASTDEPLOY_PACKAGE=fastdeploy-%PLATFORM%-%DEVICE%-%VERSION%
set RUN_CASES=ort paddle trt
) else (
set CPP_FASTDEPLOY_PACKAGE=fastdeploy-python
set CPP_FASTDEPLOY_PACKAGE=fastdeploy-%PLATFORM%-%VERSION%
set RUN_CASES=ort paddle openvino
)

Expand All @@ -36,7 +36,7 @@ set COMPARE_SHELL=%CURRENT_DIR%\compare_with_gt.py
python -c "from download import *; download_and_decompress('https://fastdeploy.bj.bcebos.com/dev/cpp/%CPP_FASTDEPLOY_PACKAGE%.zip', './')"

mkdir build && cd build
cmake .. -G "Visual Studio 16 2019" -A x64 -DFASTDEPLOY_INSTALL_DIR=%cd%\..\%CPP_FASTDEPLOY_PACKAGE% -DCUDA_DIRECTORY="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2" -DCMAKE_CXX_COMPILER=%CMAKE_CXX_COMPILER%
cmake .. -G "Visual Studio 16 2019" -A x64 -DFASTDEPLOY_INSTALL_DIR=%cd%\..\%CPP_FASTDEPLOY_PACKAGE% -DCUDA_DIRECTORY="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2"

msbuild infer_demo.sln /m:4 /p:Configuration=Release /p:Platform=x64

Expand All @@ -47,7 +47,6 @@ echo "FASTDEPLOY_HOME" %FASTDEPLOY_HOME%
copy /Y %FASTDEPLOY_HOME%\third_libs\install\onnxruntime\lib\onnxruntime* Release\
set PATH=%FASTDEPLOY_HOME%\lib;%PATH%
set PATH=%FASTDEPLOY_HOME%\third_libs\install\onnxruntime\lib;%PATH%
set PATH=%FASTDEPLOY_HOME%\third_libs\install\opencv-win-x64-3.4.16\build\x64\vc15\bin;%PATH%
set PATH=%FASTDEPLOY_HOME%\third_libs\install\paddle_inference\paddle\lib;%PATH%
set PATH=%FASTDEPLOY_HOME%\third_libs\install\paddle_inference\third_party\install\mkldnn\lib;%PATH%
set PATH=%FASTDEPLOY_HOME%\third_libs\install\paddle_inference\third_party\install\mklml\lib;%PATH%
Expand All @@ -56,9 +55,9 @@ set PATH=%FASTDEPLOY_HOME%\third_libs\install\tensorrt\lib;%PATH%
set PATH=%FASTDEPLOY_HOME%\third_libs\install\faster_tokenizer\lib;%PATH%
set PATH=%FASTDEPLOY_HOME%\third_libs\install\faster_tokenizer\third_party\lib;%PATH%
set PATH=%FASTDEPLOY_HOME%\third_libs\install\yaml-cpp\lib;%PATH%
set PATH=%FASTDEPLOY_HOME%\third_libs\install\openvino\bin;%PATH%
set PATH=%FASTDEPLOY_HOME%\third_libs\install\openvino\3rdparty\tbb\bin;%PATH%

set PATH=%FASTDEPLOY_HOME%\third_libs\install\opencv\build\x64\vc15\bin;%PATH%
set PATH=%FASTDEPLOY_HOME%\third_libs\install\openvino\runtime\bin;%PATH%
set PATH=%FASTDEPLOY_HOME%\third_libs\install\openvino\runtime\3rdparty\tbb\bin;%PATH%
echo "set path done"
cd %cd%\Release
for %%b in (%RUN_CASES%) do (
Expand All @@ -71,11 +70,9 @@ for %%b in (%RUN_CASES%) do (
if %%b == trt (
infer_ppyoloe_demo.exe --model_dir=%MODEL_PATH% --image_file=%IMAGE_PATH% --device=gpu --backend=%%b >> cpp_%%b_trt_result.txt
python %COMPARE_SHELL% --gt_path %GROUND_TRUTH_PATH% --result_path cpp_%%b_trt_result.txt --platform %PLATFORM% --device trt --conf_threshold 0.5
) else if %%b == ort (
) else (
infer_ppyoloe_demo.exe --model_dir=%MODEL_PATH% --image_file=%IMAGE_PATH% --device=gpu --backend=%%b >> cpp_%%b_gpu_result.txt
python %COMPARE_SHELL% --gt_path %GROUND_TRUTH_PATH% --result_path cpp_%%b_gpu_result.txt --platform %PLATFORM% --device gpu --conf_threshold 0.5
) else if %%b == paddle (
echo "Temporarily skip paddle gpu case in windows for Inaccurate inference precision"
)
)
)
Expand Down
12 changes: 7 additions & 5 deletions tests/release_task/cpp_run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ fi
echo $CPP_FASTDEPLOY_PACKAGE
LINUX_X64_GPU_CASE=('ort' 'paddle' 'trt')
LINUX_X64_CPU_CASE=('ort' 'paddle' 'openvino')
LINUX_AARCH_CPU_CASE=('ort' 'openvino')
#LINUX_AARCH_CPU_CASE=('ort' 'openvino')
LINUX_AARCH_CPU_CASE=('ort')
MACOS_INTEL_CPU_CASE=('ort' 'paddle' 'openvino')
MACOS_ARM64_CPU_CASE=('default')
wget -q https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz
Expand All @@ -25,6 +26,7 @@ GROUND_TRUTH_PATH=$CURRENT_DIR/release_task_groud_truth_result.txt
COMPARE_SHELL=$CURRENT_DIR/compare_with_gt.py

RUN_CASE=()
CONF_THRESHOLD=0
if [ "$DEVICE" = "gpu" ] && [ "$PLATFORM" = "linux-x64" ];then
RUN_CASE=(${LINUX_X64_GPU_CASE[*]})
elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "linux-x64" ];then
Expand All @@ -35,6 +37,7 @@ elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "osx-x86_64" ];then
RUN_CASE=(${MACOS_INTEL_CPU_CASE[*]})
elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "osx-arm64" ];then
RUN_CASE=(${MACOS_ARM64_CPU_CASE[*]})
CONF_THRESHOLD=0.5
fi

case_number=${#RUN_CASE[@]}
Expand All @@ -52,16 +55,16 @@ do
echo "Cpp Backend:" $backend
if [ "$backend" != "trt" ];then
./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=cpu --backend=$backend >> cpp_cpu_result.txt
python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_cpu_result.txt --platform $PLATFORM --device cpu
python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_cpu_result.txt --platform $PLATFORM --device cpu --conf_threshold $CONF_THRESHOLD
fi
if [ "$DEVICE" = "gpu" ];then

if [ "$backend" = "trt" ];then
./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=gpu --backend=$backend >> cpp_trt_result.txt
python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_trt_result.txt --platform $PLATFORM --device trt
python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_trt_result.txt --platform $PLATFORM --device trt --conf_threshold $CONF_THRESHOLD
else
./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=gpu --backend=$backend >> cpp_gpu_result.txt
python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_gpu_result.txt --platform $PLATFORM --device gpu
python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_gpu_result.txt --platform $PLATFORM --device gpu --conf_threshold $CONF_THRESHOLD
fi
fi
done
Expand All @@ -78,4 +81,3 @@ else
cat $res_file
exit -1
fi

6 changes: 2 additions & 4 deletions tests/release_task/py_run.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,10 @@ for %%b in (%RUN_CASES%) do (
if %%b == trt (
python infer_ppyoloe.py --model_dir=%MODEL_PATH% --image=%IMAGE_PATH% --device=gpu --backend=%%b >> py_%%b_trt_result.txt
python %COMPARE_SHELL% --gt_path %GROUND_TRUTH_PATH% --result_path py_%%b_trt_result.txt --platform %PLATFORM% --device trt --conf_threshold 0.5
) else if %%b == ort (
) else (
python infer_ppyoloe.py --model_dir=%MODEL_PATH% --image=%IMAGE_PATH% --device=gpu --backend=%%b >> py_%%b_gpu_result.txt
python %COMPARE_SHELL% --gt_path %GROUND_TRUTH_PATH% --result_path py_%%b_gpu_result.txt --platform %PLATFORM% --device gpu --conf_threshold 0.5
) else if %%b == paddle (
echo "Temporarily skip paddle gpu case in windows for Inaccurate inference precision"
)
)
)
)

Expand Down
5 changes: 3 additions & 2 deletions tests/release_task/py_run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ fi
echo $PY_FASTDEPLOY_PACKAGE
LINUX_X64_GPU_CASE=('ort' 'paddle' 'trt')
LINUX_X64_CPU_CASE=('ort' 'paddle' 'openvino')
LINUX_AARCH_CPU_CASE=('ort' 'openvino')
#LINUX_AARCH_CPU_CASE=('ort' 'openvino')
LINUX_AARCH_CPU_CASE=('ort')
MACOS_INTEL_CPU_CASE=('ort' 'paddle' 'openvino')
MACOS_ARM64_CPU_CASE=('default')
wget -q https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz
Expand Down Expand Up @@ -60,7 +61,7 @@ do
fi
done

ret=$?
ret=$?

res_file="result.txt"
if [ ! -f $res_file ];then
Expand Down