From e4f58e972c39f6a6a455190ba327487324edede3 Mon Sep 17 00:00:00 2001 From: DefTruth Date: Mon, 28 Nov 2022 21:55:47 +0800 Subject: [PATCH 1/6] [Android] Update ppseg jni via new api and optimize jni vis funcs --- .../SegmentationMainActivity.java | 4 +- .../cpp/fastdeploy_jni/vision/results_jni.cc | 38 ++-- .../segmentation/paddleseg_model_jni.cc | 2 +- .../fastdeploy_jni/vision/visualize_jni.cc | 165 +++++++++++++++--- 4 files changed, 159 insertions(+), 50 deletions(-) diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java index 88b3c3a5b4..3223fc07fc 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java @@ -250,12 +250,10 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { } boolean modified = false; - - long tc = System.currentTimeMillis(); - SegmentationResult result = new SegmentationResult(); result.setCxxBufferFlag(true); + long tc = System.currentTimeMillis(); predictor.predict(ARGB8888ImageBitmap, result); timeElapsed += (System.currentTimeMillis() - tc); diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc index 8f6199465d..4a9f0cd6e0 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc @@ -859,6 +859,13 @@ bool AllocateSegmentationResultFromJava( return false; } + // mInitialized boolean: Z + jboolean j_seg_initialized = + env->GetBooleanField(j_seg_result_obj, j_seg_initialized_id_cc); + if (j_seg_initialized == JNI_FALSE) { + return false; + } + // If 'mEnableCxxBuffer' set as true, then, we only Allocate from // cxx context to cxx result. Some users may want to use this // method to boost the performance of segmentation. @@ -872,32 +879,23 @@ bool AllocateSegmentationResultFromJava( } // Allocate from cxx context to cxx result auto c_cxx_buffer = reinterpret_cast(j_cxx_buffer); - // TODO: May use 'swap' to exchange the administrative privileges ? - // c_result_ptr->shape.swap(c_cxx_buffer->shape); - // c_result_ptr->label_map.swap(c_cxx_buffer->label_map); - // c_result_ptr->contain_score_map = c_cxx_buffer->contain_score_map; - // if (c_cxx_buffer->contain_score_map) { - // c_result_ptr->score_map.swap(c_cxx_buffer->score_map); - // } - c_result_ptr->shape.assign( - c_cxx_buffer->shape.begin(), c_cxx_buffer->shape.end()); - c_result_ptr->label_map.assign( - c_cxx_buffer->label_map.begin(), c_cxx_buffer->label_map.end()); + + // (*c_result_ptr) = std::move(*c_cxx_buffer); + c_result_ptr->shape = c_cxx_buffer->shape; + const size_t label_len = c_cxx_buffer->label_map.size(); + c_result_ptr->label_map.resize(label_len); + std::memcpy(c_result_ptr->label_map.data(), c_cxx_buffer->label_map.data(), + label_len * sizeof(uint8_t)); c_result_ptr->contain_score_map = c_cxx_buffer->contain_score_map; if (c_cxx_buffer->contain_score_map) { - c_result_ptr->score_map.assign( - c_cxx_buffer->score_map.begin(), c_cxx_buffer->score_map.end()); + const size_t score_len = c_cxx_buffer->score_map.size(); + c_result_ptr->score_map.resize(score_len); + std::memcpy(c_result_ptr->score_map.data(), c_cxx_buffer->score_map.data(), + score_len * sizeof(float)); } return true; } - // mInitialized boolean: Z - jboolean j_seg_initialized = - env->GetBooleanField(j_seg_result_obj, j_seg_initialized_id_cc); - if (j_seg_initialized == JNI_FALSE) { - return false; - } - jbyteArray j_seg_label_map_byte_arr = reinterpret_cast( env->GetObjectField(j_seg_result_obj, j_seg_label_map_id_cc)); jlongArray j_seg_shape_long_arr = reinterpret_cast( diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc index 44686076a8..69ded0f4df 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc @@ -49,7 +49,7 @@ Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_bindNative( jboolean j_is_vertical_screen = env->GetBooleanField( thiz, j_is_vertical_screen_id); bool c_is_vertical_screen = static_cast(j_is_vertical_screen); - c_model_ptr->is_vertical_screen = c_is_vertical_screen; + c_model_ptr->GetPreprocessor().SetIsVerticalScreen(c_is_vertical_screen); env->DeleteLocalRef(j_ppseg_clazz); vision::EnableFlyCV(); diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc index 0803def582..c35fca9e78 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. #include // NOLINT +#include "fastdeploy_jni/perf_jni.h" // NOLINT #include "fastdeploy_jni/bitmap_jni.h" // NOLINT #include "fastdeploy_jni/convert_jni.h" // NOLINT #include "fastdeploy_jni/vision/results_jni.h" // NOLINT @@ -19,16 +20,19 @@ namespace fni = fastdeploy::jni; namespace vision = fastdeploy::vision; -#ifdef __cplusplus -extern "C" { -#endif +namespace fastdeploy { +namespace jni { -/// VisClassification -JNIEXPORT jboolean JNICALL -Java_com_baidu_paddle_fastdeploy_vision_Visualize_visClassificationNative( - JNIEnv *env, jclass clazz, jobject argb8888_bitmap, +/// Some visualize helpers. +jboolean VisClassificationFromJava( + JNIEnv *env, jobject argb8888_bitmap, jobject result, jfloat score_threshold, jfloat font_size, jobjectArray labels) { + const jclass j_cls_result_clazz = env->FindClass( + "com/baidu/paddle/fastdeploy/vision/ClassifyResult"); + if (!env->IsInstanceOf(result, j_cls_result_clazz)) { + return JNI_FALSE; + } vision::ClassifyResult c_result; if (!fni::AllocateCxxResultFromJava( env, result, reinterpret_cast(&c_result), @@ -56,19 +60,21 @@ Java_com_baidu_paddle_fastdeploy_vision_Visualize_visClassificationNative( return JNI_TRUE; } -/// VisDetection -JNIEXPORT jboolean JNICALL -Java_com_baidu_paddle_fastdeploy_vision_Visualize_visDetectionNative( - JNIEnv *env, jclass clazz, jobject argb8888_bitmap, +jboolean VisDetectionFromJava( + JNIEnv *env, jobject argb8888_bitmap, jobject result, jfloat score_threshold, jint line_size, jfloat font_size, jobjectArray labels) { + const jclass j_det_result_clazz = env->FindClass( + "com/baidu/paddle/fastdeploy/vision/DetectionResult"); + if (!env->IsInstanceOf(result, j_det_result_clazz)) { + return JNI_FALSE; + } vision::DetectionResult c_result; if (!fni::AllocateCxxResultFromJava( env, result, reinterpret_cast(&c_result), vision::ResultType::DETECTION)) { return JNI_FALSE; } - // Get labels from Java [n] auto c_labels = fni::ConvertTo>(env, labels); @@ -90,11 +96,13 @@ Java_com_baidu_paddle_fastdeploy_vision_Visualize_visDetectionNative( return JNI_TRUE; } -/// VisOcr -JNIEXPORT jboolean JNICALL -Java_com_baidu_paddle_fastdeploy_vision_Visualize_visOcrNative( - JNIEnv *env, jclass clazz, jobject argb8888_bitmap, - jobject result) { +jboolean VisOcrFromJava( + JNIEnv *env, jobject argb8888_bitmap, jobject result) { + const jclass j_ocr_result_clazz = env->FindClass( + "com/baidu/paddle/fastdeploy/vision/OCRResult"); + if (!env->IsInstanceOf(result, j_ocr_result_clazz)) { + return JNI_FALSE; + } vision::OCRResult c_result; if (!fni::AllocateCxxResultFromJava( env, result, reinterpret_cast(&c_result), @@ -113,17 +121,20 @@ Java_com_baidu_paddle_fastdeploy_vision_Visualize_visOcrNative( return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_baidu_paddle_fastdeploy_vision_Visualize_visSegmentationNative( - JNIEnv *env, jclass clazz, jobject argb8888_bitmap, - jobject result, jfloat weight) { +jboolean VisSegmentationFromJava( + JNIEnv *env, jobject argb8888_bitmap, jobject result, jfloat weight) { + const jclass j_seg_result_clazz = env->FindClass( + "com/baidu/paddle/fastdeploy/vision/SegmentationResult"); + if (!env->IsInstanceOf(result, j_seg_result_clazz)) { + return JNI_FALSE; + } + // Allocate from Java result, may cost some times. vision::SegmentationResult c_result; if (!fni::AllocateCxxResultFromJava( env, result, reinterpret_cast(&c_result), vision::ResultType::SEGMENTATION)) { return JNI_FALSE; } - cv::Mat c_bgr; if (!fni::ARGB888Bitmap2BGR(env, argb8888_bitmap, &c_bgr)) { return JNI_FALSE; @@ -135,17 +146,61 @@ Java_com_baidu_paddle_fastdeploy_vision_Visualize_visSegmentationNative( return JNI_TRUE; } -JNIEXPORT jboolean JNICALL -Java_com_baidu_paddle_fastdeploy_vision_Visualize_visFaceDetectionNative( - JNIEnv *env, jclass clazz, jobject argb8888_bitmap, +jboolean VisSegmentationFromCxxBuffer( + JNIEnv *env, jobject argb8888_bitmap, jobject result, jfloat weight) { + const jclass j_seg_result_clazz = env->FindClass( + "com/baidu/paddle/fastdeploy/vision/SegmentationResult"); + if (!env->IsInstanceOf(result, j_seg_result_clazz)) { + return JNI_FALSE; + } + const jfieldID j_enable_cxx_buffer_id = env->GetFieldID( + j_seg_result_clazz, "mEnableCxxBuffer", "Z"); + const jfieldID j_cxx_buffer_id = env->GetFieldID( + j_seg_result_clazz, "mCxxBuffer", "J"); + const jfieldID j_seg_initialized_id = env->GetFieldID( + j_seg_result_clazz, "mInitialized", "Z"); + jboolean j_enable_cxx_buffer = + env->GetBooleanField(result, j_enable_cxx_buffer_id); + jboolean j_seg_initialized = + env->GetBooleanField(result, j_seg_initialized_id); + if (j_seg_initialized == JNI_FALSE) { + return JNI_FALSE; + } + // Use CxxBuffer directly without any copy. + if (j_enable_cxx_buffer == JNI_TRUE) { + jlong j_cxx_buffer = env->GetLongField(result, j_cxx_buffer_id); + if (j_cxx_buffer == 0) { + return JNI_FALSE; + } + // Allocate from cxx context to cxx result + auto c_cxx_buffer = reinterpret_cast(j_cxx_buffer); + cv::Mat c_bgr; + if (!fni::ARGB888Bitmap2BGR(env, argb8888_bitmap, &c_bgr)) { + return JNI_FALSE; + } + auto c_vis_im = vision::VisSegmentation(c_bgr, *c_cxx_buffer, weight); + if (!fni::BGR2ARGB888Bitmap(env, argb8888_bitmap, c_vis_im)) { + return JNI_FALSE; + } + return JNI_TRUE; + } + return JNI_FALSE; +} + +jboolean VisFaceDetectionFromJava( + JNIEnv *env, jobject argb8888_bitmap, jobject result, jint line_size, jfloat font_size) { + const jclass j_face_det_result_clazz = env->FindClass( + "com/baidu/paddle/fastdeploy/vision/FaceDetectionResult"); + if (!env->IsInstanceOf(result, j_face_det_result_clazz)) { + return JNI_FALSE; + } vision::FaceDetectionResult c_result; if (!fni::AllocateCxxResultFromJava( env, result, reinterpret_cast(&c_result), vision::ResultType::FACE_DETECTION)) { return JNI_FALSE; } - cv::Mat c_bgr; if (!fni::ARGB888Bitmap2BGR(env, argb8888_bitmap, &c_bgr)) { return JNI_FALSE; @@ -157,6 +212,64 @@ Java_com_baidu_paddle_fastdeploy_vision_Visualize_visFaceDetectionNative( return JNI_TRUE; } +} // jni +} // fastdeploy + +#ifdef __cplusplus +extern "C" { +#endif + +/// VisClassification +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_Visualize_visClassificationNative( + JNIEnv *env, jclass clazz, jobject argb8888_bitmap, + jobject result, jfloat score_threshold, jfloat font_size, + jobjectArray labels) { + return fni::VisClassificationFromJava(env, argb8888_bitmap, result, + score_threshold, font_size, labels); +} + +/// VisDetection +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_Visualize_visDetectionNative( + JNIEnv *env, jclass clazz, jobject argb8888_bitmap, + jobject result, jfloat score_threshold, jint line_size, + jfloat font_size, jobjectArray labels) { + return fni::VisDetectionFromJava(env, argb8888_bitmap, result, score_threshold, + line_size, font_size, labels); +} + +/// VisOcr +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_Visualize_visOcrNative( + JNIEnv *env, jclass clazz, jobject argb8888_bitmap, + jobject result) { + return fni::VisOcrFromJava(env, argb8888_bitmap, result); +} + +/// VisSegmentation +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_Visualize_visSegmentationNative( + JNIEnv *env, jclass clazz, jobject argb8888_bitmap, + jobject result, jfloat weight) { + // First, try visualize segmentation result via CxxBuffer. + if (fni::VisSegmentationFromCxxBuffer( + env, argb8888_bitmap, result, weight)) { + return JNI_TRUE; + } + // Then, try visualize segmentation from Java result(may cost some times). + return fni::VisSegmentationFromJava(env, argb8888_bitmap, result, weight); +} + +/// VisFaceDetection +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_Visualize_visFaceDetectionNative( + JNIEnv *env, jclass clazz, jobject argb8888_bitmap, + jobject result, jint line_size, jfloat font_size) { + return fni::VisFaceDetectionFromJava(env, argb8888_bitmap, result, + line_size, font_size); +} + #ifdef __cplusplus } #endif From 4e21aa88e336f0b3d8d5ad4b3c469d395ffc88ab Mon Sep 17 00:00:00 2001 From: DefTruth Date: Mon, 28 Nov 2022 23:07:05 +0800 Subject: [PATCH 2/6] delete local refs --- .../main/cpp/fastdeploy_jni/vision/visualize_jni.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc index c35fca9e78..354cdd89c7 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc @@ -31,8 +31,10 @@ jboolean VisClassificationFromJava( const jclass j_cls_result_clazz = env->FindClass( "com/baidu/paddle/fastdeploy/vision/ClassifyResult"); if (!env->IsInstanceOf(result, j_cls_result_clazz)) { + env->DeleteLocalRef(j_cls_result_clazz); return JNI_FALSE; } + env->DeleteLocalRef(j_cls_result_clazz); vision::ClassifyResult c_result; if (!fni::AllocateCxxResultFromJava( env, result, reinterpret_cast(&c_result), @@ -67,8 +69,10 @@ jboolean VisDetectionFromJava( const jclass j_det_result_clazz = env->FindClass( "com/baidu/paddle/fastdeploy/vision/DetectionResult"); if (!env->IsInstanceOf(result, j_det_result_clazz)) { + env->DeleteLocalRef(j_det_result_clazz); return JNI_FALSE; } + env->DeleteLocalRef(j_det_result_clazz); vision::DetectionResult c_result; if (!fni::AllocateCxxResultFromJava( env, result, reinterpret_cast(&c_result), @@ -101,8 +105,10 @@ jboolean VisOcrFromJava( const jclass j_ocr_result_clazz = env->FindClass( "com/baidu/paddle/fastdeploy/vision/OCRResult"); if (!env->IsInstanceOf(result, j_ocr_result_clazz)) { + env->DeleteLocalRef(j_ocr_result_clazz); return JNI_FALSE; } + env->DeleteLocalRef(j_ocr_result_clazz); vision::OCRResult c_result; if (!fni::AllocateCxxResultFromJava( env, result, reinterpret_cast(&c_result), @@ -126,8 +132,10 @@ jboolean VisSegmentationFromJava( const jclass j_seg_result_clazz = env->FindClass( "com/baidu/paddle/fastdeploy/vision/SegmentationResult"); if (!env->IsInstanceOf(result, j_seg_result_clazz)) { + env->DeleteLocalRef(j_seg_result_clazz); return JNI_FALSE; } + env->DeleteLocalRef(j_seg_result_clazz); // Allocate from Java result, may cost some times. vision::SegmentationResult c_result; if (!fni::AllocateCxxResultFromJava( @@ -151,6 +159,7 @@ jboolean VisSegmentationFromCxxBuffer( const jclass j_seg_result_clazz = env->FindClass( "com/baidu/paddle/fastdeploy/vision/SegmentationResult"); if (!env->IsInstanceOf(result, j_seg_result_clazz)) { + env->DeleteLocalRef(j_seg_result_clazz); return JNI_FALSE; } const jfieldID j_enable_cxx_buffer_id = env->GetFieldID( @@ -163,6 +172,8 @@ jboolean VisSegmentationFromCxxBuffer( env->GetBooleanField(result, j_enable_cxx_buffer_id); jboolean j_seg_initialized = env->GetBooleanField(result, j_seg_initialized_id); + + env->DeleteLocalRef(j_seg_result_clazz); if (j_seg_initialized == JNI_FALSE) { return JNI_FALSE; } @@ -193,8 +204,10 @@ jboolean VisFaceDetectionFromJava( const jclass j_face_det_result_clazz = env->FindClass( "com/baidu/paddle/fastdeploy/vision/FaceDetectionResult"); if (!env->IsInstanceOf(result, j_face_det_result_clazz)) { + env->DeleteLocalRef(j_face_det_result_clazz); return JNI_FALSE; } + env->DeleteLocalRef(j_face_det_result_clazz); vision::FaceDetectionResult c_result; if (!fni::AllocateCxxResultFromJava( env, result, reinterpret_cast(&c_result), From de745459fdcfe19d0fd8179cc98a70f9d9849b73 Mon Sep 17 00:00:00 2001 From: DefTruth Date: Tue, 29 Nov 2022 21:51:36 +0800 Subject: [PATCH 3/6] [Android] Add PPTinyPose jni and java api --- .../fastdeploy/src/main/cpp/CMakeLists.txt | 2 + .../src/main/cpp/fastdeploy_jni/bitmap_jni.cc | 37 ++-- .../cpp/fastdeploy_jni/runtime_option_jni.cc | 8 + .../keypointdetection_utils_jni.cc | 45 +++++ .../keypointdetection_utils_jni.h | 32 +++ .../keypointdetection/pptinypose_jni.cc | 103 ++++++++++ .../cpp/fastdeploy_jni/vision/results_jni.cc | 188 +++++++++++++++++- .../segmentation/paddleseg_model_jni.cc | 4 +- .../fastdeploy_jni/vision/visualize_jni.cc | 36 +++- .../paddle/fastdeploy/RuntimeOption.java | 10 + .../vision/KeyPointDetectionResult.java | 42 ++++ .../paddle/fastdeploy/vision/Visualize.java | 23 +++ .../vision/keypointdetection/PPTinyPose.java | 152 ++++++++++++++ 13 files changed, 661 insertions(+), 21 deletions(-) create mode 100644 java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.cc create mode 100644 java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.h create mode 100644 java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/pptinypose_jni.cc create mode 100644 java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/KeyPointDetectionResult.java create mode 100644 java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/keypointdetection/PPTinyPose.java diff --git a/java/android/fastdeploy/src/main/cpp/CMakeLists.txt b/java/android/fastdeploy/src/main/cpp/CMakeLists.txt index a8766d9ae4..6f22ece177 100644 --- a/java/android/fastdeploy/src/main/cpp/CMakeLists.txt +++ b/java/android/fastdeploy/src/main/cpp/CMakeLists.txt @@ -43,6 +43,8 @@ add_library( fastdeploy_jni/vision/facedet/scrfd_jni.cc fastdeploy_jni/vision/facedet/yolov5face_jni.cc fastdeploy_jni/vision/facedet/facedet_utils_jni.cc + fastdeploy_jni/vision/keypointdetection/pptinypose_jni.cc + fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.cc ) # Searches for a specified prebuilt library and stores the path as a diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/bitmap_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/bitmap_jni.cc index a5a2aca328..6bbc89d03f 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/bitmap_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/bitmap_jni.cc @@ -19,33 +19,42 @@ namespace fastdeploy { namespace jni { -jboolean ARGB888Bitmap2RGBA(JNIEnv *env, jobject j_argb8888_bitmap, - cv::Mat *c_rgba) { - // Convert the android bitmap(ARGB8888) to the OpenCV RGBA image. Actually, - // the data layout of ARGB8888 is R, G, B, A, it's the same as CV RGBA image, - // so it is unnecessary to do the conversion of color format, check - // https://developer.android.com/reference/android/graphics/Bitmap.Config#ARGB_8888 - // to get the more details about Bitmap.Config.ARGB8888 +cv::Mat CreateZeroCopyRGBAFromBitmap(JNIEnv *env, jobject j_argb8888_bitmap) { + cv::Mat c_rgba; AndroidBitmapInfo j_bitmap_info; if (AndroidBitmap_getInfo(env, j_argb8888_bitmap, &j_bitmap_info) < 0) { LOGE("Invoke AndroidBitmap_getInfo() failed!"); - return JNI_FALSE; + return c_rgba; } if (j_bitmap_info.format != ANDROID_BITMAP_FORMAT_RGBA_8888) { LOGE("Only Bitmap.Config.ARGB8888 color format is supported!"); - return JNI_FALSE; + return c_rgba; } void *j_bitmap_pixels; if (AndroidBitmap_lockPixels(env, j_argb8888_bitmap, &j_bitmap_pixels) < 0) { LOGE("Invoke AndroidBitmap_lockPixels() failed!"); - return JNI_FALSE; + return c_rgba; } cv::Mat j_bitmap_im(static_cast(j_bitmap_info.height), static_cast(j_bitmap_info.width), CV_8UC4, - j_bitmap_pixels); - j_bitmap_im.copyTo(*(c_rgba)); + j_bitmap_pixels); // no copied. + c_rgba = j_bitmap_im; // ref only. if (AndroidBitmap_unlockPixels(env, j_argb8888_bitmap) < 0) { LOGE("Invoke AndroidBitmap_unlockPixels() failed!"); + return c_rgba; + } + return c_rgba; +} + +jboolean ARGB888Bitmap2RGBA(JNIEnv *env, jobject j_argb8888_bitmap, + cv::Mat *c_rgba) { + // Convert the android bitmap(ARGB8888) to the OpenCV RGBA image. Actually, + // the data layout of ARGB8888 is R, G, B, A, it's the same as CV RGBA image, + // so it is unnecessary to do the conversion of color format, check + // https://developer.android.com/reference/android/graphics/Bitmap.Config#ARGB_8888 + // to get the more details about Bitmap.Config.ARGB8888 + *c_rgba = CreateZeroCopyRGBAFromBitmap(env, j_argb8888_bitmap); + if (c_rgba->empty()) { return JNI_FALSE; } return JNI_TRUE; @@ -57,6 +66,8 @@ jboolean ARGB888Bitmap2BGR(JNIEnv *env, jobject j_argb8888_bitmap, if (!ARGB888Bitmap2RGBA(env, j_argb8888_bitmap, &c_rgba)) { return JNI_FALSE; } + // TODO: Use the neon instruction to optimize this conversion. + // COLOR_RGBA2BGR will allocate memories for new mat. cv::cvtColor(c_rgba, *(c_bgr), cv::COLOR_RGBA2BGR); return JNI_TRUE; } @@ -73,9 +84,11 @@ jboolean RGBA2ARGB888Bitmap(JNIEnv *env, jobject j_argb8888_bitmap, LOGE("Invoke AndroidBitmap_lockPixels() failed!"); return JNI_FALSE; } + // no copied, but point to bitmap data. cv::Mat j_bitmap_im(static_cast(j_bitmap_info.height), static_cast(j_bitmap_info.width), CV_8UC4, j_bitmap_pixels); + // TODO: Use zero copy operation or neon to boost performance. c_rgba.copyTo(j_bitmap_im); if (AndroidBitmap_unlockPixels(env, j_argb8888_bitmap) < 0) { LOGE("Invoke AndroidBitmap_unlockPixels() failed!"); diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/runtime_option_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/runtime_option_jni.cc index 71a46963e6..1a2f8d94c8 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/runtime_option_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/runtime_option_jni.cc @@ -34,6 +34,8 @@ fastdeploy::RuntimeOption NewCxxRuntimeOption( j_runtime_option_clazz, "mCpuThreadNum", "I"); const jfieldID j_enable_lite_fp16_id = env->GetFieldID( j_runtime_option_clazz, "mEnableLiteFp16", "Z"); + const jfieldID j_enable_lite_int8_id = env->GetFieldID( + j_runtime_option_clazz, "mEnableLiteInt8", "Z"); const jfieldID j_lite_power_mode_id = env->GetFieldID( j_runtime_option_clazz, "mLitePowerMode", "Lcom/baidu/paddle/fastdeploy/LitePowerMode;"); @@ -59,6 +61,8 @@ fastdeploy::RuntimeOption NewCxxRuntimeOption( j_runtime_option_obj, j_cpu_num_thread_id); jboolean j_enable_lite_fp16 = env->GetBooleanField( j_runtime_option_obj, j_enable_lite_fp16_id); + jboolean j_enable_lite_int8 = env->GetBooleanField( + j_runtime_option_obj, j_enable_lite_int8_id); jstring j_lite_optimized_model_dir = static_cast( env->GetObjectField(j_runtime_option_obj, j_lite_optimized_model_dir_id)); jobject j_lite_power_mode_obj = env->GetObjectField( @@ -68,6 +72,7 @@ fastdeploy::RuntimeOption NewCxxRuntimeOption( int c_cpu_num_thread = static_cast(j_cpu_num_thread); bool c_enable_lite_fp16 = static_cast(j_enable_lite_fp16); + bool c_enable_lite_int8 = static_cast(j_enable_lite_int8); fastdeploy::LitePowerMode c_lite_power_mode = static_cast(j_lite_power_mode); std::string c_lite_optimized_model_dir = @@ -80,6 +85,9 @@ fastdeploy::RuntimeOption NewCxxRuntimeOption( if (c_enable_lite_fp16) { c_runtime_option.EnableLiteFP16(); } + if (c_enable_lite_int8) { + c_runtime_option.EnableLiteInt8(); + } env->DeleteLocalRef(j_runtime_option_clazz); env->DeleteLocalRef(j_lite_power_mode_clazz); diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.cc new file mode 100644 index 0000000000..f8bca25460 --- /dev/null +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.cc @@ -0,0 +1,45 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy_jni/convert_jni.h" // NOLINT +#include "fastdeploy_jni/assets_loader_jni.h" // NOLINT +#include "fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.h" // NOLINT + +namespace fastdeploy { +namespace jni { + +/// Rendering DetectionResult to ARGB888Bitmap +void RenderingKeyPointDetection( + JNIEnv *env, const cv::Mat &c_bgr, + const vision::KeyPointDetectionResult &c_result, + jobject argb8888_bitmap, bool save_image, float conf_threshold, + jstring save_path) { + if (!c_result.keypoints.empty()) { + auto t = GetCurrentTime(); + cv::Mat c_vis_im; + vision::VisKeypointDetection(c_bgr, c_result, conf_threshold); + LOGD("Visualize from native costs %f ms", GetElapsedTime(t)); + + if (!BGR2ARGB888Bitmap(env, argb8888_bitmap, c_vis_im)) { + LOGD("Write to bitmap from native failed!"); + } + auto c_saved_image_path = ConvertTo(env, save_path); + if (!c_saved_image_path.empty() && save_image) { + cv::imwrite(c_saved_image_path, c_vis_im); + } + } +} + +} // namespace jni +} // namespace fastdeploy diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.h b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.h new file mode 100644 index 0000000000..2ee356bf78 --- /dev/null +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.h @@ -0,0 +1,32 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include // NOLINT +#include "fastdeploy/vision.h" // NOLINT +#include "fastdeploy_jni/perf_jni.h" // NOLINT +#include "fastdeploy_jni/bitmap_jni.h" // NOLINT + +namespace fastdeploy { +namespace jni { + +void RenderingKeyPointDetection( + JNIEnv *env, const cv::Mat &c_bgr, + const vision::KeyPointDetectionResult &c_result, + jobject argb8888_bitmap, bool save_image, + float conf_threshold, jstring save_path); + +} // namespace jni +} // namespace fastdeploy diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/pptinypose_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/pptinypose_jni.cc new file mode 100644 index 0000000000..4d351bae39 --- /dev/null +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/pptinypose_jni.cc @@ -0,0 +1,103 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include // NOLINT +#include "fastdeploy_jni/convert_jni.h" // NOLINT +#include "fastdeploy_jni/assets_loader_jni.h" // NOLINT +#include "fastdeploy_jni/runtime_option_jni.h" // NOLINT +#include "fastdeploy_jni/vision/results_jni.h" // NOLINT +#include "fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.h" // NOLINT + +namespace fni = fastdeploy::jni; +namespace vision = fastdeploy::vision; +namespace keypointdetection = fastdeploy::vision::keypointdetection; + +#ifdef __cplusplus +extern "C" { +#endif + +JNIEXPORT jlong JNICALL +Java_com_baidu_paddle_fastdeploy_vision_keypointdetection_PPTinyPose_bindNative( + JNIEnv *env, jobject thiz, jstring model_file, jstring params_file, + jstring config_file, jobject runtime_option) { + auto c_model_file = fni::ConvertTo(env, model_file); + auto c_params_file = fni::ConvertTo(env, params_file); + auto c_config_file = fni::ConvertTo(env, config_file); + auto c_runtime_option = fni::NewCxxRuntimeOption(env, runtime_option); + auto c_model_ptr = new keypointdetection::PPTinyPose( + c_model_file, c_params_file, c_config_file, c_runtime_option); + INITIALIZED_OR_RETURN(c_model_ptr) + +#ifdef ENABLE_RUNTIME_PERF + c_model_ptr->EnableRecordTimeOfRuntime(); +#endif + + // setup use_dark param + const jclass j_pptinypose_clazz = env->GetObjectClass(thiz); + const jfieldID j_use_dark_id = env->GetFieldID( + j_pptinypose_clazz, "mUseDark", "Z"); + jboolean j_use_dark = env->GetBooleanField(thiz, j_use_dark_id); + const bool c_use_dark = static_cast(j_use_dark); + c_model_ptr->use_dark = c_use_dark; + env->DeleteLocalRef(j_pptinypose_clazz); + + vision::EnableFlyCV(); + return reinterpret_cast(c_model_ptr); +} + +JNIEXPORT jobject JNICALL +Java_com_baidu_paddle_fastdeploy_vision_keypointdetection_PPTinyPose_predictNative( + JNIEnv *env, jobject thiz, jlong cxx_context, jobject argb8888_bitmap, + jboolean save_image, jstring save_path, jboolean rendering, + jfloat conf_threshold) { + if (cxx_context == 0) { + return NULL; + } + cv::Mat c_bgr; + if (!fni::ARGB888Bitmap2BGR(env, argb8888_bitmap, &c_bgr)) { + return NULL; + } + auto c_model_ptr = reinterpret_cast(cxx_context); + vision::KeyPointDetectionResult c_result; + auto t = fni::GetCurrentTime(); + c_model_ptr->Predict(&c_bgr, &c_result); + PERF_TIME_OF_RUNTIME(c_model_ptr, t) + + if (rendering) { + fni::RenderingKeyPointDetection( + env, c_bgr, c_result, argb8888_bitmap, save_image, + conf_threshold, save_path); + } + + return fni::NewJavaResultFromCxx(env, reinterpret_cast(&c_result), + vision::ResultType::KEYPOINT_DETECTION); +} + +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_keypointdetection_PPTinyPose_releaseNative( + JNIEnv *env, jobject thiz, jlong cxx_context) { + if (cxx_context == 0) { + return JNI_FALSE; + } + auto c_model_ptr = reinterpret_cast(cxx_context); + PERF_TIME_OF_RUNTIME(c_model_ptr, -1) + + delete c_model_ptr; + LOGD("[End] Release PPTinyPose in native !"); + return JNI_TRUE; +} + +#ifdef __cplusplus +} +#endif diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc index 4a9f0cd6e0..0680a15349 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc @@ -438,9 +438,79 @@ bool AllocateJavaFaceDetectionResultFromCxx( return true; } -bool AllocateJavaResultFromCxx( - JNIEnv *env, jobject j_result_obj, void *cxx_result, - vision::ResultType type) { +bool AllocateJavaKeyPointDetectionResultFromCxx( + JNIEnv *env, jobject j_keypoint_det_result_obj, void *cxx_result) { + // WARN: Please make sure 'j_keypoint_det_result_obj' param + // is a ref of Java KeyPointDetectionResult. + // Field signatures of Java KeyPointDetectionResult: + // (1) mBoxes float[][] shape (n*num_joints,2): [[F + // (2) mScores float[] shape (n*num_joints): [F + // (3) mNumJoints int shape (1): I + // (4) mInitialized boolean: Z + // Docs: docs/api/vision_results/keypointdetection_result.md + if (cxx_result == nullptr) { + return false; + } + auto c_result_ptr = reinterpret_cast(cxx_result); + + const int len = static_cast(c_result_ptr->keypoints.size()); + if (len == 0) { + return false; + } + + const jclass j_keypoint_det_result_clazz = env->FindClass( + "com/baidu/paddle/fastdeploy/vision/KeyPointDetectionResult"); + const jclass j_keypoint_float_arr_clazz = env->FindClass("[F"); // (2,) + const jfieldID j_keypoint_det_keypoints_id = env->GetFieldID( + j_keypoint_det_result_clazz, "mKeyPoints", "[[F"); + const jfieldID j_keypoint_det_scores_id = env->GetFieldID( + j_keypoint_det_result_clazz, "mScores", "[F"); + const jfieldID j_keypoint_det_num_joints_id = env->GetFieldID( + j_keypoint_det_result_clazz, "mNumJoints", "I"); + const jfieldID j_keypoint_det_initialized_id = env->GetFieldID( + j_keypoint_det_result_clazz, "mInitialized", "Z"); + + if (!env->IsInstanceOf(j_keypoint_det_result_obj, j_keypoint_det_result_clazz)) { + return false; + } + + // mKeyPoints float[][] shape (n*num_joints,2): [[F + const auto &keypoints = c_result_ptr->keypoints; + jobjectArray j_keypoint_det_keypoints_float_arr = + env->NewObjectArray(len, j_keypoint_float_arr_clazz, NULL); + for (int i = 0; i < len; ++i) { + jfloatArray j_point = env->NewFloatArray(2); + env->SetFloatArrayRegion(j_point, 0, 2, keypoints.at(i).data()); + env->SetObjectArrayElement(j_keypoint_det_keypoints_float_arr, i, j_point); + env->DeleteLocalRef(j_point); + } + + // mScores float[] shape (n): [F + const auto &scores = c_result_ptr->scores; + const int score_len = scores.size(); + jfloatArray j_keypoint_det_scores_float_arr = env->NewFloatArray(score_len); + env->SetFloatArrayRegion(j_keypoint_det_scores_float_arr, 0, score_len, scores.data()); + + // mNumJoints int shape (1): I + jint j_keypoint_det_num_joints = static_cast(c_result_ptr->num_joints); + + // Set object fields + env->SetObjectField(j_keypoint_det_result_obj, j_keypoint_det_keypoints_id, j_keypoint_det_keypoints_float_arr); + env->SetObjectField(j_keypoint_det_result_obj, j_keypoint_det_scores_id, j_keypoint_det_scores_float_arr); + env->SetIntField(j_keypoint_det_result_obj, j_keypoint_det_num_joints_id, j_keypoint_det_num_joints); + env->SetBooleanField(j_keypoint_det_result_obj, j_keypoint_det_initialized_id, JNI_TRUE); + + // Release local Refs + env->DeleteLocalRef(j_keypoint_det_keypoints_float_arr); + env->DeleteLocalRef(j_keypoint_det_scores_float_arr); + env->DeleteLocalRef(j_keypoint_det_result_clazz); + env->DeleteLocalRef(j_keypoint_float_arr_clazz); + + return true; +} + +bool AllocateJavaResultFromCxx(JNIEnv *env, jobject j_result_obj, + void *cxx_result, vision::ResultType type) { if (type == vision::ResultType::CLASSIFY) { return AllocateJavaClassifyResultFromCxx(env, j_result_obj, cxx_result); } else if (type == vision::ResultType::DETECTION) { @@ -451,6 +521,8 @@ bool AllocateJavaResultFromCxx( return AllocateJavaSegmentationResultFromCxx(env, j_result_obj, cxx_result); } else if (type == vision::ResultType::FACE_DETECTION) { return AllocateJavaFaceDetectionResultFromCxx(env, j_result_obj, cxx_result); + } else if (type == vision::ResultType::KEYPOINT_DETECTION) { + return AllocateJavaKeyPointDetectionResultFromCxx(env, j_result_obj, cxx_result); } else { LOGE("Not support this ResultType in JNI now, type: %d", static_cast(type)); @@ -519,6 +591,18 @@ jobject NewJavaFaceDetectionResultFromCxx(JNIEnv *env, void *cxx_result) { return j_face_det_result_obj; } +jobject NewJavaKeyPointDetectionResultFromCxx(JNIEnv *env, void *cxx_result) { + const jclass j_keypoint_det_result_clazz = env->FindClass( + "com/baidu/paddle/fastdeploy/vision/KeyPointDetectionResult"); + const jmethodID j_keypoint_det_result_init = env->GetMethodID( + j_keypoint_det_result_clazz, "", "()V"); + jobject j_keypoint_det_result_obj = env->NewObject( + j_keypoint_det_result_clazz, j_keypoint_det_result_init); + AllocateJavaKeyPointDetectionResultFromCxx(env, j_keypoint_det_result_obj, cxx_result); + env->DeleteLocalRef(j_keypoint_det_result_clazz); + return j_keypoint_det_result_obj; +} + jobject NewJavaResultFromCxx( JNIEnv *env, void *cxx_result, vision::ResultType type) { if (type == vision::ResultType::CLASSIFY) { @@ -531,6 +615,8 @@ jobject NewJavaResultFromCxx( return NewJavaSegmentationResultFromCxx(env, cxx_result); } else if (type == vision::ResultType::FACE_DETECTION) { return NewJavaFaceDetectionResultFromCxx(env, cxx_result); + } else if (type == vision::ResultType::KEYPOINT_DETECTION) { + return NewJavaKeyPointDetectionResultFromCxx(env, cxx_result); } else { LOGE("Not support this ResultType in JNI now, type: %d", static_cast(type)); @@ -1058,6 +1144,95 @@ bool AllocateFaceDetectionResultFromJava( return true; } +bool AllocateKeyPointDetectionResultFromJava( + JNIEnv *env, jobject j_keypoint_det_result_obj, void *cxx_result) { + // WARN: Please make sure 'j_keypoint_det_result_obj' param + // is a ref of Java KeyPointDetectionResult. + // Field signatures of Java KeyPointDetectionResult: + // (1) mBoxes float[][] shape (n*num_joints,2): [[F + // (2) mScores float[] shape (n*num_joints): [F + // (3) mNumJoints int shape (1): I + // (4) mInitialized boolean: Z + // Docs: docs/api/vision_results/keypointdetection_result.md + if (cxx_result == nullptr || j_keypoint_det_result_obj == nullptr) { + return false; + } + auto c_result_ptr = reinterpret_cast(cxx_result); + + const jclass j_keypoint_det_result_clazz_cc = env->FindClass( + "com/baidu/paddle/fastdeploy/vision/KeyPointDetectionResult"); + const jfieldID j_keypoint_det_keypoints_id_cc = env->GetFieldID( + j_keypoint_det_result_clazz_cc, "mKeyPoints", "[[F"); + const jfieldID j_keypoint_det_scores_id_cc = env->GetFieldID( + j_keypoint_det_result_clazz_cc, "mScores", "[F"); + const jfieldID j_keypoint_det_num_joints_id_cc = env->GetFieldID( + j_keypoint_det_result_clazz_cc, "mNumJoints", "I"); + const jfieldID j_keypoint_det_initialized_id_cc = env->GetFieldID( + j_keypoint_det_result_clazz_cc, "mInitialized", "Z"); + + if (!env->IsInstanceOf(j_keypoint_det_result_obj, j_keypoint_det_result_clazz_cc)) { + return false; + } + + // mInitialized boolean: Z + jboolean j_keypoint_det_initialized = + env->GetBooleanField(j_keypoint_det_result_obj, j_keypoint_det_initialized_id_cc); + if (j_keypoint_det_initialized == JNI_FALSE) { + return false; + } + + jobjectArray j_keypoint_det_keypoints_float_arr = reinterpret_cast( + env->GetObjectField(j_keypoint_det_result_obj, j_keypoint_det_keypoints_id_cc)); + jfloatArray j_keypoint_det_scores_float_arr = reinterpret_cast( + env->GetObjectField(j_keypoint_det_result_obj, j_keypoint_det_scores_id_cc)); + jint j_keypoint_det_num_joints = env->GetIntField( + j_keypoint_det_result_obj, j_keypoint_det_num_joints_id_cc); + + int len = env->GetArrayLength(j_keypoint_det_keypoints_float_arr); + if ((len == 0) || (len != env->GetArrayLength(j_keypoint_det_scores_float_arr)) || + (j_keypoint_det_num_joints < 0)) { + return false; + } + + // Init Cxx result + c_result_ptr->Clear(); + + // mKeyPoints float[][] shape (n*num_joints,2): [[F + c_result_ptr->keypoints.resize(len); + bool c_check_validation = true; + for (int i = 0; i < len; ++i) { + auto j_point = reinterpret_cast( + env->GetObjectArrayElement(j_keypoint_det_keypoints_float_arr, i)); + if (env->GetArrayLength(j_point) == 2) { + jfloat *j_point_ptr = env->GetFloatArrayElements(j_point, nullptr); + std::memcpy(c_result_ptr->keypoints[i].data(), j_point_ptr, 2 * sizeof(float)); + env->ReleaseFloatArrayElements(j_point, j_point_ptr, 0); + } else { + c_check_validation = false; + break; + } + } + if (!c_check_validation) { + LOGE("The length of each detection box is not equal 2!"); + return false; + } + + // mScores float[] shape (n): [F + c_result_ptr->scores.resize(len); + jfloat *j_keypoint_det_scores_ptr = + env->GetFloatArrayElements(j_keypoint_det_scores_float_arr, nullptr); + std::memcpy(c_result_ptr->scores.data(), j_keypoint_det_scores_ptr, len * sizeof(float)); + env->ReleaseFloatArrayElements(j_keypoint_det_scores_float_arr, j_keypoint_det_scores_ptr, 0); + + // mNumJoints int shape (1): I + c_result_ptr->num_joints = static_cast(j_keypoint_det_num_joints); + + // Release local Refs + env->DeleteLocalRef(j_keypoint_det_result_clazz_cc); + + return true; +} + bool AllocateCxxResultFromJava( JNIEnv *env, jobject j_result_obj, void *cxx_result, vision::ResultType type) { @@ -1069,8 +1244,10 @@ bool AllocateCxxResultFromJava( return AllocateOCRResultFromJava(env, j_result_obj, cxx_result); } else if (type == vision::ResultType::SEGMENTATION) { return AllocateSegmentationResultFromJava(env, j_result_obj, cxx_result); - } else if (type == vision::ResultType::FACE_DETECTION) { + } else if (type == vision::ResultType::FACE_DETECTION) { return AllocateFaceDetectionResultFromJava(env, j_result_obj, cxx_result); + } else if (type == vision::ResultType::KEYPOINT_DETECTION) { + return AllocateKeyPointDetectionResultFromJava(env, j_result_obj, cxx_result); } else { LOGE("Not support this ResultType in JNI now, type: %d", static_cast(type)); @@ -1081,7 +1258,6 @@ bool AllocateCxxResultFromJava( } // namespace jni } // namespace fastdeploy - #ifdef __cplusplus extern "C" { #endif @@ -1109,7 +1285,7 @@ Java_com_baidu_paddle_fastdeploy_vision_SegmentationResult_releaseCxxBufferNativ auto c_result_ptr = reinterpret_cast< fastdeploy::vision::SegmentationResult *>(j_cxx_buffer); delete c_result_ptr; - LOGD("[End] Release SegmentationResult in native !"); + LOGD("[End] Release SegmentationResult & CxxBuffer in native !"); env->SetBooleanField(thiz, j_seg_initialized_id, JNI_FALSE); env->DeleteLocalRef(j_seg_result_clazz); diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc index 69ded0f4df..b39d207ad6 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc @@ -42,13 +42,13 @@ Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_bindNative( c_model_ptr->EnableRecordTimeOfRuntime(); #endif - // Setup is_vertical_screen param + // setup is_vertical_screen param const jclass j_ppseg_clazz = env->GetObjectClass(thiz); const jfieldID j_is_vertical_screen_id = env->GetFieldID( j_ppseg_clazz, "mIsVerticalScreen", "Z"); jboolean j_is_vertical_screen = env->GetBooleanField( thiz, j_is_vertical_screen_id); - bool c_is_vertical_screen = static_cast(j_is_vertical_screen); + const bool c_is_vertical_screen = static_cast(j_is_vertical_screen); c_model_ptr->GetPreprocessor().SetIsVerticalScreen(c_is_vertical_screen); env->DeleteLocalRef(j_ppseg_clazz); diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc index 354cdd89c7..af97f6c829 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc @@ -225,6 +225,33 @@ jboolean VisFaceDetectionFromJava( return JNI_TRUE; } +jboolean VisKeyPointDetectionFromJava( + JNIEnv *env, jobject argb8888_bitmap, jobject result, + jint conf_threshold) { + const jclass j_keypoint_det_result_clazz = env->FindClass( + "com/baidu/paddle/fastdeploy/vision/KeyPointDetectionResult"); + if (!env->IsInstanceOf(result, j_keypoint_det_result_clazz)) { + env->DeleteLocalRef(j_keypoint_det_result_clazz); + return JNI_FALSE; + } + env->DeleteLocalRef(j_keypoint_det_result_clazz); + vision::KeyPointDetectionResult c_result; + if (!fni::AllocateCxxResultFromJava( + env, result, reinterpret_cast(&c_result), + vision::ResultType::KEYPOINT_DETECTION)) { + return JNI_FALSE; + } + cv::Mat c_bgr; + if (!fni::ARGB888Bitmap2BGR(env, argb8888_bitmap, &c_bgr)) { + return JNI_FALSE; + } + auto c_vis_im = vision::VisKeypointDetection(c_bgr, c_result, conf_threshold); + if (!fni::BGR2ARGB888Bitmap(env, argb8888_bitmap, c_vis_im)) { + return JNI_FALSE; + } + return JNI_TRUE; +} + } // jni } // fastdeploy @@ -283,8 +310,15 @@ Java_com_baidu_paddle_fastdeploy_vision_Visualize_visFaceDetectionNative( line_size, font_size); } +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_Visualize_visKeyPointDetectionNative( + JNIEnv *env, jclass clazz, jobject argb8888_bitmap, + jobject result, jfloat conf_threshold) { + return fni::VisKeyPointDetectionFromJava(env, argb8888_bitmap, result, + conf_threshold); +} + #ifdef __cplusplus } #endif - diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/RuntimeOption.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/RuntimeOption.java index e1e7030089..c55c5d9500 100644 --- a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/RuntimeOption.java +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/RuntimeOption.java @@ -3,12 +3,14 @@ public class RuntimeOption { public int mCpuThreadNum = 1; public boolean mEnableLiteFp16 = false; + public boolean mEnableLiteInt8 = false; public LitePowerMode mLitePowerMode = LitePowerMode.LITE_POWER_NO_BIND; public String mLiteOptimizedModelDir = ""; public RuntimeOption() { mCpuThreadNum = 1; mEnableLiteFp16 = false; + mEnableLiteInt8 = false; mLitePowerMode = LitePowerMode.LITE_POWER_NO_BIND; mLiteOptimizedModelDir = ""; } @@ -21,6 +23,14 @@ public void disableLiteFP16() { mEnableLiteFp16 = false; } + public void enableLiteInt8() { + mEnableLiteInt8 = true; + } + + public void disableLiteInt8() { + mEnableLiteInt8 = false; + } + public void setCpuThreadNum(int threadNum) { mCpuThreadNum = threadNum; } diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/KeyPointDetectionResult.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/KeyPointDetectionResult.java new file mode 100644 index 0000000000..f975e69592 --- /dev/null +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/KeyPointDetectionResult.java @@ -0,0 +1,42 @@ +package com.baidu.paddle.fastdeploy.vision; + +import android.support.annotation.NonNull; + +import java.util.Arrays; + +public class KeyPointDetectionResult { + public float[][] mKeyPoints; // [n*num_joints, 2] + public float[] mScores; // [n*num_joints] + public int mNumJoints = -1; + public boolean mInitialized = false; + + public KeyPointDetectionResult() { + mInitialized = false; + } + + public boolean initialized() { + return mInitialized; + } + + public void setKeyPoints(@NonNull float[] keyPointsBuffer) { + int pointNum = keyPointsBuffer.length / 2; + if (pointNum > 0) { + mKeyPoints = new float[pointNum][2]; + for (int i = 0; i < pointNum; ++i) { + mKeyPoints[i] = Arrays.copyOfRange( + keyPointsBuffer, i * 2, (i + 1) * 2); + } + } + } + + public void setScores(@NonNull float[] scoresBuffer) { + if (scoresBuffer.length > 0) { + mScores = scoresBuffer.clone(); + } + } + + public void setNumJoints(int numJoints) { + mNumJoints = numJoints; + } + +} diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/Visualize.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/Visualize.java index 458521dbae..874dacd287 100644 --- a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/Visualize.java +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/Visualize.java @@ -161,6 +161,24 @@ public static boolean visFaceDetection(Bitmap ARGB8888Bitmap, fontSize); } + // Visualize KeyPointDetectionResult + public static boolean visKeypointDetection(Bitmap ARGB8888Bitmap, + KeyPointDetectionResult result) { + return visKeyPointDetectionNative( + ARGB8888Bitmap, + result, + 0.5f); + } + + public static boolean visKeypointDetection(Bitmap ARGB8888Bitmap, + KeyPointDetectionResult result, + float confThreshold) { + return visKeyPointDetectionNative( + ARGB8888Bitmap, + result, + confThreshold); + } + // VisDetection in native private static native boolean visDetectionNative(Bitmap ARGB8888Bitmap, DetectionResult result, @@ -191,6 +209,11 @@ private static native boolean visFaceDetectionNative(Bitmap ARGB8888Bitmap, int lineSize, float fontSize); + // VisKeypointDetection in native + private static native boolean visKeyPointDetectionNative(Bitmap ARGB8888Bitmap, + KeyPointDetectionResult result, + float confThreshold); + /* Initializes at the beginning */ static { FastDeployInitializer.init(); diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/keypointdetection/PPTinyPose.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/keypointdetection/PPTinyPose.java new file mode 100644 index 0000000000..db8ff5ffd2 --- /dev/null +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/keypointdetection/PPTinyPose.java @@ -0,0 +1,152 @@ +package com.baidu.paddle.fastdeploy.vision.keypointdetection; + +import android.graphics.Bitmap; + +import com.baidu.paddle.fastdeploy.FastDeployInitializer; +import com.baidu.paddle.fastdeploy.RuntimeOption; +import com.baidu.paddle.fastdeploy.vision.KeyPointDetectionResult; + +public class PPTinyPose { + protected long mCxxContext = 0; // Context from native. + protected boolean mUseDark = true; + protected boolean mInitialized = false; + + public PPTinyPose() { + mInitialized = false; + } + + // Constructor with default runtime option + public PPTinyPose(String modelFile, + String paramsFile, + String configFile) { + init_(modelFile, paramsFile, configFile, new RuntimeOption()); + } + + // Constructor without label file + public PPTinyPose(String modelFile, + String paramsFile, + String configFile, + RuntimeOption runtimeOption) { + init_(modelFile, paramsFile, configFile, runtimeOption); + } + + // Call init manually without label file + public boolean init(String modelFile, + String paramsFile, + String configFile, + RuntimeOption runtimeOption) { + return init_(modelFile, paramsFile, configFile, runtimeOption); + } + + public boolean release() { + mInitialized = false; + if (mCxxContext == 0) { + return false; + } + return releaseNative(mCxxContext); + } + + public boolean initialized() { + return mInitialized; + } + + // Predict without image saving and bitmap rendering. + public KeyPointDetectionResult predict(Bitmap ARGB8888Bitmap) { + if (mCxxContext == 0) { + return new KeyPointDetectionResult(); + } + // Only support ARGB8888 bitmap in native now. + KeyPointDetectionResult result = predictNative(mCxxContext, ARGB8888Bitmap, + false, "", false, 0.f); + if (result == null) { + return new KeyPointDetectionResult(); + } + return result; + } + + public KeyPointDetectionResult predict(Bitmap ARGB8888Bitmap, + boolean rendering, + float confThreshold) { + if (mCxxContext == 0) { + return new KeyPointDetectionResult(); + } + // Only support ARGB8888 bitmap in native now. + KeyPointDetectionResult result = predictNative(mCxxContext, ARGB8888Bitmap, + false, "", rendering, confThreshold); + if (result == null) { + return new KeyPointDetectionResult(); + } + return result; + } + + // Predict with image saving and bitmap rendering (will cost more times) + public KeyPointDetectionResult predict(Bitmap ARGB8888Bitmap, + String savedImagePath, + float confThreshold) { + // confThreshold is for visualizing only. + if (mCxxContext == 0) { + return new KeyPointDetectionResult(); + } + // Only support ARGB8888 bitmap in native now. + KeyPointDetectionResult result = predictNative( + mCxxContext, ARGB8888Bitmap, true, + savedImagePath, true, confThreshold); + if (result == null) { + return new KeyPointDetectionResult(); + } + return result; + } + + private boolean init_(String modelFile, + String paramsFile, + String configFile, + RuntimeOption runtimeOption) { + if (!mInitialized) { + mCxxContext = bindNative( + modelFile, + paramsFile, + configFile, + runtimeOption); + if (mCxxContext != 0) { + mInitialized = true; + } + return mInitialized; + } else { + // release current native context and bind a new one. + if (release()) { + mCxxContext = bindNative( + modelFile, + paramsFile, + configFile, + runtimeOption); + if (mCxxContext != 0) { + mInitialized = true; + } + return mInitialized; + } + return false; + } + } + + // Bind predictor from native context. + private native long bindNative(String modelFile, + String paramsFile, + String configFile, + RuntimeOption runtimeOption); + + // Call prediction from native context with rendering. + private native KeyPointDetectionResult predictNative(long CxxContext, + Bitmap ARGB8888Bitmap, + boolean saveImage, + String savePath, + boolean rendering, + float confThreshold); + + // Release buffers allocated in native context. + private native boolean releaseNative(long CxxContext); + + // Initializes at the beginning. + static { + FastDeployInitializer.init(); + } +} From fbb78caffa472e5d2a2c43a41b73fe69505e56cb Mon Sep 17 00:00:00 2001 From: DefTruth Date: Wed, 30 Nov 2022 11:55:57 +0800 Subject: [PATCH 4/6] [Android] Update gradle download tasks info --- java/android/app/build.gradle | 24 +++++++++++-------- .../KeyPointDetectionMainActivity.java | 4 ++++ .../KeyPointDetectionSettingsActivity.java | 4 ++++ java/android/fastdeploy/build.gradle | 7 ++++-- .../keypointdetection_utils_jni.cc | 2 +- java/android/ui/build.gradle | 1 + 6 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionMainActivity.java create mode 100644 java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionSettingsActivity.java diff --git a/java/android/app/build.gradle b/java/android/app/build.gradle index f1378ad327..bf28c2b28f 100644 --- a/java/android/app/build.gradle +++ b/java/android/app/build.gradle @@ -19,17 +19,16 @@ android { proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' } } - } dependencies { - implementation fileTree(include: ['*.aar'], dir: 'libs') + // implementation fileTree(include: ['*.aar'], dir: 'libs') implementation 'com.android.support:appcompat-v7:28.0.0' //noinspection GradleDependency implementation 'com.android.support.constraint:constraint-layout:1.1.3' implementation 'com.android.support:design:28.0.0' implementation 'org.jetbrains:annotations:15.0' - // implementation project(path: ':fastdeploy') + implementation project(path: ':fastdeploy') implementation project(path: ':ui') //noinspection GradleDependency testImplementation 'junit:junit:4.12' @@ -81,7 +80,7 @@ def FD_JAVA_SDK = [ task downloadAndExtractModels(type: DefaultTask) { doFirst { - println "Downloading and extracting fastdeploy models ..." + println "[INFO] Downloading and extracting fastdeploy models ..." } doLast { String cachePath = "cache" @@ -91,19 +90,22 @@ task downloadAndExtractModels(type: DefaultTask) { FD_MODEL.eachWithIndex { model, index -> String[] modelPaths = model.src.split("/") String modelName = modelPaths[modelPaths.length - 1] + String modelPrefix = modelName.substring(0, modelName.length() - 4) // Download the target model if not exists - boolean copyFiles = !file("${model.dest}").exists() + boolean copyFiles = !file("${model.dest}/${modelPrefix}").exists() if (!file("${cachePath}/${modelName}").exists()) { - println "Downloading ${model.src} -> ${cachePath}/${modelName}" + println "[INFO] Downloading ${model.src} -> ${cachePath}/${modelName}" ant.get(src: model.src, dest: file("${cachePath}/${modelName}")) copyFiles = true } if (copyFiles) { - println "Coping ${cachePath}/${modelName} -> ${model.dest}" + println "[INFO] Taring ${cachePath}/${modelName} -> ${model.dest}/${modelPrefix}" copy { from tarTree("${cachePath}/${modelName}") into "${model.dest}" } + } else { + println "[INFO] ${model.dest}/${modelPrefix} already exists!" } } } @@ -111,7 +113,7 @@ task downloadAndExtractModels(type: DefaultTask) { task downloadAndExtractSDKs(type: DefaultTask) { doFirst { - println "Downloading and extracting fastdeploy android java sdk ..." + println "[INFO] Downloading and extracting fastdeploy android java sdk ..." } doLast { String cachePath = "cache" @@ -124,16 +126,18 @@ task downloadAndExtractSDKs(type: DefaultTask) { // Download the target SDK if not exists boolean copyFiles = !file("${sdk.dest}/${sdkName}").exists() if (!file("${cachePath}/${sdkName}").exists()) { - println "Downloading ${sdk.src} -> ${cachePath}/${sdkName}" + println "[INFO] Downloading ${sdk.src} -> ${cachePath}/${sdkName}" ant.get(src: sdk.src, dest: file("${cachePath}/${sdkName}")) copyFiles = true } if (copyFiles) { - println "Coping ${cachePath}/${sdkName} -> ${sdk.dest}/${sdkName}" + println "[INFO] Coping ${cachePath}/${sdkName} -> ${sdk.dest}/${sdkName}" copy { from "${cachePath}/${sdkName}" into "${sdk.dest}" } + } else { + println "[INFO] ${sdk.dest}/${sdkName} already exists!" } } } diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionMainActivity.java new file mode 100644 index 0000000000..11448639c5 --- /dev/null +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionMainActivity.java @@ -0,0 +1,4 @@ +package com.baidu.paddle.fastdeploy.app.examples.keypointdetection; + +public class KeyPointDetectionMainActivity { +} diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionSettingsActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionSettingsActivity.java new file mode 100644 index 0000000000..ac0134fc25 --- /dev/null +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionSettingsActivity.java @@ -0,0 +1,4 @@ +package com.baidu.paddle.fastdeploy.app.examples.keypointdetection; + +public class KeyPointDetectionSettingsActivity { +} diff --git a/java/android/fastdeploy/build.gradle b/java/android/fastdeploy/build.gradle index 688d8e795c..f162484408 100644 --- a/java/android/fastdeploy/build.gradle +++ b/java/android/fastdeploy/build.gradle @@ -56,7 +56,7 @@ def FD_CXX_LIB = [ task downloadAndExtractLibs(type: DefaultTask) { doFirst { - println "Downloading and extracting fastdeploy android c++ lib ..." + println "[INFO] Downloading and extracting fastdeploy android c++ lib ..." } doLast { String cachePath = "cache" @@ -69,15 +69,18 @@ task downloadAndExtractLibs(type: DefaultTask) { libName = libName.split("\\.")[0] boolean copyFiles = !file("${lib.dest}/${libName}").exists() if (!file("${cachePath}/${libName}.tgz").exists()) { - println "Downloading ${lib.src} -> ${cachePath}/${libName}.tgz" + println "[INFO] Downloading ${lib.src} -> ${cachePath}/${libName}.tgz" ant.get(src: lib.src, dest: file("${cachePath}/${libName}.tgz")) copyFiles = true } if (copyFiles) { + println "[INFO] Taring ${cachePath}/${libName}.tgz -> ${lib.dest}/${libName}" copy { from tarTree("${cachePath}/${libName}.tgz") into "${lib.dest}" } + } else { + println "[INFO] ${lib.dest}/${libName} already exists!" } } } diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.cc index f8bca25460..b0e74dfe26 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.cc @@ -19,7 +19,7 @@ namespace fastdeploy { namespace jni { -/// Rendering DetectionResult to ARGB888Bitmap +/// Rendering KeyPointDetectionResult to ARGB888Bitmap void RenderingKeyPointDetection( JNIEnv *env, const cv::Mat &c_bgr, const vision::KeyPointDetectionResult &c_result, diff --git a/java/android/ui/build.gradle b/java/android/ui/build.gradle index 6a524e0b5b..a574cf398f 100644 --- a/java/android/ui/build.gradle +++ b/java/android/ui/build.gradle @@ -7,6 +7,7 @@ android { defaultConfig { minSdkVersion 15 + //noinspection ExpiredTargetSdkVersion targetSdkVersion 28 testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" From a7c320fea464f2d525207d090e719c5f01acc944 Mon Sep 17 00:00:00 2001 From: DefTruth Date: Wed, 30 Nov 2022 16:20:20 +0800 Subject: [PATCH 5/6] [Android] Add PPTinyPose Android app example --- java/android/app/build.gradle | 4 + java/android/app/src/main/AndroidManifest.xml | 4 +- .../KeyPointDetectionMainActivity.java | 401 +++++++++++++++++- .../KeyPointDetectionSettingsActivity.java | 162 ++++++- .../SegmentationMainActivity.java | 2 +- .../SegmentationSettingsActivity.java | 2 - .../keypointdetection_activity_main.xml | 14 + .../layout/keypointdetection_camera_page.xml | 159 +++++++ .../layout/keypointdetection_result_page.xml | 160 +++++++ .../app/src/main/res/values/strings.xml | 3 + .../res/xml/keypointdetection_settting.xml | 37 ++ .../keypointdetection_utils_jni.cc | 4 +- .../fastdeploy_jni/vision/visualize_jni.cc | 2 +- .../vision/keypointdetection/PPTinyPose.java | 4 + .../vision/segmentation/PaddleSegModel.java | 17 +- 15 files changed, 959 insertions(+), 16 deletions(-) create mode 100644 java/android/app/src/main/res/layout/keypointdetection_activity_main.xml create mode 100644 java/android/app/src/main/res/layout/keypointdetection_camera_page.xml create mode 100644 java/android/app/src/main/res/layout/keypointdetection_result_page.xml create mode 100644 java/android/app/src/main/res/xml/keypointdetection_settting.xml diff --git a/java/android/app/build.gradle b/java/android/app/build.gradle index bf28c2b28f..5db33d35bf 100644 --- a/java/android/app/build.gradle +++ b/java/android/app/build.gradle @@ -68,6 +68,10 @@ def FD_MODEL = [ [ 'src' : 'https://bj.bcebos.com/paddlehub/fastdeploy/portrait_pp_humansegv2_lite_256x144_inference_model.tgz', 'dest': 'src/main/assets/models' + ], + [ + 'src' : 'https://bj.bcebos.com/paddlehub/fastdeploy/PP_TinyPose_128x96_infer.tgz', + 'dest': 'src/main/assets/models' ] ] diff --git a/java/android/app/src/main/AndroidManifest.xml b/java/android/app/src/main/AndroidManifest.xml index 858ef04f62..16b6ae5a5b 100644 --- a/java/android/app/src/main/AndroidManifest.xml +++ b/java/android/app/src/main/AndroidManifest.xml @@ -15,14 +15,14 @@ android:roundIcon="@mipmap/ic_launcher_round" android:supportsRtl="true" android:theme="@style/AppTheme"> - + diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionMainActivity.java index 11448639c5..ab7acc6245 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionMainActivity.java @@ -1,4 +1,403 @@ package com.baidu.paddle.fastdeploy.app.examples.keypointdetection; -public class KeyPointDetectionMainActivity { +import static com.baidu.paddle.fastdeploy.ui.Utils.decodeBitmap; +import static com.baidu.paddle.fastdeploy.ui.Utils.getRealPathFromURI; + +import android.Manifest; +import android.annotation.SuppressLint; +import android.app.Activity; +import android.app.AlertDialog; +import android.content.DialogInterface; +import android.content.Intent; +import android.content.SharedPreferences; +import android.content.pm.PackageManager; +import android.graphics.Bitmap; +import android.net.Uri; +import android.os.Bundle; +import android.os.SystemClock; +import android.preference.PreferenceManager; +import android.support.annotation.NonNull; +import android.support.v4.app.ActivityCompat; +import android.support.v4.content.ContextCompat; +import android.util.Log; +import android.view.View; +import android.view.ViewGroup; +import android.view.Window; +import android.view.WindowManager; +import android.widget.ImageButton; +import android.widget.ImageView; +import android.widget.TextView; + +import com.baidu.paddle.fastdeploy.RuntimeOption; +import com.baidu.paddle.fastdeploy.app.examples.R; +import com.baidu.paddle.fastdeploy.ui.Utils; +import com.baidu.paddle.fastdeploy.ui.view.CameraSurfaceView; +import com.baidu.paddle.fastdeploy.ui.view.ResultListView; +import com.baidu.paddle.fastdeploy.ui.view.model.BaseResultModel; +import com.baidu.paddle.fastdeploy.vision.SegmentationResult; +import com.baidu.paddle.fastdeploy.vision.Visualize; +import com.baidu.paddle.fastdeploy.vision.KeyPointDetectionResult; +import com.baidu.paddle.fastdeploy.vision.keypointdetection.PPTinyPose; + +import java.util.ArrayList; +import java.util.List; + + +public class KeyPointDetectionMainActivity extends Activity implements View.OnClickListener, CameraSurfaceView.OnTextureChangedListener { + private static final String TAG = KeyPointDetectionMainActivity.class.getSimpleName(); + + CameraSurfaceView svPreview; + TextView tvStatus; + ImageButton btnSwitch; + ImageButton btnShutter; + ImageButton btnSettings; + ImageView realtimeToggleButton; + boolean isRealtimeStatusRunning = false; + ImageView backInPreview; + private ImageView albumSelectButton; + private View cameraPageView; + private ViewGroup resultPageView; + private ImageView resultImage; + private ImageView backInResult; + private ResultListView resultView; + private Bitmap shutterBitmap; + private Bitmap picBitmap; + private boolean isShutterBitmapCopied = false; + + public static final int TYPE_UNKNOWN = -1; + public static final int BTN_SHUTTER = 0; + public static final int ALBUM_SELECT = 1; + public static final int REALTIME_DETECT = 2; + private static int TYPE = REALTIME_DETECT; + + private static final int REQUEST_PERMISSION_CODE_STORAGE = 101; + private static final int INTENT_CODE_PICK_IMAGE = 100; + private static final int TIME_SLEEP_INTERVAL = 50; // ms + + long timeElapsed = 0; + long frameCounter = 0; + + // Call 'init' and 'release' manually later + PPTinyPose predictor = new PPTinyPose(); + private List results = new ArrayList<>(); + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + // Fullscreen + requestWindowFeature(Window.FEATURE_NO_TITLE); + getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN, WindowManager.LayoutParams.FLAG_FULLSCREEN); + + setContentView(R.layout.keypointdetection_activity_main); + + // Clear all setting items to avoid app crashing due to the incorrect settings + initSettings(); + + // Check and request CAMERA and WRITE_EXTERNAL_STORAGE permissions + if (!checkAllPermissions()) { + requestAllPermissions(); + } + + // Init the camera preview and UI components + initView(); + } + + @SuppressLint("NonConstantResourceId") + @Override + public void onClick(View v) { + switch (v.getId()) { + case R.id.btn_switch: + svPreview.switchCamera(); + break; + case R.id.btn_shutter: + TYPE = BTN_SHUTTER; + shutterAndPauseCamera(); + resultView.setAdapter(null); + break; + case R.id.btn_settings: + startActivity(new Intent(this, KeyPointDetectionSettingsActivity.class)); + break; + case R.id.realtime_toggle_btn: + toggleRealtimeStyle(); + break; + case R.id.back_in_preview: + finish(); + break; + case R.id.album_select: + TYPE = ALBUM_SELECT; + // Judge whether authority has been granted. + if (ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) { + // If this permission was requested before the application but the user refused the request, this method will return true. + ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.WRITE_EXTERNAL_STORAGE}, REQUEST_PERMISSION_CODE_STORAGE); + } else { + Intent intent = new Intent(Intent.ACTION_PICK); + intent.setType("image/*"); + startActivityForResult(intent, INTENT_CODE_PICK_IMAGE); + } + resultView.setAdapter(null); + break; + case R.id.back_in_result: + back(); + break; + } + } + + @Override + public void onBackPressed() { + super.onBackPressed(); + back(); + } + + private void back() { + resultPageView.setVisibility(View.GONE); + cameraPageView.setVisibility(View.VISIBLE); + TYPE = REALTIME_DETECT; + isShutterBitmapCopied = false; + svPreview.onResume(); + results.clear(); + } + + private void shutterAndPauseCamera() { + new Thread(new Runnable() { + @Override + public void run() { + try { + // Sleep some times to ensure picture has been correctly shut. + Thread.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms + } catch (InterruptedException e) { + e.printStackTrace(); + } + runOnUiThread(new Runnable() { + @SuppressLint("SetTextI18n") + public void run() { + // These codes will run in main thread. + svPreview.onPause(); + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); + if (shutterBitmap != null && !shutterBitmap.isRecycled()) { + detail(shutterBitmap); + } else { + new AlertDialog.Builder(KeyPointDetectionMainActivity.this) + .setTitle("Empty Result!") + .setMessage("Current picture is empty, please shutting it again!") + .setCancelable(true) + .show(); + } + } + }); + } + }).start(); + } + + private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { + if (isShutterBitmapCopied || ARGB8888ImageBitmap == null) { + return; + } + if (!ARGB8888ImageBitmap.isRecycled()) { + synchronized (this) { + shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + } + SystemClock.sleep(TIME_SLEEP_INTERVAL); + isShutterBitmapCopied = true; + } + } + + @Override + protected void onActivityResult(int requestCode, int resultCode, Intent data) { + super.onActivityResult(requestCode, resultCode, data); + if (requestCode == INTENT_CODE_PICK_IMAGE) { + if (resultCode == Activity.RESULT_OK) { + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); + Uri uri = data.getData(); + String path = getRealPathFromURI(this, uri); + Bitmap bitmap = decodeBitmap(path, 720, 1280); + picBitmap = bitmap.copy(Bitmap.Config.ARGB_8888, true); + SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms + detail(picBitmap); + } + } + } + + private void toggleRealtimeStyle() { + if (isRealtimeStatusRunning) { + isRealtimeStatusRunning = false; + realtimeToggleButton.setImageResource(R.drawable.realtime_stop_btn); + svPreview.setOnTextureChangedListener(this); + tvStatus.setVisibility(View.VISIBLE); + } else { + isRealtimeStatusRunning = true; + realtimeToggleButton.setImageResource(R.drawable.realtime_start_btn); + tvStatus.setVisibility(View.GONE); + isShutterBitmapCopied = false; + // Camera is still working but detecting loop is on pause. + svPreview.setOnTextureChangedListener(new CameraSurfaceView.OnTextureChangedListener() { + @Override + public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + if (TYPE == BTN_SHUTTER) { + copyBitmapFromCamera(ARGB8888ImageBitmap); + } + return false; + } + }); + } + } + + @Override + public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + if (TYPE == BTN_SHUTTER) { + copyBitmapFromCamera(ARGB8888ImageBitmap); + return false; + } + + boolean modified = false; + + long tc = System.currentTimeMillis(); + KeyPointDetectionResult result = predictor.predict(ARGB8888ImageBitmap); + timeElapsed += (System.currentTimeMillis() - tc); + + Visualize.visKeypointDetection(ARGB8888ImageBitmap, result, 0.f); + + modified = result.initialized(); + + frameCounter++; + if (frameCounter >= 30) { + final int fps = (int) (1000 / (timeElapsed / 30)); + runOnUiThread(new Runnable() { + @SuppressLint("SetTextI18n") + public void run() { + tvStatus.setText(Integer.toString(fps) + "fps"); + } + }); + frameCounter = 0; + timeElapsed = 0; + } + return modified; + } + + @Override + protected void onResume() { + super.onResume(); + // Reload settings and re-initialize the predictor + checkAndUpdateSettings(); + // Open camera until the permissions have been granted + if (!checkAllPermissions()) { + svPreview.disableCamera(); + } else { + svPreview.enableCamera(); + } + svPreview.onResume(); + } + + @Override + protected void onPause() { + super.onPause(); + svPreview.onPause(); + } + + @Override + protected void onDestroy() { + if (predictor != null) { + predictor.release(); + } + super.onDestroy(); + } + + public void initView() { + TYPE = REALTIME_DETECT; + // (1) EXPECTED_PREVIEW_WIDTH should mean 'height' and EXPECTED_PREVIEW_HEIGHT + // should mean 'width' if the camera display orientation is 90 | 270 degree + // (Hold the phone upright to record video) + // (2) Smaller resolution is more suitable for Human Pose detection on mobile + // device. So, we set this preview size (720,480) here. Reference: + // https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5/configs/keypoint/tiny_pose + CameraSurfaceView.EXPECTED_PREVIEW_WIDTH = 720; + CameraSurfaceView.EXPECTED_PREVIEW_HEIGHT = 480; + svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); + svPreview.setOnTextureChangedListener(this); + + tvStatus = (TextView) findViewById(R.id.tv_status); + btnSwitch = (ImageButton) findViewById(R.id.btn_switch); + btnSwitch.setOnClickListener(this); + btnShutter = (ImageButton) findViewById(R.id.btn_shutter); + btnShutter.setOnClickListener(this); + btnSettings = (ImageButton) findViewById(R.id.btn_settings); + btnSettings.setOnClickListener(this); + realtimeToggleButton = findViewById(R.id.realtime_toggle_btn); + realtimeToggleButton.setOnClickListener(this); + backInPreview = findViewById(R.id.back_in_preview); + backInPreview.setOnClickListener(this); + albumSelectButton = findViewById(R.id.album_select); + albumSelectButton.setOnClickListener(this); + cameraPageView = findViewById(R.id.camera_page); + resultPageView = findViewById(R.id.result_page); + resultImage = findViewById(R.id.result_image); + backInResult = findViewById(R.id.back_in_result); + backInResult.setOnClickListener(this); + resultView = findViewById(R.id.result_list_view); + } + + private void detail(Bitmap bitmap) { + predictor.predict(bitmap, true, 5.f); + resultImage.setImageBitmap(bitmap); + } + + @SuppressLint("ApplySharedPref") + public void initSettings() { + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this); + SharedPreferences.Editor editor = sharedPreferences.edit(); + editor.clear(); + editor.commit(); + KeyPointDetectionSettingsActivity.resetSettings(); + } + + public void checkAndUpdateSettings() { + if (KeyPointDetectionSettingsActivity.checkAndUpdateSettings(this)) { + String realModelDir = getCacheDir() + "/" + KeyPointDetectionSettingsActivity.modelDir; + Utils.copyDirectoryFromAssets(this, KeyPointDetectionSettingsActivity.modelDir, realModelDir); + + String modelFile = realModelDir + "/" + "model.pdmodel"; + String paramsFile = realModelDir + "/" + "model.pdiparams"; + String configFile = realModelDir + "/" + "infer_cfg.yml"; + RuntimeOption option = new RuntimeOption(); + option.setCpuThreadNum(KeyPointDetectionSettingsActivity.cpuThreadNum); + option.setLitePowerMode(KeyPointDetectionSettingsActivity.cpuPowerMode); + if (Boolean.parseBoolean(KeyPointDetectionSettingsActivity.enableLiteFp16)) { + option.enableLiteFp16(); + } + predictor.setUseDark(true); + predictor.init(modelFile, paramsFile, configFile, option); + } + } + + @Override + public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, + @NonNull int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + if (grantResults[0] != PackageManager.PERMISSION_GRANTED || grantResults[1] != PackageManager.PERMISSION_GRANTED) { + new AlertDialog.Builder(KeyPointDetectionMainActivity.this) + .setTitle("Permission denied") + .setMessage("Click to force quit the app, then open Settings->Apps & notifications->Target " + + "App->Permissions to grant all of the permissions.") + .setCancelable(false) + .setPositiveButton("Exit", new DialogInterface.OnClickListener() { + @Override + public void onClick(DialogInterface dialog, int which) { + KeyPointDetectionMainActivity.this.finish(); + } + }).show(); + } + } + + private void requestAllPermissions() { + ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.WRITE_EXTERNAL_STORAGE, + Manifest.permission.CAMERA}, 0); + } + + private boolean checkAllPermissions() { + return ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) == PackageManager.PERMISSION_GRANTED + && ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; + } + } diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionSettingsActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionSettingsActivity.java index ac0134fc25..d80abd7bdc 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionSettingsActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/keypointdetection/KeyPointDetectionSettingsActivity.java @@ -1,4 +1,164 @@ package com.baidu.paddle.fastdeploy.app.examples.keypointdetection; -public class KeyPointDetectionSettingsActivity { +import android.annotation.SuppressLint; +import android.content.Context; +import android.content.SharedPreferences; +import android.os.Bundle; +import android.preference.EditTextPreference; +import android.preference.ListPreference; +import android.preference.PreferenceManager; +import android.support.v7.app.ActionBar; + +import com.baidu.paddle.fastdeploy.app.examples.R; +import com.baidu.paddle.fastdeploy.ui.Utils; +import com.baidu.paddle.fastdeploy.ui.view.AppCompatPreferenceActivity; + +import java.util.ArrayList; +import java.util.List; + + +public class KeyPointDetectionSettingsActivity extends AppCompatPreferenceActivity implements + SharedPreferences.OnSharedPreferenceChangeListener { + private static final String TAG = KeyPointDetectionSettingsActivity.class.getSimpleName(); + + static public int selectedModelIdx = -1; + static public String modelDir = ""; + static public int cpuThreadNum = 2; + static public String cpuPowerMode = ""; + static public String enableLiteFp16 = "true"; + + ListPreference lpChoosePreInstalledModel = null; + EditTextPreference etModelDir = null; + ListPreference lpCPUThreadNum = null; + ListPreference lpCPUPowerMode = null; + ListPreference lpEnableLiteFp16 = null; + + List preInstalledModelDirs = null; + List preInstalledCPUThreadNums = null; + List preInstalledCPUPowerModes = null; + List preInstalledEnableLiteFp16s = null; + + @Override + public void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + addPreferencesFromResource(R.xml.keypointdetection_settting); + ActionBar supportActionBar = getSupportActionBar(); + if (supportActionBar != null) { + supportActionBar.setDisplayHomeAsUpEnabled(true); + } + + // Initialize pre-installed models + preInstalledModelDirs = new ArrayList(); + preInstalledCPUThreadNums = new ArrayList(); + preInstalledCPUPowerModes = new ArrayList(); + preInstalledEnableLiteFp16s = new ArrayList(); + preInstalledModelDirs.add(getString(R.string.KEYPOINT_DETECTION_MODEL_DIR_DEFAULT)); + preInstalledCPUThreadNums.add(getString(R.string.CPU_THREAD_NUM_DEFAULT)); + preInstalledCPUPowerModes.add(getString(R.string.CPU_POWER_MODE_DEFAULT)); + preInstalledEnableLiteFp16s.add(getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + + // Setup UI components + lpChoosePreInstalledModel = + (ListPreference) findPreference(getString(R.string.CHOOSE_PRE_INSTALLED_MODEL_KEY)); + String[] preInstalledModelNames = new String[preInstalledModelDirs.size()]; + for (int i = 0; i < preInstalledModelDirs.size(); i++) { + preInstalledModelNames[i] = preInstalledModelDirs.get(i).substring(preInstalledModelDirs.get(i).lastIndexOf("/") + 1); + } + lpChoosePreInstalledModel.setEntries(preInstalledModelNames); + lpChoosePreInstalledModel.setEntryValues(preInstalledModelDirs.toArray(new String[preInstalledModelDirs.size()])); + lpCPUThreadNum = (ListPreference) findPreference(getString(R.string.CPU_THREAD_NUM_KEY)); + lpCPUPowerMode = (ListPreference) findPreference(getString(R.string.CPU_POWER_MODE_KEY)); + etModelDir = (EditTextPreference) findPreference(getString(R.string.MODEL_DIR_KEY)); + etModelDir.setTitle("Model dir (SDCard: " + Utils.getSDCardDirectory() + ")"); + lpEnableLiteFp16 = (ListPreference) findPreference(getString(R.string.ENABLE_LITE_FP16_MODE_KEY)); + } + + @SuppressLint("ApplySharedPref") + private void reloadSettingsAndUpdateUI() { + SharedPreferences sharedPreferences = getPreferenceScreen().getSharedPreferences(); + + String selected_model_dir = sharedPreferences.getString(getString(R.string.CHOOSE_PRE_INSTALLED_MODEL_KEY), + getString(R.string.KEYPOINT_DETECTION_MODEL_DIR_DEFAULT)); + int selected_model_idx = lpChoosePreInstalledModel.findIndexOfValue(selected_model_dir); + if (selected_model_idx >= 0 && selected_model_idx < preInstalledModelDirs.size() && selected_model_idx != selectedModelIdx) { + SharedPreferences.Editor editor = sharedPreferences.edit(); + editor.putString(getString(R.string.MODEL_DIR_KEY), preInstalledModelDirs.get(selected_model_idx)); + editor.putString(getString(R.string.CPU_THREAD_NUM_KEY), preInstalledCPUThreadNums.get(selected_model_idx)); + editor.putString(getString(R.string.CPU_POWER_MODE_KEY), preInstalledCPUPowerModes.get(selected_model_idx)); + editor.putString(getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT), preInstalledEnableLiteFp16s.get(selected_model_idx)); + editor.commit(); + lpChoosePreInstalledModel.setSummary(selected_model_dir); + selectedModelIdx = selected_model_idx; + } + + String model_dir = sharedPreferences.getString(getString(R.string.MODEL_DIR_KEY), + getString(R.string.KEYPOINT_DETECTION_MODEL_DIR_DEFAULT)); + String cpu_thread_num = sharedPreferences.getString(getString(R.string.CPU_THREAD_NUM_KEY), + getString(R.string.CPU_THREAD_NUM_DEFAULT)); + String cpu_power_mode = sharedPreferences.getString(getString(R.string.CPU_POWER_MODE_KEY), + getString(R.string.CPU_POWER_MODE_DEFAULT)); + String enable_lite_fp16 = sharedPreferences.getString(getString(R.string.ENABLE_LITE_FP16_MODE_KEY), + getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + + etModelDir.setSummary(model_dir); + lpCPUThreadNum.setValue(cpu_thread_num); + lpCPUThreadNum.setSummary(cpu_thread_num); + lpCPUPowerMode.setValue(cpu_power_mode); + lpCPUPowerMode.setSummary(cpu_power_mode); + lpEnableLiteFp16.setValue(enable_lite_fp16); + lpEnableLiteFp16.setSummary(enable_lite_fp16); + } + + static boolean checkAndUpdateSettings(Context ctx) { + boolean settingsChanged = false; + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(ctx); + + String model_dir = sharedPreferences.getString(ctx.getString(R.string.MODEL_DIR_KEY), + ctx.getString(R.string.KEYPOINT_DETECTION_MODEL_DIR_DEFAULT)); + settingsChanged |= !modelDir.equalsIgnoreCase(model_dir); + modelDir = model_dir; + + String cpu_thread_num = sharedPreferences.getString(ctx.getString(R.string.CPU_THREAD_NUM_KEY), + ctx.getString(R.string.CPU_THREAD_NUM_DEFAULT)); + settingsChanged |= cpuThreadNum != Integer.parseInt(cpu_thread_num); + cpuThreadNum = Integer.parseInt(cpu_thread_num); + + String cpu_power_mode = sharedPreferences.getString(ctx.getString(R.string.CPU_POWER_MODE_KEY), + ctx.getString(R.string.CPU_POWER_MODE_DEFAULT)); + settingsChanged |= !cpuPowerMode.equalsIgnoreCase(cpu_power_mode); + cpuPowerMode = cpu_power_mode; + + String enable_lite_fp16 = sharedPreferences.getString(ctx.getString(R.string.ENABLE_LITE_FP16_MODE_KEY), + ctx.getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + settingsChanged |= !enableLiteFp16.equalsIgnoreCase(enable_lite_fp16); + enableLiteFp16 = enable_lite_fp16; + + return settingsChanged; + } + + static void resetSettings() { + selectedModelIdx = -1; + modelDir = ""; + cpuThreadNum = 2; + cpuPowerMode = ""; + enableLiteFp16 = "true"; + } + + @Override + protected void onResume() { + super.onResume(); + getPreferenceScreen().getSharedPreferences().registerOnSharedPreferenceChangeListener(this); + reloadSettingsAndUpdateUI(); + } + + @Override + protected void onPause() { + super.onPause(); + getPreferenceScreen().getSharedPreferences().unregisterOnSharedPreferenceChangeListener(this); + } + + @Override + public void onSharedPreferenceChanged(SharedPreferences sharedPreferences, String key) { + reloadSettingsAndUpdateUI(); + } } diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java index 563a56a524..ec2ebbffad 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java @@ -367,7 +367,7 @@ public void checkAndUpdateSettings() { if (Boolean.parseBoolean(SegmentationSettingsActivity.enableLiteFp16)) { option.enableLiteFp16(); } - predictor.setVerticalScreenFlag(true); + predictor.setIsVerticalScreen(true); predictor.init(modelFile, paramsFile, configFile, option); } } diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationSettingsActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationSettingsActivity.java index c7357a2319..ae32e960aa 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationSettingsActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationSettingsActivity.java @@ -96,8 +96,6 @@ private void reloadSettingsAndUpdateUI() { getString(R.string.CPU_THREAD_NUM_DEFAULT)); String cpu_power_mode = sharedPreferences.getString(getString(R.string.CPU_POWER_MODE_KEY), getString(R.string.CPU_POWER_MODE_DEFAULT)); - String score_threshold = sharedPreferences.getString(getString(R.string.SCORE_THRESHOLD_KEY), - getString(R.string.SCORE_THRESHOLD_FACEDET)); String enable_lite_fp16 = sharedPreferences.getString(getString(R.string.ENABLE_LITE_FP16_MODE_KEY), getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); diff --git a/java/android/app/src/main/res/layout/keypointdetection_activity_main.xml b/java/android/app/src/main/res/layout/keypointdetection_activity_main.xml new file mode 100644 index 0000000000..cdebe4d37e --- /dev/null +++ b/java/android/app/src/main/res/layout/keypointdetection_activity_main.xml @@ -0,0 +1,14 @@ + + + + + + + diff --git a/java/android/app/src/main/res/layout/keypointdetection_camera_page.xml b/java/android/app/src/main/res/layout/keypointdetection_camera_page.xml new file mode 100644 index 0000000000..fde1e55c31 --- /dev/null +++ b/java/android/app/src/main/res/layout/keypointdetection_camera_page.xml @@ -0,0 +1,159 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/keypointdetection_result_page.xml b/java/android/app/src/main/res/layout/keypointdetection_result_page.xml new file mode 100644 index 0000000000..f1e4cbd92c --- /dev/null +++ b/java/android/app/src/main/res/layout/keypointdetection_result_page.xml @@ -0,0 +1,160 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/values/strings.xml b/java/android/app/src/main/res/values/strings.xml index 267871056f..d1bfd9ace2 100644 --- a/java/android/app/src/main/res/values/strings.xml +++ b/java/android/app/src/main/res/values/strings.xml @@ -21,6 +21,7 @@ 0.4 0.1 0.25 + 0.25 true @@ -36,6 +37,8 @@ models/scrfd_500m_bnkps_shape320x320_pd models/portrait_pp_humansegv2_lite_256x144_inference_model + + models/PP_TinyPose_128x96_infer 拍照识别 实时识别 diff --git a/java/android/app/src/main/res/xml/keypointdetection_settting.xml b/java/android/app/src/main/res/xml/keypointdetection_settting.xml new file mode 100644 index 0000000000..c43965fe98 --- /dev/null +++ b/java/android/app/src/main/res/xml/keypointdetection_settting.xml @@ -0,0 +1,37 @@ + + + + + + + + \ No newline at end of file diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.cc index b0e74dfe26..02a67d9860 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/keypointdetection/keypointdetection_utils_jni.cc @@ -27,8 +27,8 @@ void RenderingKeyPointDetection( jstring save_path) { if (!c_result.keypoints.empty()) { auto t = GetCurrentTime(); - cv::Mat c_vis_im; - vision::VisKeypointDetection(c_bgr, c_result, conf_threshold); + auto c_vis_im = vision::VisKeypointDetection( + c_bgr, c_result, conf_threshold); LOGD("Visualize from native costs %f ms", GetElapsedTime(t)); if (!BGR2ARGB888Bitmap(env, argb8888_bitmap, c_vis_im)) { diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc index af97f6c829..42468ee4dd 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/visualize_jni.cc @@ -227,7 +227,7 @@ jboolean VisFaceDetectionFromJava( jboolean VisKeyPointDetectionFromJava( JNIEnv *env, jobject argb8888_bitmap, jobject result, - jint conf_threshold) { + jfloat conf_threshold) { const jclass j_keypoint_det_result_clazz = env->FindClass( "com/baidu/paddle/fastdeploy/vision/KeyPointDetectionResult"); if (!env->IsInstanceOf(result, j_keypoint_det_result_clazz)) { diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/keypointdetection/PPTinyPose.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/keypointdetection/PPTinyPose.java index db8ff5ffd2..f017d2e487 100644 --- a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/keypointdetection/PPTinyPose.java +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/keypointdetection/PPTinyPose.java @@ -30,6 +30,10 @@ public PPTinyPose(String modelFile, init_(modelFile, paramsFile, configFile, runtimeOption); } + public void setUseDark(boolean flag) { + mUseDark = flag; + } + // Call init manually without label file public boolean init(String modelFile, String paramsFile, diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/segmentation/PaddleSegModel.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/segmentation/PaddleSegModel.java index 8ca586cb35..31e3a7393c 100644 --- a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/segmentation/PaddleSegModel.java +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/segmentation/PaddleSegModel.java @@ -22,12 +22,6 @@ public PaddleSegModel(String modelFile, init_(modelFile, paramsFile, configFile, new RuntimeOption()); } - // Is vertical screen or not, for PP-HumanSeg on vertical screen, - // this flag must be 'true'. - public void setVerticalScreenFlag(boolean flag) { - mIsVerticalScreen = flag; - } - // Constructor with custom runtime option public PaddleSegModel(String modelFile, String paramsFile, @@ -43,6 +37,17 @@ public boolean init(String modelFile, return init_(modelFile, paramsFile, configFile, runtimeOption); } + // Deprecated. Please use setIsVerticalScreen instead. + public void setVerticalScreenFlag(boolean flag) { + mIsVerticalScreen = flag; + } + + // Is vertical screen or not, for PP-HumanSeg on vertical screen, + // this flag must be 'true'. + public void setIsVerticalScreen(boolean flag) { + mIsVerticalScreen = flag; + } + public boolean release() { mInitialized = false; if (mCxxContext == 0) { From a6bd4ca23f6d33204c4fa67b94d7597aff0ead57 Mon Sep 17 00:00:00 2001 From: DefTruth Date: Wed, 30 Nov 2022 16:24:19 +0800 Subject: [PATCH 6/6] update app build.gradle --- java/android/app/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/android/app/build.gradle b/java/android/app/build.gradle index 5db33d35bf..d58cb3dd57 100644 --- a/java/android/app/build.gradle +++ b/java/android/app/build.gradle @@ -22,13 +22,13 @@ android { } dependencies { - // implementation fileTree(include: ['*.aar'], dir: 'libs') + implementation fileTree(include: ['*.aar'], dir: 'libs') implementation 'com.android.support:appcompat-v7:28.0.0' //noinspection GradleDependency implementation 'com.android.support.constraint:constraint-layout:1.1.3' implementation 'com.android.support:design:28.0.0' implementation 'org.jetbrains:annotations:15.0' - implementation project(path: ':fastdeploy') + // implementation project(path: ':fastdeploy') implementation project(path: ':ui') //noinspection GradleDependency testImplementation 'junit:junit:4.12'