How to get facial landmarks on Android platform?

Android/Java
1

I tried to use the dlib library on Android platform, but the detection speed is very slow. Is there any other way to get facial landmarks？

2

if you can build with contrib modules,
there are landmarks detectors here
(68 pts, similar to dlib)
(i’d recommend the LBF one)

if you can live with 5 lms (eyes, nose, mouth), there’s a combined face / lm detector here
(it’s not (yet) in the docs, but you can find example usage here)

i find that hard to believe.
are you doing something silly, like using their face detector before that (gross slow!),
or even - load the 68mb model each time you use it ?

please share (relevant piece of) code !

3

// at the beginning, i load the modle file(shape_predictor_68_face_landmarks.dat). this file is at the assets.

void FaceDetector::loadShapePredictor(JNIEnv *env, jobject thiz,
                        jobject assetManager,
                        jstring fileName) {
    const char *file_name = env->GetStringUTFChars(fileName, nullptr);
    env->ReleaseStringUTFChars(fileName, file_name);

    //get AAssetManager
    AAssetManager *native_asset = AAssetManager_fromJava(env, assetManager);

    //open file
    AAsset *assetFile = AAssetManager_open(native_asset, file_name, AASSET_MODE_BUFFER);
    //get file length
    size_t file_length = static_cast<size_t>(AAsset_getLength(assetFile));
    char *model_buffer = (char *) malloc(file_length);
    //read file data
    AAsset_read(assetFile, model_buffer, file_length);
    //the data has been copied to model_buffer, so , close it
    AAsset_close(assetFile);

    //LOGI("asset file length %d", file_length);

    //char* to istream
    membuf mem_buf(model_buffer, model_buffer + file_length);
    std::istream in(&mem_buf);

    //load shape_predictor_68_face_landmarks.dat from memory
    dlib::deserialize(this->predictor, in);

    //free malloc
    free(model_buffer);
}

// in ImageReader.OnImageAvailableListener(), get image and detect faces

private ImageReader.OnImageAvailableListener mOnPreviewImageAvailableListener = new ImageReader.OnImageAvailableListener() {
        @Override
        public void onImageAvailable(ImageReader reader) {
            Image image = reader.acquireLatestImage();
            System.out.println("image.getFormat(): " + image.getFormat());
            if (image != null) {
                if (mCamera2FrameCallback != null) {
                	// convert image to yuv_420_888_data
                    byte[] yuv_420_888_data = CameraUtil.YUV_420_888_data(image);
                    mCamera2FrameCallback.onPreviewFrame(yuv_420_888_data, image.getWidth(), image.getHeight());
                    int width = image.getWidth();
                    int height = image.getHeight();
                    int length = width * height * ImageFormat.getBitsPerPixel(ImageFormat.YUV_420_888) / 8;
                    // detect faces
                    int countFaces = mFaceTracker.countFacesYUV(yuv_420_888_data, width, height);
                    System.out.println("it has " + countFaces + " faces");
                }
                image.close();
            }
        }
    };

extern "C"
JNIEXPORT jint JNICALL
Java_cn_twan_headposedlib_face_FaceTracker_countFacesYUV(JNIEnv *env, jobject thiz,
                                                         jbyteArray data, jint width, jint height) {
    // TODO: implement countFacesYUV()

    LOGCATE("countFacesYUV");
    jbyte *data_ = env->GetByteArrayElements(data, NULL);
    FaceDetector *mDetPtr = FaceDetector::getFaceTracker(env, thiz);
    cv::Mat rgbaMat(height, width, CV_8UC1, data_);

    cv::Mat bgrMat;
    cv::cvtColor(rgbaMat, bgrMat, cv::COLOR_YUV2BGR_NV21);
    jint size = mDetPtr->countFaces(bgrMat);

    return size;
}

int FaceDetector::countFaces(const cv::Mat &image) {
    if (image.empty())
        return 0;

    LOGD("image.channels: %d", image.channels());

    if (image.channels() == 1) {
        cv::cvtColor(image, image, cv::COLOR_GRAY2BGR);
    }

    cv::Mat img_gray;
    cv::cvtColor(image, img_gray, cv::COLOR_BGR2GRAY);
    dlib::array2d<dlib::bgr_pixel> dlib_image;
    dlib::assign_image(dlib_image, dlib::cv_image<uchar>(img_gray));

    det_rects.clear();

    det_rects = face_detector(dlib_image);

    for (unsigned long i = 0; i < det_rects.size(); i++) {
        LOGD("before predictor");
        dlib::full_object_detection shape = predictor(dlib_image, det_rects[i]);
        LOGD("shape.num_parts: %ld", shape.num_parts());
        for (unsigned long j = 0; j < shape.num_parts(); j++) {
            cv::circle(image, cv::Point2l(shape.part(j).x(), shape.part(j).y()), 2, cv::COLORMAP_PINK);
            LOGD("shapes[%ld].part(%ld): (%ld, %ld)", i, j, shape.part(j).x(), shape.part(j).y());
        }
        shapes.push_back(shape);
    }

    LOGD("shapes.size(): %d", shapes.size());

    return det_rects.size();
}
4

why on earth is it even using landmarks ?
it’s only counting faces !

please profile this

algo works on grayscale images, convert directly:
https://docs.opencv.org/4.x/d8/d01/group__imgproc__color__conversions.html#gga4e0972be5de079fed4e3a10e24ef5ef0a1cba7092b4f0302693526ca578159ea0

why ? does not make sense

5

I want to get landmarks to add stickers on the faces. at first, i want to judge whether it has faces, so, i return the count of faces. i haven’t used landmarks yet.
And i met another problem. the count of faces is always 0 when i’m facing the front camera. i guess it may caused by the pictures data. but i have no idea.

i changed cv::COLOR_YUV2BGR_NV21 into cv::COLOR_YUV420p2GRAY.

thanks for your help.