From a8217fdec39dbcb570726a4af5246501b8406468 Mon Sep 17 00:00:00 2001 From: searching46dof <107732149+searching46dof@users.noreply.github.com> Date: Mon, 12 Sep 2022 21:55:45 -0400 Subject: [PATCH 1/6] losing tracking due to small rectangle returned by proc_face_detect for larger distances. The symptom is currently losing face tracking at distance of approx 2 meters. the additional margin of 0.1 * width and 0.1 * height is only effective for short ranges (e.g. less than 1 meters) since the face is relatively large. For distances up to 2 meters, need additional margins approx 0.2 x width and 0.2 x height since the face is smaller For distances up to 3 meters , need additional margins approx 0.3 x width and 0.3 x height. For distances up to 4 meters, need additional margins approx 0.4 x width and 0.4 x height. --- AITracker/src/model.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/AITracker/src/model.cpp b/AITracker/src/model.cpp index 22b6fb8..3d239ce 100644 --- a/AITracker/src/model.cpp +++ b/AITracker/src/model.cpp @@ -199,10 +199,13 @@ void Tracker::proc_face_detect(float* face, float width, float height) float w = face[2]; float h = face[3]; - int crop_x1 = (int)(x); - int crop_y1 = (int)(y); - int crop_x2 = (int)(x + w); - int crop_y2 = (int)(y + h + h * 0.1f); // force a little taller BB so the chin tends to be covered + int additional_width_margin = (int)(w * 0.4f); + int additional_height_margin = (int)(h * 0.4f); + + int crop_x1 = (int)(x - additional_width_margin); + int crop_y1 = (int)(y - additional_height_margin); + int crop_x2 = (int)(x + w + additional_width_margin); + int crop_y2 = (int)(y + h + additional_height_margin); // force a little taller BB so the chin tends to be covered face[0] = (float)std::max(0, crop_x1); face[1] = (float)std::max(0, crop_y1); From ff6a4313e1ebfc42a61d189c077ef07a551d21a5 Mon Sep 17 00:00:00 2001 From: searching46dof <107732149+searching46dof@users.noreply.github.com> Date: Mon, 12 Sep 2022 22:46:00 -0400 Subject: [PATCH 2/6] CameraFactory::getCameras should enumerate multiple PS3 Eye cameras Currently CameraFactory::getCameras only enumerates 1 PS3 eye camera and up to 4 OCV cameras. Should be able to enumerated multiple PS3 eye cameras --- Client/src/camera/CameraFactory.cpp | 43 +++++++++++++++++------------ 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/Client/src/camera/CameraFactory.cpp b/Client/src/camera/CameraFactory.cpp index d5bc034..3766e53 100644 --- a/Client/src/camera/CameraFactory.cpp +++ b/Client/src/camera/CameraFactory.cpp @@ -42,32 +42,41 @@ std::vector> CameraFactory::getCameras(CameraSettings& s { std::vector> cams; - // Search first for any PS3 camera. - try - { - cams.push_back(std::make_shared(640, 480, 60)); - cams[0]->set_settings(settings); - } - catch (...) - { - std::cout << "No PS3 camera available." << std::endl; - cams.clear(); - } - - for (int i = 0; i < 5; i++) { + bool bFoundPs3Camera = false; + // Search first for any PS3 camera. try { - std::shared_ptr c = std::make_shared(settings.width, settings.height, settings.fps, i); + std::shared_ptr c = std::make_shared(640, 480, 60); c->set_settings(settings); // Brightness / Exposure cams.push_back(std::move(c)); - std::cout << "Found ID: " << i << std::endl; + bFoundPs3Camera = true; + std::cout << "Found PS3 camera ID: " << i << std::endl; } - catch (const std::exception&) + catch (...) { - std::cout << "Not found device" << i << std::endl; } + + if (!bFoundPs3Camera) + { + // Then search or OCV Camera. + try + { + std::shared_ptr c = std::make_shared(settings.width, settings.height, settings.fps, i); + c->set_settings(settings); // Brightness / Exposure + cams.push_back(std::move(c)); + std::cout << "Found OCV camera ID: " << i << std::endl; + } + catch (const std::exception&) + { + } + } + + } + if (cams.size() == 0) + { + std::cout << "No cameras found" << std::endl; } return cams; From 3ba8f1264a264d62c1d2e139c5b88dafc0315210 Mon Sep 17 00:00:00 2001 From: searching46dof <107732149+searching46dof@users.noreply.github.com> Date: Mon, 12 Sep 2022 22:46:39 -0400 Subject: [PATCH 3/6] Revert "Add center weighted face detection to version 0.6.6 using cv::FaceDetectorYN" This reverts commit 943aeae9d548a9ca198d40518e5bfe2d6ae3b64e. --- AITracker/src/model.cpp | 54 +++++------------------------------------ AITracker/src/model.h | 3 --- 2 files changed, 6 insertions(+), 51 deletions(-) diff --git a/AITracker/src/model.cpp b/AITracker/src/model.cpp index 3d239ce..cf13664 100644 --- a/AITracker/src/model.cpp +++ b/AITracker/src/model.cpp @@ -30,8 +30,8 @@ Tracker::Tracker(std::unique_ptr&& solver, std::wstring& detecti session_lm = std::make_unique(*enviro, landmark_model_path.data(), session_options); // Face detector - float score_threshold = .8f; - float nms_threshold = .3f; + float score_threshold = .8; + float nms_threshold = .3; int topK = 50; face_detector = cv::FaceDetectorYN::create( std::string(detection_model_path.begin(), detection_model_path.end()), @@ -82,44 +82,6 @@ float inline logit(float p) return log(p) / 16.0f; } -float Tracker::get_distance_squared(float x0, float y0, float x1, float y1) -{ - // calculate distance squared. - // no need to for sqrt to obtain the smallest distance for optimization - float x_distance = (x1 - x0); - float y_distance = (y1 - y0); - float distance_squared = (x_distance * x_distance) + (y_distance * y_distance); - return distance_squared; -} - -int Tracker::get_center_weighted_faces_row(const cv::Mat& image, const cv::Mat& faces) -{ - // get center coordinates for image - float image_center_x = (float)(image.rows / 2); - float image_center_y = (float)(image.cols / 2); - - float smallest_distance_squared = 0.0f; - int center_weighted_face_row = -1; - for (int row = 0; row < faces.rows; row++) - { - // get center coordinates for faces at row - float x0 = faces.at(row, 0); - float y0 = faces.at(row, 1); - float face_w = faces.at(row, 2); - float face_h = faces.at(row, 3); - float face_center_x = x0 + (face_w / 2); - float face_center_y = y0 + (face_h / 2); - - float distance_squared = get_distance_squared(image_center_x, image_center_y, face_center_x, face_center_y); - if ((center_weighted_face_row == -1) || (distance_squared < smallest_distance_squared)) - { - center_weighted_face_row = row; - smallest_distance_squared = distance_squared; - } - } - return center_weighted_face_row; -} - void Tracker::detect_face(const cv::Mat& image, FaceData& face_data) { cv::Mat resized, faces; @@ -139,14 +101,10 @@ void Tracker::detect_face(const cv::Mat& image, FaceData& face_data) if (faces.rows > 0) { face_data.face_detected = true; - int faces_row = 0; - bool center_weighted = true; // make center weighted face detection configurable - if (center_weighted) - faces_row = get_center_weighted_faces_row(image, faces); - float x0 = faces.at(faces_row, 0); - float y0 = faces.at(faces_row, 1); - float face_w = faces.at(faces_row, 2); - float face_h = faces.at(faces_row, 3); + float x0 = faces.at(0, 0); + float y0 = faces.at(0, 1); + float face_w = faces.at(0, 2); + float face_h = faces.at(0, 3); float w_ratio = (width / im_width); diff --git a/AITracker/src/model.h b/AITracker/src/model.h index 739a834..a7a2804 100644 --- a/AITracker/src/model.h +++ b/AITracker/src/model.h @@ -48,7 +48,4 @@ class Tracker void proc_face_detect(float* face, float width = 1080, float height = 720); void detect_landmarks(const cv::Mat& image, int x0, int y0, float scale_x, float scale_y, FaceData& face_data); void proc_heatmaps(float* heatmaps, int x0, int y0, float scale_x, float scale_y, FaceData& face_data); - - float get_distance_squared(float x0, float y0, float x1, float y1); - int get_center_weighted_faces_row(const cv::Mat& image, const cv::Mat& faces); }; From 55bde3fa94d7a139741a51598058c70f0608fb09 Mon Sep 17 00:00:00 2001 From: searching46dof <107732149+searching46dof@users.noreply.github.com> Date: Tue, 10 Jan 2023 20:56:32 -0500 Subject: [PATCH 4/6] Update model.cpp Need to increase the boundary for face detection by 10% to compensate for increased size due to yaw, pitch and roll But we also need a minimum amount of pixels for detection at farther distances where the face is smaller --- AITracker/src/model.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/AITracker/src/model.cpp b/AITracker/src/model.cpp index cf13664..df3cc00 100644 --- a/AITracker/src/model.cpp +++ b/AITracker/src/model.cpp @@ -157,8 +157,16 @@ void Tracker::proc_face_detect(float* face, float width, float height) float w = face[2]; float h = face[3]; - int additional_width_margin = (int)(w * 0.4f); - int additional_height_margin = (int)(h * 0.4f); + /* Need to increase the boundary for face detection by 10% to compensate for increased size due to yaw, pitch and roll */ + /* But we also need a minimum amount of pixels for detection at farther distances where the face is smaller */ + int additional_width_margin = (int)(w * 0.1f); + int minimum_width_margin = (int)width / (4 * 10); + if (additional_width_margin < minimum_width_margin) + additional_width_margin = minimum_width_margin; + int additional_height_margin = (int)(h * 0.1f); + int minimum_height_margin = (int)height / (4 * 10); + if (additional_height_margin < minimum_height_margin) + additional_height_margin = minimum_height_margin; int crop_x1 = (int)(x - additional_width_margin); int crop_y1 = (int)(y - additional_height_margin); From da450af7e260f1e847facdb40fda79c577861e24 Mon Sep 17 00:00:00 2001 From: searching46dof <107732149+searching46dof@users.noreply.github.com> Date: Tue, 10 Jan 2023 21:19:01 -0500 Subject: [PATCH 5/6] Update model.cpp resolve conflicts --- AITracker/src/model.cpp | 46 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/AITracker/src/model.cpp b/AITracker/src/model.cpp index df3cc00..c30c7ba 100644 --- a/AITracker/src/model.cpp +++ b/AITracker/src/model.cpp @@ -30,9 +30,9 @@ Tracker::Tracker(std::unique_ptr&& solver, std::wstring& detecti session_lm = std::make_unique(*enviro, landmark_model_path.data(), session_options); // Face detector - float score_threshold = .8; - float nms_threshold = .3; - int topK = 50; + float score_threshold = .8f; + float nms_threshold = .5f; + int topK = 7; face_detector = cv::FaceDetectorYN::create( std::string(detection_model_path.begin(), detection_model_path.end()), "", @@ -82,7 +82,45 @@ float inline logit(float p) return log(p) / 16.0f; } -void Tracker::detect_face(const cv::Mat& image, FaceData& face_data) +float StandardTracker::get_distance_squared(float x0, float y0, float x1, float y1) +{ + // calculate distance squared. + // no need to for sqrt to obtain the smallest distance for optimization + float x_distance = (x1 - x0); + float y_distance = (y1 - y0); + float distance_squared = (x_distance * x_distance) + (y_distance * y_distance); + return distance_squared; +} + +int StandardTracker::get_center_weighted_faces_row(const cv::Mat& image, const cv::Mat& faces) +{ + // get center coordinates for image + float image_center_x = (float)(image.rows / 2); + float image_center_y = (float)(image.cols / 2); + + float smallest_distance_squared = 0.0f; + int center_weighted_face_row = -1; + for (int row = 0; row < faces.rows; row++) + { + // get center coordinates for faces at row + float x0 = faces.at(row, 0); + float y0 = faces.at(row, 1); + float face_w = faces.at(row, 2); + float face_h = faces.at(row, 3); + float face_center_x = x0 + (face_w / 2); + float face_center_y = y0 + (face_h / 2); + + float distance_squared = get_distance_squared(image_center_x, image_center_y, face_center_x, face_center_y); + if ((center_weighted_face_row == -1) || (distance_squared < smallest_distance_squared)) + { + center_weighted_face_row = row; + smallest_distance_squared = distance_squared; + } + } + return center_weighted_face_row; +} + +void StandardTracker::detect_face(const cv::Mat& image, FaceData& face_data) { cv::Mat resized, faces; cv::resize(image, resized, cv::Size(224, 224), NULL, NULL, cv::INTER_LINEAR); From f521283417c2d60ea4c94d41c7bc6b5e288296eb Mon Sep 17 00:00:00 2001 From: searching46dof <107732149+searching46dof@users.noreply.github.com> Date: Tue, 10 Jan 2023 21:41:50 -0500 Subject: [PATCH 6/6] Update model.cpp resolve conflicts --- AITracker/src/model.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/AITracker/src/model.cpp b/AITracker/src/model.cpp index c30c7ba..3572796 100644 --- a/AITracker/src/model.cpp +++ b/AITracker/src/model.cpp @@ -139,12 +139,15 @@ void StandardTracker::detect_face(const cv::Mat& image, FaceData& face_data) if (faces.rows > 0) { face_data.face_detected = true; - float x0 = faces.at(0, 0); - float y0 = faces.at(0, 1); - float face_w = faces.at(0, 2); - float face_h = faces.at(0, 3); - - + int faces_row = 0; + bool center_weighted = true; // make center weighted face detection configurable + if (center_weighted) + faces_row = get_center_weighted_faces_row(image, faces); + float x0 = faces.at(faces_row, 0); + float y0 = faces.at(faces_row, 1); + float face_w = faces.at(faces_row, 2); + float face_h = faces.at(faces_row, 3); + float w_ratio = (width / im_width); float h_ratio = (height / im_height);