From d11550e68ec21f1d3bf83a02983d0b01447e5d13 Mon Sep 17 00:00:00 2001 From: Gavin Henderson Date: Fri, 11 Mar 2022 09:18:45 +0000 Subject: [PATCH 1/7] Allow for only tracking one eye on hold mode --- .../hooks/use-blink/use-hold-blink.js | 58 ++++++++++++++++++- src/react-app/lib/store-consts.js | 1 + 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/src/react-app/hooks/use-blink/use-hold-blink.js b/src/react-app/hooks/use-blink/use-hold-blink.js index f7dded58..8d5ce561 100644 --- a/src/react-app/hooks/use-blink/use-hold-blink.js +++ b/src/react-app/hooks/use-blink/use-hold-blink.js @@ -1,6 +1,7 @@ import { useCallback } from "react"; import { BLINK_LENGTH_KEY, + HOLD_BLINK_EYE_TO_TRACK, CHANGE_THRESHOLD_HOLD_KEY, } from "../../lib/store-consts"; import { useStoreValue } from "../use-store"; @@ -25,9 +26,16 @@ export const useHoldBlink = (onBlink, setDisplayOnSlider) => { update: updateBlinkLength, } = useStoreValue(BLINK_LENGTH_KEY, 500); + const { + loading: loadingEyeToTrack, + value: eyeToTrack, + update: updateEyeToTrack, + } = useStoreValue(HOLD_BLINK_EYE_TO_TRACK, "both"); + const noop = useCallback( (results, currentTimestamp, distanceHistory) => { if (loadingBlinkThreshold) return; + if (loadingEyeToTrack) return; if (results.multiFaceLandmarks) { for (const landmarks of results.multiFaceLandmarks) { @@ -54,7 +62,17 @@ export const useHoldBlink = (onBlink, setDisplayOnSlider) => { const reRatio = rhDistance / rvDistance; const leRatio = lhDistance / lvDistance; - const ratio = (reRatio + leRatio) / 2; + let ratio = null; + + if (eyeToTrack === "both") { + ratio = (reRatio + leRatio) / 2; + } else if (eyeToTrack === "right") { + ratio = reRatio; + } else if (eyeToTrack === "left") { + ratio = leRatio; + } else { + throw new Error(`Invalid eyeToTrack: ${eyeToTrack}`); + } const currentFrame = { time: currentTimestamp, @@ -95,14 +113,21 @@ export const useHoldBlink = (onBlink, setDisplayOnSlider) => { blinkLength, onBlink, setDisplayOnSlider, + loadingEyeToTrack, + eyeToTrack, ] ); + const eyesToTrackHighlights = { + both: { leftEye: true, rightEye: true }, + left: { leftEye: true, rightEye: false }, + right: { leftEye: false, rightEye: true }, + }; + return { detectBlink: noop, highlights: { - leftEye: true, - rightEye: true, + ...eyesToTrackHighlights[eyeToTrack], face: true, leftPupil: false, rightPupil: false, @@ -136,6 +161,33 @@ export const useHoldBlink = (onBlink, setDisplayOnSlider) => { updateBlinkLength(newValue); }, }, + { + loadingOption: loadingEyeToTrack, + type: "radio", + options: [ + { + value: "both", + name: "Both", + tooltip: "Detects both eyes", + }, + { + value: "left", + name: "Left", + tooltip: "Detects only the left eye", + }, + { + value: "right", + name: "Right", + tooltip: "Detects only the right eye", + }, + ], + defaultValue: loadingEyeToTrack ? "both" : eyeToTrack, + label: "Eyes to track", + tooltip: "Decide which eyes you want to track", + onChange: (newValue) => { + updateEyeToTrack(newValue); + }, + }, ], }; }; diff --git a/src/react-app/lib/store-consts.js b/src/react-app/lib/store-consts.js index 60e6b59b..9d50bc4c 100644 --- a/src/react-app/lib/store-consts.js +++ b/src/react-app/lib/store-consts.js @@ -18,3 +18,4 @@ export const DIRECTION_DEPTH_HOLD_KEY = "DIRECTION_DEPTH_HOLD_KEY"; export const WHICH_EYES_HOLD = "WHICH_EYES_HOLD"; export const DIRECTION_HOLD = "DIRECTION_HOLD"; export const DIRECTION_LENGTH_KEY = "DIRECTION_LENGTH_KEY"; +export const HOLD_BLINK_EYE_TO_TRACK = "HOLD_BLINK_EYE_TO_TRACK"; From f13556250736196857ee6db20031d20c01ce146b Mon Sep 17 00:00:00 2001 From: Gavin Henderson Date: Fri, 11 Mar 2022 09:39:55 +0000 Subject: [PATCH 2/7] Add eye selection for basic mode --- .../hooks/use-blink/use-basic-blink.js | 69 +++++++++++++++++-- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/src/react-app/hooks/use-blink/use-basic-blink.js b/src/react-app/hooks/use-blink/use-basic-blink.js index cfefb3f8..4a1c1585 100644 --- a/src/react-app/hooks/use-blink/use-basic-blink.js +++ b/src/react-app/hooks/use-blink/use-basic-blink.js @@ -1,5 +1,8 @@ import { useCallback } from "react"; -import { CHANGE_THRESHOLD_BASIC_KEY } from "../../lib/store-consts"; +import { + CHANGE_THRESHOLD_BASIC_KEY, + BASIC_BASIC_EYE_TO_TRACK, +} from "../../lib/store-consts"; import { useStoreValue } from "../use-store"; const euclaideanDistance = (point, point1) => { @@ -16,9 +19,16 @@ export const useBasicBlink = (onBlink, setDisplayOnSlider) => { update: updateBlinkThreshold, } = useStoreValue(CHANGE_THRESHOLD_BASIC_KEY, 5); + const { + loading: loadingEyeToTrack, + value: eyeToTrack, + update: updateEyeToTrack, + } = useStoreValue(BASIC_BASIC_EYE_TO_TRACK, "both"); + const noop = useCallback( (results, currentTimestamp, distanceHistory) => { if (loadingBlinkThreshold) return; + if (loadingEyeToTrack) return; if (results.multiFaceLandmarks) { for (const landmarks of results.multiFaceLandmarks) { @@ -45,7 +55,17 @@ export const useBasicBlink = (onBlink, setDisplayOnSlider) => { const reRatio = rhDistance / rvDistance; const leRatio = lhDistance / lvDistance; - const ratio = (reRatio + leRatio) / 2; + let ratio = null; + + if (eyeToTrack === "both") { + ratio = (reRatio + leRatio) / 2; + } else if (eyeToTrack === "right") { + ratio = reRatio; + } else if (eyeToTrack === "left") { + ratio = leRatio; + } else { + throw new Error(`Invalid eyeToTrack: ${eyeToTrack}`); + } const currentFrame = { time: currentTimestamp, @@ -66,14 +86,26 @@ export const useBasicBlink = (onBlink, setDisplayOnSlider) => { } } }, - [blinkThreshold, loadingBlinkThreshold, onBlink, setDisplayOnSlider] + [ + blinkThreshold, + loadingBlinkThreshold, + onBlink, + setDisplayOnSlider, + loadingEyeToTrack, + eyeToTrack, + ] ); + const eyesToTrackHighlights = { + both: { leftEye: true, rightEye: true }, + left: { leftEye: true, rightEye: false }, + right: { leftEye: false, rightEye: true }, + }; + return { detectBlink: noop, highlights: { - leftEye: true, - rightEye: true, + ...eyesToTrackHighlights[eyeToTrack], face: true, leftPupil: false, rightPupil: false, @@ -94,6 +126,33 @@ export const useBasicBlink = (onBlink, setDisplayOnSlider) => { updateBlinkThreshold(newValue / 10); }, }, + { + loadingOption: loadingEyeToTrack, + type: "radio", + options: [ + { + value: "both", + name: "Both", + tooltip: "Detects both eyes", + }, + { + value: "left", + name: "Left", + tooltip: "Detects only the left eye", + }, + { + value: "right", + name: "Right", + tooltip: "Detects only the right eye", + }, + ], + defaultValue: loadingEyeToTrack ? "both" : eyeToTrack, + label: "Eyes to track", + tooltip: "Decide which eyes you want to track", + onChange: (newValue) => { + updateEyeToTrack(newValue); + }, + }, ], }; }; From 73896bfd5687cd872b783068b11edbe0a0ab9aba Mon Sep 17 00:00:00 2001 From: Gavin Henderson Date: Fri, 11 Mar 2022 09:40:08 +0000 Subject: [PATCH 3/7] Add eye selection for speed mode --- .../hooks/use-blink/use-speed-blink.js | 69 +++++++++++++++++-- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/src/react-app/hooks/use-blink/use-speed-blink.js b/src/react-app/hooks/use-blink/use-speed-blink.js index 7a5c5f5b..680724b8 100644 --- a/src/react-app/hooks/use-blink/use-speed-blink.js +++ b/src/react-app/hooks/use-blink/use-speed-blink.js @@ -1,5 +1,8 @@ import { useCallback } from "react"; -import { CHANGE_THRESHOLD_SPEED_KEY } from "../../lib/store-consts"; +import { + CHANGE_THRESHOLD_SPEED_KEY, + BASIC_SPEED_EYE_TO_TRACK, +} from "../../lib/store-consts"; import { useStoreValue } from "../use-store"; const TIME_BETWEEN = 100; @@ -18,9 +21,16 @@ export const useSpeedBlink = (onBlink, setDisplayOnSlider) => { update: updateBlinkThreshold, } = useStoreValue(CHANGE_THRESHOLD_SPEED_KEY, 5); + const { + loading: loadingEyeToTrack, + value: eyeToTrack, + update: updateEyeToTrack, + } = useStoreValue(BASIC_SPEED_EYE_TO_TRACK, "both"); + const noop = useCallback( (results, currentTimestamp, distanceHistory) => { if (loadingBlinkThreshold) return; + if (loadingEyeToTrack) return; if (results.multiFaceLandmarks) { for (const landmarks of results.multiFaceLandmarks) { @@ -47,7 +57,17 @@ export const useSpeedBlink = (onBlink, setDisplayOnSlider) => { const reRatio = rhDistance / rvDistance; const leRatio = lhDistance / lvDistance; - const ratio = (reRatio + leRatio) / 2; + let ratio = null; + + if (eyeToTrack === "both") { + ratio = (reRatio + leRatio) / 2; + } else if (eyeToTrack === "right") { + ratio = reRatio; + } else if (eyeToTrack === "left") { + ratio = leRatio; + } else { + throw new Error(`Invalid eyeToTrack: ${eyeToTrack}`); + } const currentFrame = { time: currentTimestamp, @@ -85,14 +105,26 @@ export const useSpeedBlink = (onBlink, setDisplayOnSlider) => { } } }, - [blinkThreshold, loadingBlinkThreshold, onBlink, setDisplayOnSlider] + [ + blinkThreshold, + loadingBlinkThreshold, + onBlink, + setDisplayOnSlider, + loadingEyeToTrack, + eyeToTrack, + ] ); + const eyesToTrackHighlights = { + both: { leftEye: true, rightEye: true }, + left: { leftEye: true, rightEye: false }, + right: { leftEye: false, rightEye: true }, + }; + return { detectBlink: noop, highlights: { - leftEye: true, - rightEye: true, + ...eyesToTrackHighlights[eyeToTrack], face: true, leftPupil: false, rightPupil: false, @@ -113,6 +145,33 @@ export const useSpeedBlink = (onBlink, setDisplayOnSlider) => { updateBlinkThreshold(newValue / 10); }, }, + { + loadingOption: loadingEyeToTrack, + type: "radio", + options: [ + { + value: "both", + name: "Both", + tooltip: "Detects both eyes", + }, + { + value: "left", + name: "Left", + tooltip: "Detects only the left eye", + }, + { + value: "right", + name: "Right", + tooltip: "Detects only the right eye", + }, + ], + defaultValue: loadingEyeToTrack ? "both" : eyeToTrack, + label: "Eyes to track", + tooltip: "Decide which eyes you want to track", + onChange: (newValue) => { + updateEyeToTrack(newValue); + }, + }, ], }; }; From b9dbcc0d595b3eebf0515aff0c45951eab95069c Mon Sep 17 00:00:00 2001 From: Gavin Henderson Date: Fri, 11 Mar 2022 09:40:13 +0000 Subject: [PATCH 4/7] Add consts --- src/react-app/lib/store-consts.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/react-app/lib/store-consts.js b/src/react-app/lib/store-consts.js index 9d50bc4c..2194d1fa 100644 --- a/src/react-app/lib/store-consts.js +++ b/src/react-app/lib/store-consts.js @@ -19,3 +19,5 @@ export const WHICH_EYES_HOLD = "WHICH_EYES_HOLD"; export const DIRECTION_HOLD = "DIRECTION_HOLD"; export const DIRECTION_LENGTH_KEY = "DIRECTION_LENGTH_KEY"; export const HOLD_BLINK_EYE_TO_TRACK = "HOLD_BLINK_EYE_TO_TRACK"; +export const BASIC_BLINK_EYE_TO_TRACK = "BASIC_BLINK_EYE_TO_TRACK"; +export const SPEED_BLINK_EYE_TO_TRACK = "SPEED_BLINK_EYE_TO_TRACK"; From 9b7c32d3b2f4de11bc5523935ccba7af89262559 Mon Sep 17 00:00:00 2001 From: Gavin Henderson Date: Fri, 11 Mar 2022 09:41:53 +0000 Subject: [PATCH 5/7] Fix const usage --- src/react-app/hooks/use-blink/use-basic-blink.js | 4 ++-- src/react-app/hooks/use-blink/use-speed-blink.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/react-app/hooks/use-blink/use-basic-blink.js b/src/react-app/hooks/use-blink/use-basic-blink.js index 4a1c1585..cb539082 100644 --- a/src/react-app/hooks/use-blink/use-basic-blink.js +++ b/src/react-app/hooks/use-blink/use-basic-blink.js @@ -1,7 +1,7 @@ import { useCallback } from "react"; import { CHANGE_THRESHOLD_BASIC_KEY, - BASIC_BASIC_EYE_TO_TRACK, + BASIC_BLINK_EYE_TO_TRACK, } from "../../lib/store-consts"; import { useStoreValue } from "../use-store"; @@ -23,7 +23,7 @@ export const useBasicBlink = (onBlink, setDisplayOnSlider) => { loading: loadingEyeToTrack, value: eyeToTrack, update: updateEyeToTrack, - } = useStoreValue(BASIC_BASIC_EYE_TO_TRACK, "both"); + } = useStoreValue(BASIC_BLINK_EYE_TO_TRACK, "both"); const noop = useCallback( (results, currentTimestamp, distanceHistory) => { diff --git a/src/react-app/hooks/use-blink/use-speed-blink.js b/src/react-app/hooks/use-blink/use-speed-blink.js index 680724b8..bfcdf943 100644 --- a/src/react-app/hooks/use-blink/use-speed-blink.js +++ b/src/react-app/hooks/use-blink/use-speed-blink.js @@ -1,7 +1,7 @@ import { useCallback } from "react"; import { CHANGE_THRESHOLD_SPEED_KEY, - BASIC_SPEED_EYE_TO_TRACK, + SPEED_BLINK_EYE_TO_TRACK, } from "../../lib/store-consts"; import { useStoreValue } from "../use-store"; @@ -25,7 +25,7 @@ export const useSpeedBlink = (onBlink, setDisplayOnSlider) => { loading: loadingEyeToTrack, value: eyeToTrack, update: updateEyeToTrack, - } = useStoreValue(BASIC_SPEED_EYE_TO_TRACK, "both"); + } = useStoreValue(SPEED_BLINK_EYE_TO_TRACK, "both"); const noop = useCallback( (results, currentTimestamp, distanceHistory) => { From 1475a7c780252e7bfa176f0f4bca8191c4da40f6 Mon Sep 17 00:00:00 2001 From: Gavin Henderson Date: Fri, 11 Mar 2022 09:53:44 +0000 Subject: [PATCH 6/7] Add eye highlights for direction mode --- .../hooks/use-blink/use-direction-basic.js | 26 ++++++++++++++++--- .../hooks/use-blink/use-direction-hold.js | 26 ++++++++++++++++--- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/src/react-app/hooks/use-blink/use-direction-basic.js b/src/react-app/hooks/use-blink/use-direction-basic.js index df380b1f..36890df5 100644 --- a/src/react-app/hooks/use-blink/use-direction-basic.js +++ b/src/react-app/hooks/use-blink/use-direction-basic.js @@ -156,16 +156,34 @@ export const useDirectionBasic = (onBlink, setDisplayOnSlider) => { ] ); + const eyesToTrackHighlights = { + both: { + leftPupil: true, + rightPupil: true, + leftEyeEdgePoints: true, + rightEyeEdgePoints: true, + }, + left: { + leftPupil: true, + rightPupil: false, + leftEyeEdgePoints: true, + rightEyeEdgePoints: false, + }, + right: { + leftPupil: false, + rightPupil: true, + leftEyeEdgePoints: false, + rightEyeEdgePoints: true, + }, + }; + return { detectBlink: noop, highlights: { leftEye: false, rightEye: false, face: true, - leftPupil: true, - rightPupil: true, - leftEyeEdgePoints: true, - rightEyeEdgePoints: true, + ...eyesToTrackHighlights[whichEyes], }, options: [ { diff --git a/src/react-app/hooks/use-blink/use-direction-hold.js b/src/react-app/hooks/use-blink/use-direction-hold.js index 6129ca6d..ec0b93e7 100644 --- a/src/react-app/hooks/use-blink/use-direction-hold.js +++ b/src/react-app/hooks/use-blink/use-direction-hold.js @@ -178,16 +178,34 @@ export const useDirectionHold = (onBlink, setDisplayOnSlider) => { ] ); + const eyesToTrackHighlights = { + both: { + leftPupil: true, + rightPupil: true, + leftEyeEdgePoints: true, + rightEyeEdgePoints: true, + }, + left: { + leftPupil: true, + rightPupil: false, + leftEyeEdgePoints: true, + rightEyeEdgePoints: false, + }, + right: { + leftPupil: false, + rightPupil: true, + leftEyeEdgePoints: false, + rightEyeEdgePoints: true, + }, + }; + return { detectBlink: noop, highlights: { leftEye: false, rightEye: false, face: true, - leftPupil: true, - rightPupil: true, - leftEyeEdgePoints: true, - rightEyeEdgePoints: true, + ...eyesToTrackHighlights[whichEyes], }, options: [ { From 81aadb4f335984b5b0432f9febf80382abe0f6d6 Mon Sep 17 00:00:00 2001 From: Gavin Henderson Date: Fri, 11 Mar 2022 10:52:38 +0000 Subject: [PATCH 7/7] FIX UI size --- src/react-app/hooks/use-blink/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/react-app/hooks/use-blink/index.js b/src/react-app/hooks/use-blink/index.js index 82ecd037..62df20bf 100644 --- a/src/react-app/hooks/use-blink/index.js +++ b/src/react-app/hooks/use-blink/index.js @@ -40,6 +40,8 @@ export const BLINK_MODES = [ }, ]; +const BIGGER_VIEWS = ["DIRECTION_HOLD", "HOLD"]; + export const useBlink = (...params) => { const { loading: blinkModeLoading, value: blinkMode } = useStoreValue( BLINK_MODE, @@ -48,7 +50,7 @@ export const useBlink = (...params) => { useResizer({ width: 900, - height: !blinkModeLoading && blinkMode === "DIRECTION_HOLD" ? 523 : 440, + height: !blinkModeLoading && BIGGER_VIEWS.includes(blinkMode) ? 523 : 440, }); const [displayOnSlider, setDisplayOnSlider] = useState({