From cbf72d500e6aa1585492dfd3b879bd2a3b9a84da Mon Sep 17 00:00:00 2001 From: Brian Vaughn Date: Tue, 5 Mar 2024 18:41:43 -0500 Subject: [PATCH] Replace declarative Video component with imperative controller --- packages/e2e-tests/helpers/screenshot.ts | 16 +- packages/e2e-tests/tests/repaint-05.test.ts | 4 +- packages/protocol/PaintsCache.ts | 15 +- packages/protocol/RepaintGraphicsCache.ts | 16 ++ packages/protocol/StreamingScreenShotCache.ts | 109 ------------ src/ui/actions/timeline.ts | 22 ++- .../Comments/VideoComments/VideoComment.tsx | 6 +- src/ui/components/NodePickerContext.tsx | 2 +- src/ui/components/Timeline/ProgressBars.tsx | 14 +- .../components/Video/MutableGraphicsState.tsx | 74 --------- .../PreviewNodeHighlighter.tsx | 4 +- src/ui/components/Video/RecordedCursor.tsx | 112 +++++++------ src/ui/components/Video/Video.module.css | 5 + src/ui/components/Video/Video.tsx | 153 +++++++++-------- .../Video/VideoContextMenu.module.css | 4 - .../Video/imperative/MutableGraphicsState.tsx | 116 +++++++++++++ .../Video/imperative/createState.ts | 51 ++++++ .../getGraphicsTimeAndExecutionPoint.ts} | 34 ++-- .../{ => imperative}/getMouseEventPosition.ts | 5 +- .../repositionGraphicsOverlayElements.ts | 17 ++ .../Video/imperative/runVideoPlayback.ts | 80 +++++++++ .../imperative/subscribeToMutableSources.ts | 115 +++++++++++++ .../Video/imperative/updateGraphics.ts | 156 ++++++++++++++++++ .../Video/useDisplayedScreenShot.ts | 41 ----- .../Video/useImperativeVideoPlayback.ts | 110 ------------ .../Video/useUpdateGraphicsContext.ts | 53 ------ .../components/Video/useVideoContextMenu.tsx | 8 +- src/ui/reducers/timeline.ts | 6 - src/ui/setup/store.ts | 1 - src/ui/state/timeline.ts | 8 +- 30 files changed, 765 insertions(+), 592 deletions(-) create mode 100644 packages/protocol/RepaintGraphicsCache.ts delete mode 100644 packages/protocol/StreamingScreenShotCache.ts delete mode 100644 src/ui/components/Video/MutableGraphicsState.tsx delete mode 100644 src/ui/components/Video/VideoContextMenu.module.css create mode 100644 src/ui/components/Video/imperative/MutableGraphicsState.tsx create mode 100644 src/ui/components/Video/imperative/createState.ts rename src/ui/components/Video/{useSmartTimeAndExecutionPoint.ts => imperative/getGraphicsTimeAndExecutionPoint.ts} (70%) rename src/ui/components/Video/{ => imperative}/getMouseEventPosition.ts (70%) create mode 100644 src/ui/components/Video/imperative/repositionGraphicsOverlayElements.ts create mode 100644 src/ui/components/Video/imperative/runVideoPlayback.ts create mode 100644 src/ui/components/Video/imperative/subscribeToMutableSources.ts create mode 100644 src/ui/components/Video/imperative/updateGraphics.ts delete mode 100644 src/ui/components/Video/useDisplayedScreenShot.ts delete mode 100644 src/ui/components/Video/useImperativeVideoPlayback.ts delete mode 100644 src/ui/components/Video/useUpdateGraphicsContext.ts diff --git a/packages/e2e-tests/helpers/screenshot.ts b/packages/e2e-tests/helpers/screenshot.ts index ccda1855942..c516960cb71 100644 --- a/packages/e2e-tests/helpers/screenshot.ts +++ b/packages/e2e-tests/helpers/screenshot.ts @@ -96,23 +96,21 @@ export async function getGraphicsTime(page: Page): Promise { export async function getGraphicsPixelColor(page: Page, x: number, y: number) { return await page.evaluate( ([x, y]) => { - const element = document.querySelector("#graphics"); - if (element == null) { + const element = document.querySelector("#graphics") as HTMLImageElement; + if (!element?.getAttribute("src")) { return null; } - const imageElement = element as HTMLImageElement; - const canvas = document.createElement("canvas"); - canvas.width = imageElement.width; - canvas.height = imageElement.height; + canvas.width = element.width; + canvas.height = element.height; const context = canvas.getContext("2d"); if (context == null) { return null; } - context.drawImage(imageElement, 0, 0); + context.drawImage(element, 0, 0); const { data } = context.getImageData(x, y, 1, 1); @@ -127,7 +125,9 @@ export async function getGraphicsPixelColor(page: Page, x: number, y: number) { } export async function waitForGraphicsToLoad(page: Page) { + await debugPrint(page, `Waiting for graphics to load...`, "waitForGraphicsToLoad"); + await waitFor(async () => { - await expect(await getGraphicsStatus(page)).not.toBe("fetching-cached-paint"); + await expect(await getGraphicsStatus(page)).toBe("loaded"); }); } diff --git a/packages/e2e-tests/tests/repaint-05.test.ts b/packages/e2e-tests/tests/repaint-05.test.ts index 279bbadb34f..f24b88b25bd 100644 --- a/packages/e2e-tests/tests/repaint-05.test.ts +++ b/packages/e2e-tests/tests/repaint-05.test.ts @@ -5,8 +5,8 @@ import test, { Page, expect } from "../testFixtureCloneRecording"; test.use({ exampleKey: "paint_at_intervals.html" }); -async function seekToTimePercentAndWaitForPaint(page: Page, time: number) { - await seekToTimePercent(page, time); +async function seekToTimePercentAndWaitForPaint(page: Page, percent: number) { + await seekToTimePercent(page, percent); await waitForGraphicsToLoad(page); } diff --git a/packages/protocol/PaintsCache.ts b/packages/protocol/PaintsCache.ts index 5d8b0233e95..fd0f4dc5e19 100644 --- a/packages/protocol/PaintsCache.ts +++ b/packages/protocol/PaintsCache.ts @@ -1,7 +1,7 @@ import { createSingleEntryCache } from "suspense"; -import { StreamingScreenShotCache } from "protocol/StreamingScreenShotCache"; import { recordingTargetCache } from "replay-next/src/suspense/BuildIdCache"; +import { screenshotCache } from "replay-next/src/suspense/ScreenshotCache"; import { find, findIndexGTE, findIndexLTE } from "replay-next/src/utils/array"; import { getDimensions } from "replay-next/src/utils/image"; import { replayClient } from "shared/client/ReplayClientContext"; @@ -41,17 +41,18 @@ export async function findFirstMeaningfulPaint() { const paint = paints[index]; try { - const { value } = await StreamingScreenShotCache.readAsync( + const screenShot = await screenshotCache.readAsync( replayClient, - paint.time, - paint.point + paint.point, + paint.paintHash ); - if (value && value.hash) { - const { width, height } = await getDimensions(value.hash, value.mimeType); + + if (screenShot && screenShot.hash) { + const { width, height } = await getDimensions(screenShot.hash, screenShot.mimeType); // Estimate how "interesting" the screen is based on what % of the image is different pixels. // This is done to avoid showing something like a blank page or a mostly empty loading screen. - if (value.data.length > (width * height) / 40) { + if (screenShot.data.length > (width * height) / 40) { return paint; } } diff --git a/packages/protocol/RepaintGraphicsCache.ts b/packages/protocol/RepaintGraphicsCache.ts new file mode 100644 index 00000000000..57e9d6ec681 --- /dev/null +++ b/packages/protocol/RepaintGraphicsCache.ts @@ -0,0 +1,16 @@ +import { PauseId, repaintGraphicsResult } from "@replayio/protocol"; +import { Cache, createCache } from "suspense"; + +import { ReplayClientInterface } from "shared/client/types"; + +export const RepaintGraphicsCache: Cache< + [replayClient: ReplayClientInterface, pauseId: PauseId], + repaintGraphicsResult | null +> = createCache({ + config: { immutable: true }, + debugLabel: "RepaintGraphicsCache", + getKey: ([replayClient, pauseId]) => pauseId, + load: async ([replayClient, pauseId]) => { + return replayClient.repaintGraphics(pauseId); + }, +}); diff --git a/packages/protocol/StreamingScreenShotCache.ts b/packages/protocol/StreamingScreenShotCache.ts deleted file mode 100644 index 7773383367d..00000000000 --- a/packages/protocol/StreamingScreenShotCache.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { ExecutionPoint, PauseId, ScreenShot, repaintGraphicsResult } from "@replayio/protocol"; -import { Cache, createCache, createStreamingCache } from "suspense"; - -import { PaintsCache, findMostRecentPaint } from "protocol/PaintsCache"; -import { pauseIdCache } from "replay-next/src/suspense/PauseCache"; -import { screenshotCache } from "replay-next/src/suspense/ScreenshotCache"; -import { ReplayClientInterface } from "shared/client/types"; - -export const RepaintGraphicsCache: Cache< - [replayClient: ReplayClientInterface, pauseId: PauseId], - repaintGraphicsResult | null -> = createCache({ - config: { immutable: true }, - debugLabel: "RepaintGraphicsCache", - getKey: ([replayClient, pauseId]) => pauseId, - load: async ([replayClient, pauseId]) => { - return replayClient.repaintGraphics(pauseId); - }, -}); - -export type StreamingScreenShotCacheStatus = - | "before-first-paint" - | "complete" - | "fetching-cached-paint" - | "fetching-repaint" - | "loading-failed"; - -export const StreamingScreenShotCache = createStreamingCache< - [replayClient: ReplayClientInterface, time: number, executionPoint: ExecutionPoint | null], - ScreenShot | undefined, - StreamingScreenShotCacheStatus ->({ - debugLabel: "StreamingScreenShotCache", - getKey: (replayClient, time, executionPoint) => `${time}:${executionPoint}`, - load: async ({ update, reject, resolve }, replayClient, time, executionPoint) => { - let screenShot: ScreenShot | undefined = undefined; - - update(screenShot, 0, "fetching-cached-paint"); - - let didLoadImpreciseScreenShot = false; - - // Wait for paints to (finish) loading - await PaintsCache.readAsync(); - - const paintPoint = findMostRecentPaint(time); - if (!paintPoint || !paintPoint.paintHash) { - // Don't try to paint (or repaint) if the current time is before the first cached paint - update(undefined, 1, "before-first-paint"); - resolve(); - return; - } - - if (paintPoint && paintPoint.paintHash) { - try { - screenShot = await screenshotCache.readAsync( - replayClient, - paintPoint.point, - paintPoint.paintHash - ); - - didLoadImpreciseScreenShot = true; - - if (paintPoint.point == executionPoint) { - update(screenShot, 1, "complete"); - resolve(); - - return; - } else { - update(screenShot, 0.5, "fetching-repaint"); - } - } catch (error) { - update(undefined, 1, "loading-failed"); - reject(error); - - return; - } - } - - if (executionPoint && executionPoint !== "0") { - const pauseId = await pauseIdCache.readAsync(replayClient, executionPoint, time); - - try { - const result = await RepaintGraphicsCache.readAsync(replayClient, pauseId); - - if (result?.screenShot) { - screenShot = result.screenShot; - - update(screenShot, 1, "complete"); - resolve(); - - return; - } - } catch (error) { - console.error(error); - } - } - - // TRICKY: - // - // There are two scenarios where we need to update the graphics: - // (1) The current execution point has changed (e.g. the timeline has changed) - // (2) The user is hovering over something (like a test step) - // - // In the second case, only a time value will be specified. - // So long as we can load a cached screenshot this cache should be considered complete. - update(screenShot, 1, screenShot != null ? "complete" : "loading-failed"); - resolve(); - }, -}); diff --git a/src/ui/actions/timeline.ts b/src/ui/actions/timeline.ts index 173dc96c77a..9fde16579c5 100644 --- a/src/ui/actions/timeline.ts +++ b/src/ui/actions/timeline.ts @@ -262,6 +262,11 @@ export function seek({ location?: Location; }): UIThunkAction> { return async (dispatch, getState, { replayClient }) => { + const isPlaying = getPlayback(getState()) !== null; + if (isPlaying) { + dispatch(stopPlayback()); + } + const seekLock = new Object(); dispatch(pauseRequestedAt({ seekLock, executionPoint, time, location })); dispatch(setTimelineState({ currentTime: time, playback: null })); @@ -378,7 +383,7 @@ export function togglePlayback(): UIThunkAction { } export function startPlayback( - { beginTime: optBeginTime, endTime: optEndTime }: PlaybackOptions = { + { beginPoint = null, beginTime, endPoint = null, endTime }: PlaybackOptions = { beginTime: null, endTime: null, } @@ -387,21 +392,20 @@ export function startPlayback( const state = getState(); const currentTime = getCurrentTime(state); - const endTime = - optEndTime || + endTime = + endTime || (getPlaybackFocusWindow(state) && replayClient.getCurrentFocusWindow()?.end.time) || getZoomRegion(state).endTime; - const beginDate = Date.now(); - const beginTime = - optBeginTime || + beginTime = + beginTime || (currentTime >= endTime ? (getPlaybackFocusWindow(state) && replayClient.getCurrentFocusWindow()?.begin.time) || 0 : currentTime); dispatch( setTimelineState({ - playback: { beginTime, beginDate, endTime, time: beginTime }, + playback: { beginPoint, beginTime, endPoint, endTime, time: beginTime }, currentTime: beginTime, }) ); @@ -412,6 +416,8 @@ export function startPlayback( export function stopPlayback(updateTime: boolean = true): UIThunkAction { return async (dispatch, getState) => { + dispatch(setTimelineState({ playback: null })); + if (updateTime) { const playback = getPlayback(getState()); @@ -419,8 +425,6 @@ export function stopPlayback(updateTime: boolean = true): UIThunkAction { dispatch(seek({ time: playback.time })); } } - - dispatch(setTimelineState({ playback: null })); }; } diff --git a/src/ui/components/Comments/VideoComments/VideoComment.tsx b/src/ui/components/Comments/VideoComments/VideoComment.tsx index 0f5e343d782..967d10d387f 100644 --- a/src/ui/components/Comments/VideoComments/VideoComment.tsx +++ b/src/ui/components/Comments/VideoComments/VideoComment.tsx @@ -4,7 +4,7 @@ import { useLayoutEffect, useRef } from "react"; import { isVisualCommentTypeData } from "replay-next/components/sources/utils/comments"; import { Comment } from "shared/graphql/types"; import { setHoveredCommentId, setSelectedCommentId } from "ui/actions/app"; -import { subscribe } from "ui/components/Video/MutableGraphicsState"; +import { state } from "ui/components/Video/imperative/MutableGraphicsState"; import { useAppDispatch } from "ui/setup/hooks"; import styles from "./VideoComment.module.css"; @@ -26,8 +26,8 @@ export default function VideoComment({ const element = elementRef.current; if (element) { // Imperatively position and scale these graphics to avoid "render lag" when resizes occur - return subscribe(state => { - const { height, width } = state; + return state.listen(state => { + const { height, width } = state.graphicsRect; const { scaledX, scaledY } = typeData; element.style.left = `${scaledX * width}px`; diff --git a/src/ui/components/NodePickerContext.tsx b/src/ui/components/NodePickerContext.tsx index b3dcb02adb4..8d5eb8e982a 100644 --- a/src/ui/components/NodePickerContext.tsx +++ b/src/ui/components/NodePickerContext.tsx @@ -13,7 +13,7 @@ import { import { highlightNode, unhighlightNode } from "devtools/client/inspector/markup/actions/markup"; import { useMostRecentLoadedPause } from "replay-next/src/hooks/useMostRecentLoadedPause"; import { ReplayClientContext } from "shared/client/ReplayClientContext"; -import { getMouseEventPosition } from "ui/components/Video/getMouseEventPosition"; +import { getMouseEventPosition } from "ui/components/Video/imperative/getMouseEventPosition"; import { useAppDispatch } from "ui/setup/hooks"; import { boundingRectsCache, getMouseTarget } from "ui/suspense/nodeCaches"; diff --git a/src/ui/components/Timeline/ProgressBars.tsx b/src/ui/components/Timeline/ProgressBars.tsx index 3613b09b7ca..c02f1aeba35 100644 --- a/src/ui/components/Timeline/ProgressBars.tsx +++ b/src/ui/components/Timeline/ProgressBars.tsx @@ -1,35 +1,27 @@ import clamp from "lodash/clamp"; -import React from "react"; -import { - getCurrentTime, - getHoverTime, - getPlaybackPrecachedTime, - getZoomRegion, -} from "ui/reducers/timeline"; +import { getCurrentTime, getHoverTime, getZoomRegion } from "ui/reducers/timeline"; import { useAppSelector } from "ui/setup/hooks"; import { getVisiblePosition } from "ui/utils/timeline"; export default function ProgressBars() { const currentTime = useAppSelector(getCurrentTime); const hoverTime = useAppSelector(getHoverTime); - const precachedTime = useAppSelector(getPlaybackPrecachedTime); const zoomRegion = useAppSelector(getZoomRegion); const percent = getVisiblePosition({ time: currentTime, zoom: zoomRegion }) * 100; const hoverPercent = getVisiblePosition({ time: hoverTime, zoom: zoomRegion }) * 100; - const precachedPercent = getVisiblePosition({ time: precachedTime, zoom: zoomRegion }) * 100; return ( <>
void; -type Unsubscribe = () => void; - -export const mutableState: GraphicsState = { - height: 0, - left: 0, - localScale: 1, - recordingScale: 1, - top: 0, - width: 0, -}; - -const subscribers: Set = new Set(); - -export function subscribe(callback: Callback): Unsubscribe { - subscribers.add(callback); - - callback(mutableState); - - return () => { - subscribers.delete(callback); - }; -} - -export function update({ element, scale }: { element: HTMLImageElement; scale?: number }) { - const { clientHeight, clientWidth, naturalWidth } = element; - - const { left, top } = element.getBoundingClientRect(); - const localScale = isNaN(clientWidth / naturalWidth) - ? mutableState.localScale - : clientWidth / naturalWidth; - const recordingScale = scale ?? mutableState.recordingScale; - - const state: GraphicsState = { - height: clientHeight, - left, - localScale, - recordingScale, - top, - width: clientWidth, - }; - - if (!shallowEqual(mutableState, state)) { - Object.assign(mutableState, state); - - subscribers.forEach(callback => { - callback(mutableState); - }); - } -} diff --git a/src/ui/components/Video/NodeHighlighter/PreviewNodeHighlighter.tsx b/src/ui/components/Video/NodeHighlighter/PreviewNodeHighlighter.tsx index d853c0443a2..596d9798b62 100644 --- a/src/ui/components/Video/NodeHighlighter/PreviewNodeHighlighter.tsx +++ b/src/ui/components/Video/NodeHighlighter/PreviewNodeHighlighter.tsx @@ -3,7 +3,7 @@ import { ReactNode, useLayoutEffect, useRef } from "react"; import { getNodeBoxModelById } from "devtools/client/inspector/markup/reducers/markup"; import { assert } from "protocol/utils"; -import { subscribe } from "ui/components/Video/MutableGraphicsState"; +import { state } from "ui/components/Video/imperative/MutableGraphicsState"; import { useAppSelector } from "ui/setup/hooks"; // Note that the order of items in this array is important because it is used @@ -112,7 +112,7 @@ export function PreviewNodeHighlighter({ nodeId }: { nodeId: string }) { useLayoutEffect(() => { const element = elementRef.current; if (element) { - return subscribe(state => { + return state.listen(state => { const { localScale, recordingScale } = state; const scale = localScale * recordingScale; diff --git a/src/ui/components/Video/RecordedCursor.tsx b/src/ui/components/Video/RecordedCursor.tsx index 4562b60fc0e..3555273fed8 100644 --- a/src/ui/components/Video/RecordedCursor.tsx +++ b/src/ui/components/Video/RecordedCursor.tsx @@ -1,82 +1,84 @@ import assert from "assert"; -import { useContext, useLayoutEffect, useRef } from "react"; +import { useLayoutEffect, useRef } from "react"; import { findMostRecentClickEvent, findMostRecentMouseEvent } from "protocol/RecordedEventsCache"; -import { subscribe } from "ui/components/Video/MutableGraphicsState"; +import { state } from "ui/components/Video/imperative/MutableGraphicsState"; const CLICK_TIMING_THRESHOLD_MS = 200; -export function RecordedCursor({ time }: { time: number }) { +export function RecordedCursor() { const elementRef = useRef(null); - const mouseEvent = findMostRecentMouseEvent(time); - const shouldDrawCursor = mouseEvent != null; - - const clickEvent = findMostRecentClickEvent(time); - const shouldDrawClick = clickEvent && clickEvent.time + CLICK_TIMING_THRESHOLD_MS >= time; - useLayoutEffect(() => { const element = elementRef.current; assert(element); - if (mouseEvent) { - // Imperatively position and scale these graphics to avoid "render lag" when resizes occur - return subscribe(state => { - const { height, localScale, recordingScale, width } = state; + // Imperatively position and scale these graphics to avoid "render lag" when resizes occur + const unsubscribe = state.listen( + ({ currentTime, graphicsRect, localScale, recordingScale }) => { + const { height, width } = graphicsRect; + + const mouseEvent = findMostRecentMouseEvent(currentTime); + const clickEvent = findMostRecentClickEvent(currentTime); + const shouldDrawClick = + clickEvent && clickEvent.time + CLICK_TIMING_THRESHOLD_MS >= currentTime; + + if (mouseEvent) { + const originalHeight = height / localScale; + const originalWidth = width / localScale; - const originalHeight = height / localScale; - const originalWidth = width / localScale; + const mouseX = (mouseEvent.clientX / originalWidth) * width; + const mouseY = (mouseEvent.clientY / originalHeight) * height; - const mouseX = (mouseEvent.clientX / originalWidth) * width; - const mouseY = (mouseEvent.clientY / originalHeight) * height; + const cursorScale = Math.min(1, Math.max(0.25, localScale * recordingScale)); - const cursorScale = Math.min(1, Math.max(0.25, localScale * recordingScale)); + element.style.display = "block"; + element.style.left = `${mouseX}px`; + element.style.top = `${mouseY}px`; + element.style.transform = `scale(${cursorScale})`; + element.style.setProperty("--click-display", shouldDrawClick ? "block" : "none"); + } else { + element.style.display = "none"; + element.style.setProperty("--click-display", "none"); + } + } + ); - element.style.left = `${mouseX}px`; - element.style.top = `${mouseY}px`; - element.style.transform = `scale(${cursorScale})`; - }); - } - }, [mouseEvent]); + return () => { + unsubscribe(); + }; + }, []); return (
- {shouldDrawClick && ( -
- )} - {shouldDrawCursor && ( - - - - )} +
+ + +
); } diff --git a/src/ui/components/Video/Video.module.css b/src/ui/components/Video/Video.module.css index 2bb5b63d3d6..9588cd8f5b8 100644 --- a/src/ui/components/Video/Video.module.css +++ b/src/ui/components/Video/Video.module.css @@ -52,3 +52,8 @@ width: 1rem; height: 1rem; } + +.ContextMenuIcon { + width: 1.5rem; + height: 1rem; +} diff --git a/src/ui/components/Video/Video.tsx b/src/ui/components/Video/Video.tsx index b55e39f8e76..6ad0f28efd5 100644 --- a/src/ui/components/Video/Video.tsx +++ b/src/ui/components/Video/Video.tsx @@ -1,7 +1,5 @@ -import { MouseEvent, useContext, useLayoutEffect } from "react"; -import { useStreamingValue } from "suspense"; +import { MouseEvent, useContext, useLayoutEffect, useState } from "react"; -import { StreamingScreenShotCache } from "protocol/StreamingScreenShotCache"; import Icon from "replay-next/components/Icon"; import { LoadingProgressBar } from "replay-next/components/LoadingProgressBar"; import { ReplayClientContext } from "shared/client/ReplayClientContext"; @@ -10,67 +8,97 @@ import CommentsOverlay from "ui/components/Comments/VideoComments"; import { NodePickerContext } from "ui/components/NodePickerContext"; import ReplayLogo from "ui/components/shared/ReplayLogo"; import ToggleButton from "ui/components/TestSuite/views/Toggle/ToggleButton"; -import { subscribe } from "ui/components/Video/MutableGraphicsState"; +import { State, state } from "ui/components/Video/imperative/MutableGraphicsState"; +import { subscribeToMutableSources } from "ui/components/Video/imperative/subscribeToMutableSources"; import NodeHighlighter from "ui/components/Video/NodeHighlighter"; import { RecordedCursor } from "ui/components/Video/RecordedCursor"; -import { useDisplayedScreenShot } from "ui/components/Video/useDisplayedScreenShot"; -import { useImperativeVideoPlayback } from "ui/components/Video/useImperativeVideoPlayback"; -import { useSmartTimeAndExecutionPoint } from "ui/components/Video/useSmartTimeAndExecutionPoint"; -import { useUpdateGraphicsContext } from "ui/components/Video/useUpdateGraphicsContext"; import useVideoContextMenu from "ui/components/Video/useVideoContextMenu"; import { getSelectedPrimaryPanel } from "ui/reducers/layout"; -import { useAppDispatch, useAppSelector } from "ui/setup/hooks"; +import { useAppDispatch, useAppSelector, useAppStore } from "ui/setup/hooks"; import styles from "./Video.module.css"; +const SHOW_LOADING_INDICATOR_AFTER_MS = 1_500; + export default function Video() { + const reduxStore = useAppStore(); const replayClient = useContext(ReplayClientContext); + const { status: nodePickerStatus } = useContext(NodePickerContext); - const dispatch = useAppDispatch(); - const panel = useAppSelector(getSelectedPrimaryPanel); + const [showLoadingIndicator, setShowLoadingIndicator] = useState(false); + const [showError, setShowError] = useState(false); - const { executionPoint, time } = useSmartTimeAndExecutionPoint(); + useLayoutEffect(() => { + const containerElement = document.getElementById("video") as HTMLDivElement; + const graphicsElement = document.getElementById("graphics") as HTMLImageElement; + const graphicsOverlayElement = document.getElementById("overlay-graphics") as HTMLDivElement; + + let prevState: Partial = {}; + let stalledTimeout: NodeJS.Timeout | null = null; + + // Keep graphics in sync with the imperatively managed screenshot state + state.listen(nextState => { + if (nextState.screenShot != prevState.screenShot) { + const { screenShot } = nextState; + if (screenShot) { + graphicsElement.src = `data:${screenShot.mimeType};base64,${screenShot.data}`; + } else { + graphicsElement.src = ""; + } + } - const [playbackTime, onCommitCallback] = useImperativeVideoPlayback(); + // Show loading progress bar if graphics stall for longer than 5s + const isLoading = nextState.status === "loading"; + const wasLoading = prevState.status === "loading"; + if (isLoading && !wasLoading) { + stalledTimeout = setTimeout(() => { + setShowLoadingIndicator(true); + }, SHOW_LOADING_INDICATOR_AFTER_MS); + } else if (!isLoading && wasLoading) { + if (stalledTimeout != null) { + clearTimeout(stalledTimeout); + } + + setShowLoadingIndicator(false); + } - const executionPointToSuspend = playbackTime != null ? null : executionPoint; - const timeToSuspend = playbackTime != null ? playbackTime : time; + if (nextState.status === "failed") { + setShowError(true); + } else { + setShowError(false); + } - const streaming = StreamingScreenShotCache.stream( - replayClient, - timeToSuspend, - executionPointToSuspend - ); - const { - data: status = "fetching-cached-paint", - progress = 0, - value: screenShot, - } = useStreamingValue(streaming); + // The data attributes are sed for e2e tests + containerElement.setAttribute("data-execution-point", nextState.currentExecutionPoint ?? ""); + containerElement.setAttribute("data-status", "" + nextState.status); + containerElement.setAttribute("data-time", "" + nextState.currentTime); + graphicsElement.setAttribute("data-scale", nextState.localScale.toString()); - const { addComment, contextMenu, onContextMenu } = useVideoContextMenu(); + Object.assign(prevState, nextState); + }); - useUpdateGraphicsContext(screenShot); + const unsubscribe = subscribeToMutableSources({ + containerElement, + graphicsElement, + graphicsOverlayElement, + reduxStore, + replayClient, + }); - useLayoutEffect(() => { - // When playback is active, this commit callback notifies the imperative code that a screenshot has been rendered - // and it's okay to advance the playback timer to the next frame. - // Without this explicit ack, the imperative playback code could advance too quickly and cause "starvation" - // where the React scheduler didn't finish rendering a previous update before another one was requested. - if (playbackTime != null && screenShot != null) { - onCommitCallback(playbackTime); - } - }, [onCommitCallback, playbackTime, screenShot]); + return () => { + unsubscribe(); - useLayoutEffect(() => { - subscribe(state => { - const graphicsElement = document.getElementById("graphics"); - if (graphicsElement) { - // Scale is used by e2e tests to click on specific elements - graphicsElement.setAttribute("data-scale", state.localScale.toString()); + if (stalledTimeout != null) { + clearTimeout(stalledTimeout); } - }); - }, []); + }; + }, [reduxStore, replayClient]); + + const dispatch = useAppDispatch(); + const panel = useAppSelector(getSelectedPrimaryPanel); + + const { addComment, contextMenu, onContextMenu } = useVideoContextMenu(); const onClick = (event: MouseEvent) => { dispatch(stopPlayback()); @@ -80,57 +108,38 @@ export default function Video() { } }; - const screenShotToRender = useDisplayedScreenShot(screenShot, status, timeToSuspend); - - const showLoader = progress < 1 && playbackTime == null; const showBeforeAfterTestStepToggles = panel === "cypress"; - let showError = false; - if (status === "loading-failed") { - showError = executionPoint != null; // !preferHoverTime && !preferPlaybackTime; - } - return (
- {/* Screenshots are rendered in this HTMLImageElement; if there is no screenshot to render, show the Replay logo instead */} - {screenShotToRender ? ( - - ) : ( -
- -
- )} +
+ +
+ + {/* Graphics that are relative to the rendered screenshot go here; this container is automatically positioned to align with the screenshot */}
- +
- {showLoader && } + {showLoadingIndicator && } + {showError && (
Could not load screenshot
)} + {showBeforeAfterTestStepToggles && } {contextMenu}
diff --git a/src/ui/components/Video/VideoContextMenu.module.css b/src/ui/components/Video/VideoContextMenu.module.css deleted file mode 100644 index 678c44b2bc6..00000000000 --- a/src/ui/components/Video/VideoContextMenu.module.css +++ /dev/null @@ -1,4 +0,0 @@ -.Icon { - width: 1.5rem; - height: 1rem; -} diff --git a/src/ui/components/Video/imperative/MutableGraphicsState.tsx b/src/ui/components/Video/imperative/MutableGraphicsState.tsx new file mode 100644 index 00000000000..2cf1c839811 --- /dev/null +++ b/src/ui/components/Video/imperative/MutableGraphicsState.tsx @@ -0,0 +1,116 @@ +// The recording graphics (screenshots) are global state; there is only one rendered (in the DOM) at any point in time +// Although declarative (React) components render the graphics elements (e.g. comments, DOM highlights, recorded cursor) +// it is better for performance if imperative code (re)positions these elements, +// otherwise there may be "render lag" when windows are resized. +// +// This module exports a subscribe method for that imperative positioning logic, +// and an accompanying update method for notifying subscribers. +// +// This module also exports the global/mutable state directly, in case other imperative code requires it for single use purposes, +// e.g. calculate relative position information from a MouseEvent. +// +// This approach is unusual, but it's arguably cleaner than sharing these values via the DOM. + +import { ExecutionPoint, ScreenShot } from "@replayio/protocol"; + +import { fitImageToContainer, getDimensions } from "replay-next/src/utils/image"; +import { shallowEqual } from "shared/utils/compare"; +import { createState } from "ui/components/Video/imperative/createState"; + +interface Rect { + height: number; + left: number; + top: number; + width: number; +} + +export type Status = "failed" | "loaded" | "loading"; + +export interface State { + currentExecutionPoint: ExecutionPoint | null; + currentTime: number; + graphicsRect: Rect; + localScale: number; + recordingScale: number; + screenShot: ScreenShot | undefined; + status: Status; +} + +export const state = createState({ + currentExecutionPoint: null, + currentTime: 0, + graphicsRect: { + height: 0, + left: 0, + top: 0, + width: 0, + }, + localScale: 1, + recordingScale: 1, + screenShot: undefined, + status: "loading", +}); + +export async function updateState( + containerElement: HTMLElement, + options: Partial<{ + didResize?: boolean; + executionPoint: ExecutionPoint | null; + screenShot: ScreenShot | null; + status: Status; + time: number; + }> = {} +) { + const prevState = state.read(); + + let { + didResize, + executionPoint = prevState.currentExecutionPoint, + screenShot = prevState.screenShot, + status = prevState.status, + time = prevState.currentTime, + } = options; + + let graphicsRect = prevState.graphicsRect; + let localScale = prevState.localScale; + let recordingScale = prevState.recordingScale; + if (screenShot && (screenShot != prevState.screenShot || didResize)) { + const naturalDimensions = await getDimensions(screenShot.data, screenShot.mimeType); + const naturalHeight = naturalDimensions.height; + const naturalWidth = naturalDimensions.width; + + const containerRect = containerElement.getBoundingClientRect(); + const scaledDimensions = await fitImageToContainer({ + containerHeight: containerRect.height, + containerWidth: containerRect.width, + imageHeight: naturalHeight, + imageWidth: naturalWidth, + }); + const clientHeight = scaledDimensions.height; + const clientWidth = scaledDimensions.width; + + localScale = clientWidth / naturalWidth; + recordingScale = screenShot.scale; + + graphicsRect = { + height: clientHeight, + left: containerRect.left + (containerRect.width - clientWidth) / 2, + top: containerRect.top + (containerRect.height - clientHeight) / 2, + width: clientWidth, + }; + } + + const nextState: State = { + currentExecutionPoint: executionPoint, + currentTime: time, + graphicsRect, + localScale, + recordingScale, + screenShot: screenShot || undefined, + status, + }; + + if (!shallowEqual(prevState, nextState)) { + state.update(nextState); + } +} diff --git a/src/ui/components/Video/imperative/createState.ts b/src/ui/components/Video/imperative/createState.ts new file mode 100644 index 00000000000..007d0218f0b --- /dev/null +++ b/src/ui/components/Video/imperative/createState.ts @@ -0,0 +1,51 @@ +export type Listener = (value: Type, prevValue: Type) => void; + +export interface State { + listen: (callback: Listener) => () => void; + read: () => Type; + update: (nextValue: Type) => Type; +} + +export function createState(defaultValue: Type): State { + let currentValue = defaultValue; + let prevValue = currentValue; + + const listeners: Set> = new Set(); + + function listen(callback: Listener) { + listeners.add(callback); + + try { + return () => { + listeners.delete(callback); + }; + } finally { + callback(currentValue, prevValue); + } + } + + function read() { + return currentValue; + } + + function update(nextValue: Type) { + if (currentValue === nextValue) { + return currentValue; + } + + prevValue = currentValue; + currentValue = nextValue; + + listeners.forEach(callback => { + callback(nextValue, prevValue); + }); + + return nextValue; + } + + return { + listen, + read, + update, + }; +} diff --git a/src/ui/components/Video/useSmartTimeAndExecutionPoint.ts b/src/ui/components/Video/imperative/getGraphicsTimeAndExecutionPoint.ts similarity index 70% rename from src/ui/components/Video/useSmartTimeAndExecutionPoint.ts rename to src/ui/components/Video/imperative/getGraphicsTimeAndExecutionPoint.ts index bbafaf73d51..b2932e94cfb 100644 --- a/src/ui/components/Video/useSmartTimeAndExecutionPoint.ts +++ b/src/ui/components/Video/imperative/getGraphicsTimeAndExecutionPoint.ts @@ -1,7 +1,6 @@ -import { useDeferredValue } from "react"; +import { TimeStampedPointRange } from "@replayio/protocol"; import { getExecutionPoint, getTime } from "devtools/client/debugger/src/selectors"; -import { useCurrentFocusWindow } from "replay-next/src/hooks/useCurrentFocusWindow"; import { isTimeInRegion } from "shared/utils/time"; import { getCurrentTime, @@ -9,19 +8,24 @@ import { getPlayback, getShowHoverTimeGraphics, } from "ui/reducers/timeline"; -import { useAppSelector } from "ui/setup/hooks"; +import { UIState } from "ui/state"; -export function useSmartTimeAndExecutionPoint() { - const playbackState = useAppSelector(getPlayback); - const hoverTime = useAppSelector(getHoverTime); - const preferHoverTime = useAppSelector(getShowHoverTimeGraphics); - const pauseExecutionPoint = useAppSelector(getExecutionPoint); - const pauseTime = useAppSelector(getTime); - const currentTime = useAppSelector(getCurrentTime); - const focusWindow = useCurrentFocusWindow(); +export function getGraphicsTimeAndExecutionPoint( + state: UIState, + focusWindow: TimeStampedPointRange | null +) { + const playbackState = getPlayback(state); + const hoverTime = getHoverTime(state); + const preferHoverTime = getShowHoverTimeGraphics(state); + const pauseExecutionPoint = getExecutionPoint(state); + const pauseTime = getTime(state); + const currentTime = getCurrentTime(state); + + const isHovering = preferHoverTime; + const isPlaying = playbackState != null; let preferCurrentTime = false; - if (playbackState != null) { + if (isPlaying) { preferCurrentTime = true; } else if ( focusWindow && @@ -47,16 +51,16 @@ export function useSmartTimeAndExecutionPoint() { preferCurrentTime = true; } + let time: number; let executionPoint: string | null = null; - let time = 0; if (preferCurrentTime) { time = currentTime; - } else if (preferHoverTime && hoverTime != null) { + } else if (isHovering && hoverTime != null) { time = hoverTime; } else { time = pauseTime; executionPoint = pauseExecutionPoint; } - return useDeferredValue({ executionPoint, time }); + return { executionPoint, time }; } diff --git a/src/ui/components/Video/getMouseEventPosition.ts b/src/ui/components/Video/imperative/getMouseEventPosition.ts similarity index 70% rename from src/ui/components/Video/getMouseEventPosition.ts rename to src/ui/components/Video/imperative/getMouseEventPosition.ts index b04bee2daf4..11dc4d8f766 100644 --- a/src/ui/components/Video/getMouseEventPosition.ts +++ b/src/ui/components/Video/imperative/getMouseEventPosition.ts @@ -1,8 +1,9 @@ -import { mutableState } from "ui/components/Video/MutableGraphicsState"; +import { state } from "ui/components/Video/imperative/MutableGraphicsState"; // Get the x/y coordinate of a mouse event relative to the recording's DOM. export function getMouseEventPosition(event: MouseEvent) { - const { height, left, localScale, recordingScale, top, width } = mutableState; + const { graphicsRect, localScale, recordingScale } = state.read(); + const { height, left, top, width } = graphicsRect; if ( event.clientX < left || diff --git a/src/ui/components/Video/imperative/repositionGraphicsOverlayElements.ts b/src/ui/components/Video/imperative/repositionGraphicsOverlayElements.ts new file mode 100644 index 00000000000..de05ab72fdc --- /dev/null +++ b/src/ui/components/Video/imperative/repositionGraphicsOverlayElements.ts @@ -0,0 +1,17 @@ +export function repositionGraphicsOverlayElements({ + containerElement, + graphicsElement, + graphicsOverlayElement, +}: { + containerElement: HTMLElement; + graphicsElement: HTMLElement; + graphicsOverlayElement: HTMLElement; +}) { + const containerRect = containerElement.getBoundingClientRect(); + const imageRect = graphicsElement.getBoundingClientRect(); + + graphicsOverlayElement.style.left = `${imageRect.left - containerRect.left}px`; + graphicsOverlayElement.style.top = `${imageRect.top - containerRect.top}px`; + graphicsOverlayElement.style.width = `${imageRect.width}px`; + graphicsOverlayElement.style.height = `${imageRect.height}px`; +} diff --git a/src/ui/components/Video/imperative/runVideoPlayback.ts b/src/ui/components/Video/imperative/runVideoPlayback.ts new file mode 100644 index 00000000000..686d1628490 --- /dev/null +++ b/src/ui/components/Video/imperative/runVideoPlayback.ts @@ -0,0 +1,80 @@ +import { ExecutionPoint } from "@replayio/protocol"; + +import { ReplayClientInterface } from "shared/client/types"; +import { seek, stopPlayback } from "ui/actions/timeline"; +import { updateGraphics } from "ui/components/Video/imperative/updateGraphics"; +import { setTimelineState } from "ui/reducers/timeline"; +import { AppStore } from "ui/setup/store"; + +export async function runVideoPlayback({ + abortSignal, + beginPoint, + beginTime, + containerElement, + endPoint, + endTime, + reduxStore, + replayClient, +}: { + abortSignal: AbortSignal; + beginPoint: ExecutionPoint | null; + beginTime: number; + containerElement: HTMLElement; + endPoint: ExecutionPoint | null; + endTime: number; + reduxStore: AppStore; + replayClient: ReplayClientInterface; +}) { + let previousDateNow = Date.now(); + let previousTimelineTime = beginTime; + + playbackLoop: while (true) { + const currentDateNow = Date.now(); + const elapsedTime = currentDateNow - previousDateNow; + previousDateNow = currentDateNow; + + // Playback may have stalled waiting for graphics to load + // We shouldn't jump too far ahead in the timeline, or it won't feel smooth + // So cap the max amount of time we advance to 100ms + const nextTimelineTimeMax = previousTimelineTime + 100; + const nextTimelineTimeMin = previousTimelineTime + 10; + const nextTimelineTime = Math.min( + nextTimelineTimeMax, + Math.max(nextTimelineTimeMin, Math.min(endTime, previousTimelineTime + elapsedTime)) + ); + previousTimelineTime = nextTimelineTime; + + reduxStore.dispatch( + setTimelineState({ + currentTime: nextTimelineTime, + playback: { beginPoint, beginTime, endPoint, endTime, time: nextTimelineTime }, + }) + ); + + if (nextTimelineTime >= endTime) { + reduxStore.dispatch(stopPlayback(true)); + reduxStore.dispatch(seek({ executionPoint: endPoint || undefined, time: endTime })); + + break playbackLoop; + } else { + const updateGraphicsPromise = updateGraphics({ + abortSignal, + containerElement, + executionPoint: null, + replayClient, + time: nextTimelineTime, + }); + + // We may have already cached screenshot data, or we may be able to fetch it quickly + // In order to avoid starving the rendering pipeline, wait a minimum amount of time before continuing + // 16ms chosen because it allows an overall rate of 60fps + await Promise.all([new Promise(resolve => setTimeout(resolve, 16)), updateGraphicsPromise]); + + if (abortSignal.aborted) { + reduxStore.dispatch(stopPlayback()); + + break playbackLoop; + } + } + } +} diff --git a/src/ui/components/Video/imperative/subscribeToMutableSources.ts b/src/ui/components/Video/imperative/subscribeToMutableSources.ts new file mode 100644 index 00000000000..047738a708b --- /dev/null +++ b/src/ui/components/Video/imperative/subscribeToMutableSources.ts @@ -0,0 +1,115 @@ +import { ExecutionPoint } from "@replayio/protocol"; + +import { ReplayClientInterface } from "shared/client/types"; +import { getGraphicsTimeAndExecutionPoint } from "ui/components/Video/imperative/getGraphicsTimeAndExecutionPoint"; +import { updateState } from "ui/components/Video/imperative/MutableGraphicsState"; +import { repositionGraphicsOverlayElements } from "ui/components/Video/imperative/repositionGraphicsOverlayElements"; +import { runVideoPlayback } from "ui/components/Video/imperative/runVideoPlayback"; +import { updateGraphics } from "ui/components/Video/imperative/updateGraphics"; +import { getPlayback, isPlaying } from "ui/reducers/timeline"; +import { AppStore } from "ui/setup/store"; + +export function subscribeToMutableSources({ + containerElement, + graphicsElement, + graphicsOverlayElement, + reduxStore, + replayClient, +}: { + containerElement: HTMLElement; + graphicsElement: HTMLImageElement; + graphicsOverlayElement: HTMLElement; + reduxStore: AppStore; + replayClient: ReplayClientInterface; +}) { + let abortController: AbortController | null = null; + let prevTime: number | null = null; + let prevExecutionPoint: ExecutionPoint | null = null; + let prevIsPlaying = false; + + const resizeObserver = new ResizeObserver(() => { + repositionGraphicsOverlayElements({ + containerElement, + graphicsElement, + graphicsOverlayElement, + }); + + // When the graphics element resizes, the mutable state should recalculate the its layout + updateState(containerElement, { didResize: true }); + }); + + resizeObserver.observe(containerElement); + resizeObserver.observe(graphicsElement); + + const update = () => { + const state = reduxStore.getState(); + + const playbackState = getPlayback(state); + + const isPlaying = playbackState != null; + + const { executionPoint, time } = getGraphicsTimeAndExecutionPoint( + state, + replayClient.getCurrentFocusWindow() + ); + + if (prevExecutionPoint === executionPoint && prevIsPlaying === isPlaying && prevTime === time) { + return; + } + + const didStartPlaying = isPlaying && !prevIsPlaying; + + prevTime = time; + prevExecutionPoint = executionPoint; + prevIsPlaying = isPlaying; + + if (didStartPlaying) { + if (abortController != null) { + abortController.abort(); + abortController = null; + } + + abortController = new AbortController(); + + runVideoPlayback({ + abortSignal: abortController.signal, + beginPoint: playbackState.beginPoint, + beginTime: playbackState.beginTime, + containerElement, + endPoint: playbackState.endPoint, + endTime: playbackState.endTime, + reduxStore, + replayClient, + }); + } else if (!isPlaying) { + if (abortController != null) { + abortController.abort(); + abortController = null; + } + + abortController = new AbortController(); + + updateGraphics({ + abortSignal: abortController.signal, + containerElement, + executionPoint, + replayClient, + time, + }); + } + }; + + const unsubscribeFromReduxStore = reduxStore.subscribe(update); + + update(); + + return () => { + if (abortController != null) { + abortController.abort(); + } + + unsubscribeFromReduxStore(); + + resizeObserver.disconnect(); + }; +} diff --git a/src/ui/components/Video/imperative/updateGraphics.ts b/src/ui/components/Video/imperative/updateGraphics.ts new file mode 100644 index 00000000000..6e0ed46e760 --- /dev/null +++ b/src/ui/components/Video/imperative/updateGraphics.ts @@ -0,0 +1,156 @@ +import { ExecutionPoint, ScreenShot } from "@replayio/protocol"; + +import { PaintsCache, findMostRecentPaint } from "protocol/PaintsCache"; +import { RepaintGraphicsCache } from "protocol/RepaintGraphicsCache"; +import { pauseIdCache } from "replay-next/src/suspense/PauseCache"; +import { screenshotCache } from "replay-next/src/suspense/ScreenshotCache"; +import { ReplayClientInterface } from "shared/client/types"; +import { updateState } from "ui/components/Video/imperative/MutableGraphicsState"; + +export async function updateGraphics({ + abortSignal, + containerElement, + executionPoint, + replayClient, + time, +}: { + abortSignal: AbortSignal; + containerElement: HTMLElement; + executionPoint: ExecutionPoint | null; + time: number; + replayClient: ReplayClientInterface; +}) { + updateState(containerElement, { + executionPoint, + status: "loading", + time, + }); + + await PaintsCache.readAsync(); + + // If the current time is before the first paint, we should show nothing + const paintPoint = findMostRecentPaint(time); + if (!paintPoint || !paintPoint.paintHash) { + updateState(containerElement, { + executionPoint, + screenShot: null, + status: "loaded", + time, + }); + return; + } + + const promises: Promise[] = [ + fetchPaintContents({ + abortSignal, + containerElement, + replayClient, + time, + }), + ]; + + let repaintGraphicsScreenShot: ScreenShot | undefined = undefined; + if (executionPoint) { + const promise = fetchRepaintGraphics({ + executionPoint, + replayClient, + time, + }).then(screenShot => { + repaintGraphicsScreenShot = screenShot; + + return screenShot; + }); + + promises.push(promise); + } + + // Fetch paint contents and repaint graphics in parallel + const screenShot = await Promise.race(promises); + if (abortSignal.aborted) { + return; + } + + // Show the first screenshot we get back (if we've found one) + if (screenShot != null) { + updateState(containerElement, { + executionPoint, + screenShot, + status: "loaded", + time, + }); + + if (repaintGraphicsScreenShot != null) { + // If the repaint graphics promise finished first, we can bail out + // this is guaranteed to be the more up-to-date screenshot + return true; + } + } + + if (executionPoint) { + // Otherwise wait until the repaint graphics promise finishes + await Promise.all(promises); + if (abortSignal.aborted) { + return; + } + + if (repaintGraphicsScreenShot != null) { + // The repaint graphics API fails a lot; it should fail quietly here + updateState(containerElement, { + executionPoint, + screenShot: repaintGraphicsScreenShot, + status: "loaded", + time, + }); + } + } +} + +async function fetchPaintContents({ + abortSignal, + containerElement, + replayClient, + time, +}: { + abortSignal: AbortSignal; + containerElement: HTMLElement; + time: number; + replayClient: ReplayClientInterface; +}): Promise { + const paintPoint = findMostRecentPaint(time); + if (!paintPoint || !paintPoint.paintHash) { + // Don't try to paint (or repaint) if the current time is before the first cached paint + return undefined; + } + + try { + return await screenshotCache.readAsync(replayClient, paintPoint.point, paintPoint.paintHash); + } catch (error) { + if (abortSignal.aborted) { + return; + } + + updateState(containerElement, { + status: "failed", + }); + } +} + +async function fetchRepaintGraphics({ + executionPoint, + replayClient, + time, +}: { + executionPoint: ExecutionPoint; + time: number; + replayClient: ReplayClientInterface; +}): Promise { + const pauseId = await pauseIdCache.readAsync(replayClient, executionPoint, time); + + try { + const result = await RepaintGraphicsCache.readAsync(replayClient, pauseId); + + return result?.screenShot; + } catch (error) { + // Repaint graphics are currently expected to fail + } +} diff --git a/src/ui/components/Video/useDisplayedScreenShot.ts b/src/ui/components/Video/useDisplayedScreenShot.ts deleted file mode 100644 index e01323dd837..00000000000 --- a/src/ui/components/Video/useDisplayedScreenShot.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { ScreenShot } from "@replayio/protocol"; -import { useEffect, useState } from "react"; - -import { StreamingScreenShotCacheStatus } from "protocol/StreamingScreenShotCache"; -import { getShowHoverTimeGraphics } from "ui/reducers/timeline"; -import { useAppSelector } from "ui/setup/hooks"; - -export function useDisplayedScreenShot( - screenShot: ScreenShot | undefined, - cacheStatus: StreamingScreenShotCacheStatus, - time: number -) { - const preferHoverTime = useAppSelector(getShowHoverTimeGraphics); - - const [prevScreenShotData, setPrevScreenShotData] = useState<{ - screenShot: ScreenShot | undefined; - time: number; - }>({ - screenShot: undefined, - time: 0, - }); - - useEffect(() => { - if (screenShot != null && !preferHoverTime) { - setPrevScreenShotData({ - screenShot, - time, - }); - } - }, [screenShot, preferHoverTime, time]); - - if (cacheStatus === "before-first-paint") { - // Special case: - // If the screenshot failed to load at a point before the first cached paint, we should show an empty screen - return undefined; - } - - // If we're seeking (or playing) forward, continue to show the previous screenshot - // This prevents "flickering" while the new one loads - return screenShot ?? prevScreenShotData.screenShot; -} diff --git a/src/ui/components/Video/useImperativeVideoPlayback.ts b/src/ui/components/Video/useImperativeVideoPlayback.ts deleted file mode 100644 index 363dba30af4..00000000000 --- a/src/ui/components/Video/useImperativeVideoPlayback.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { useCallback, useContext, useLayoutEffect, useRef, useState } from "react"; -import { Deferred, createDeferred } from "suspense"; - -import { waitForTime } from "protocol/utils"; -import { ReplayClientContext } from "shared/client/ReplayClientContext"; -import { seek, stopPlayback } from "ui/actions/timeline"; -import { getPlayback, setTimelineState } from "ui/reducers/timeline"; -import { useAppDispatch, useAppSelector } from "ui/setup/hooks"; - -// When playing starts, this hook will listen for changes in cached screenshots/status and advance the playback time -// It's not enough to listen to the cache directly; the screenshot has to render and commit (be visible to the user) -export function useImperativeVideoPlayback(): [ - playbackTime: number | null, - onCommitCallback: (time: number) => void -] { - const replayClient = useContext(ReplayClientContext); - const dispatch = useAppDispatch(); - - const { beginDate, beginTime, endTime } = useAppSelector(getPlayback) ?? {}; - - const [playbackTime, setPlaybackTime] = useState(null); - - const committedTimesMapRef = useRef>>(new Map()); - - useLayoutEffect(() => { - if (beginTime == null || beginDate == null || endTime == null) { - return; - } - - let previousTime = beginTime; - let stop = false; - - const committedTimesMap = committedTimesMapRef.current; - - const playVideo = async () => { - committedTimesMap.clear(); - - while (true) { - if (stop) { - break; - } - - const nextTime = Math.max( - previousTime + 1, - Math.min(endTime, beginTime + (Date.now() - beginDate)) - ); - - previousTime = nextTime; - - dispatch( - setTimelineState({ - currentTime: nextTime, - playback: { beginTime, beginDate, endTime, time: nextTime }, - }) - ); - - if (nextTime >= endTime) { - dispatch(stopPlayback()); - break; - } else { - setPlaybackTime(nextTime); - - const deferred = createDeferred(); - - committedTimesMap.set(nextTime, deferred); - - await Promise.all([ - // If the screenshot nearest to the current time has already been cached, - // we might end up with an immediate resolution here; - // Since this is a while loop, we don't want to block the main thread. - // So add some minimum amount of delay in here; - // any amount should be okay but let's shoot for a 60fps - waitForTime(16.7), - Promise.race([ - // Wait until the Video component has loaded and displayed the next screenshot - deferred.promise, - // Give up and move on if it takes longer than 500ms for the snapshot to load - waitForTime(250), - ]), - ]); - } - } - }; - - playVideo(); - - return () => { - stop = true; - - // Jump to the nearest recorded event (mouse or paint) - // so that Redux state is updated with approximately the same time as playback was stopped at - dispatch(seek({ time: previousTime })); - - setPlaybackTime(null); - }; - }, [beginDate, beginTime, dispatch, endTime, replayClient]); - - const onCommitCallback = useCallback((time: number | null) => { - const deferred = committedTimesMapRef.current.get(time); - if (deferred) { - if (deferred.status === "pending") { - deferred.resolve(); - } else { - console.warn(`Already marked time ${time} as committed`); - } - } - }, []); - - return [playbackTime, onCommitCallback]; -} diff --git a/src/ui/components/Video/useUpdateGraphicsContext.ts b/src/ui/components/Video/useUpdateGraphicsContext.ts deleted file mode 100644 index 8836d51b088..00000000000 --- a/src/ui/components/Video/useUpdateGraphicsContext.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { ScreenShot } from "@replayio/protocol"; -import { useLayoutEffect } from "react"; - -import { update } from "ui/components/Video/MutableGraphicsState"; - -export function useUpdateGraphicsContext(screenShot: ScreenShot | undefined) { - // Whenever we render a new screenshot, update the bounding rect and scale info - useLayoutEffect(() => { - const imageElement = document.getElementById("graphics"); - if (imageElement != null && screenShot != null) { - update({ element: imageElement as HTMLImageElement, scale: screenShot.scale }); - } - - repositionGraphics(); - }); - - // Whenever the video element resizes, update the bounding rect info - useLayoutEffect(() => { - const element = document.getElementById("video"); - if (element) { - const observer = new ResizeObserver(() => { - repositionGraphics(); - - const imageElement = document.getElementById("graphics"); - if (imageElement != null) { - update({ element: imageElement as HTMLImageElement }); - } - }); - - observer.observe(element); - - return () => { - observer.disconnect(); - }; - } - }, []); -} - -function repositionGraphics() { - const container = document.getElementById("video"); - const graphicsElement = document.getElementById("graphics"); - const overlayGraphicsElement = document.getElementById("overlay-graphics"); - - if (container != null && graphicsElement != null && overlayGraphicsElement != null) { - const containerRect = container.getBoundingClientRect(); - const imageRect = graphicsElement.getBoundingClientRect(); - - overlayGraphicsElement.style.left = `${imageRect.left - containerRect.left}px`; - overlayGraphicsElement.style.top = `${imageRect.top - containerRect.top}px`; - overlayGraphicsElement.style.width = `${imageRect.width}px`; - overlayGraphicsElement.style.height = `${imageRect.height}px`; - } -} diff --git a/src/ui/components/Video/useVideoContextMenu.tsx b/src/ui/components/Video/useVideoContextMenu.tsx index dbb31582743..abe0e3aefcd 100644 --- a/src/ui/components/Video/useVideoContextMenu.tsx +++ b/src/ui/components/Video/useVideoContextMenu.tsx @@ -17,12 +17,12 @@ import { Nag } from "shared/graphql/types"; import { createFrameComment } from "ui/actions/comments"; import { setSelectedPanel, setViewMode } from "ui/actions/layout"; import { stopPlayback } from "ui/actions/timeline"; -import { getMouseEventPosition } from "ui/components/Video/getMouseEventPosition"; +import { getMouseEventPosition } from "ui/components/Video/imperative/getMouseEventPosition"; import { isPlaying as isPlayingSelector } from "ui/reducers/timeline"; import { useAppDispatch, useAppSelector } from "ui/setup/hooks"; import { boundingRectsCache, getMouseTarget } from "ui/suspense/nodeCaches"; -import styles from "./VideoContextMenu.module.css"; +import styles from "./Video.module.css"; export default function useVideoContextMenu() { const { showCommentsPanel } = useContext(InspectorContext); @@ -128,14 +128,14 @@ export default function useVideoContextMenu() { onSelect={() => addComment(mouseEventDataRef.current)} > <> - + Add comment )} <> - + Inspect element diff --git a/src/ui/reducers/timeline.ts b/src/ui/reducers/timeline.ts index 74f61d44a99..f9d64d1b601 100644 --- a/src/ui/reducers/timeline.ts +++ b/src/ui/reducers/timeline.ts @@ -20,7 +20,6 @@ function initialTimelineState(): TimelineState { hoverTime: null, playback: null, playbackFocusWindow: false, - playbackPrecachedTime: 0, endpoint: { time: 0, point: "0" }, recordingDuration: null, shouldAnimate: true, @@ -51,9 +50,6 @@ const timelineSlice = createSlice({ setHoveredItem(state, action: PayloadAction) { state.hoveredItem = action.payload; }, - setPlaybackPrecachedTime(state, action: PayloadAction) { - state.playbackPrecachedTime = action.payload; - }, setPlaybackFocusWindow(state, action: PayloadAction) { state.playbackFocusWindow = action.payload; }, @@ -70,7 +66,6 @@ export const { setDragging, setHoveredItem, setMarkTimeStampPoint, - setPlaybackPrecachedTime, setPlaybackFocusWindow, setFocusWindow, setTimelineState, @@ -92,7 +87,6 @@ export const getTimelineDimensions = (state: UIState) => state.timeline.timeline export const getMarkTimeStampedPoint = (state: UIState) => state.timeline.markTimeStampedPoint; export const getHoveredItem = (state: UIState) => state.timeline.hoveredItem; export const getEndpoint = (state: UIState) => state.timeline.endpoint; -export const getPlaybackPrecachedTime = (state: UIState) => state.timeline.playbackPrecachedTime; export const getPlaybackFocusWindow = (state: UIState) => state.timeline.playbackFocusWindow; export const getFocusWindow = (state: UIState) => state.timeline.focusWindow; export const isMaximumFocusWindow = (state: UIState) => { diff --git a/src/ui/setup/store.ts b/src/ui/setup/store.ts index 778d645b63c..ac22d3b0553 100644 --- a/src/ui/setup/store.ts +++ b/src/ui/setup/store.ts @@ -72,7 +72,6 @@ const reduxDevToolsOptions: ReduxDevToolsOptions = { "protocolMessages/responseReceived", "protocolMessages/errorReceived", "protocolMessages/requestSent", - "timeline/setPlaybackPrecachedTime", ], }; diff --git a/src/ui/state/timeline.ts b/src/ui/state/timeline.ts index 7284ebfbdf7..7e4170b37dc 100644 --- a/src/ui/state/timeline.ts +++ b/src/ui/state/timeline.ts @@ -1,4 +1,4 @@ -import { Location, TimeStampedPoint } from "@replayio/protocol"; +import { ExecutionPoint, Location, TimeStampedPoint } from "@replayio/protocol"; export interface TimeRange { begin: number; @@ -20,13 +20,13 @@ export interface TimelineState { markTimeStampedPoint: TimeStampedPoint | null; maxRecordingDurationForRoutines: number; playback: { + beginPoint: ExecutionPoint | null; beginTime: number; - beginDate: number; + endPoint: ExecutionPoint | null; endTime: number; time: number; } | null; playbackFocusWindow: boolean; - playbackPrecachedTime: number; endpoint: TimeStampedPoint; recordingDuration: number | null; shouldAnimate: boolean; @@ -55,6 +55,8 @@ export enum FocusOperation { } export type PlaybackOptions = { + beginPoint?: ExecutionPoint | null; beginTime: number | null; + endPoint?: ExecutionPoint | null; endTime: number | null; };