Skip to content

Commit

Permalink
Improvements for embedding / playback (#297)
Browse files Browse the repository at this point in the history 
* Improvements for embedding / playback

* Add comment
  • Loading branch information
@brentyi
brentyi authored Oct 14, 2024
1 parent 5f5977b commit 76f1ebf
Show file tree
Hide file tree
Showing 4 changed files with 230 additions and 155 deletions.
1 change: 1 addition & 0 deletions src/viser/client/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-error-boundary": "^4.0.10",
"react-intersection-observer": "^9.13.1",
"react-qr-code": "^2.0.12",
"react-router-dom": "^6.10.0",
"rehype-color-chips": "^0.1.3",
Expand Down
336 changes: 181 additions & 155 deletions src/viser/client/src/App.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import "@mantine/core/styles.css";
import "@mantine/notifications/styles.css";
import "./App.css";
import { useInView } from "react-intersection-observer";

import { Notifications } from "@mantine/notifications";

Expand Down Expand Up @@ -281,176 +282,201 @@ function ViewerContents({ children }: { children: React.ReactNode }) {
);
}

const DisableRender = () => useFrame(() => null, 1000);

function ViewerCanvas({ children }: { children: React.ReactNode }) {
const viewer = React.useContext(ViewerContext)!;
const sendClickThrottled = useThrottledMessageSender(20);
const theme = useMantineTheme();

// Make sure we don't re-mount the camera controls, since that will reset the camera position.
const memoizedCameraControls = React.useMemo(
() => <SynchronizedCameraControls />,
[],
);

// We'll disable rendering if the canvas is not in view.
const { ref: inViewRef, inView } = useInView();

return (
<Canvas
camera={{ position: [-3.0, 3.0, -3.0], near: 0.05 }}
gl={{ preserveDrawingBuffer: true }}
<div
ref={inViewRef}
style={{
position: "relative",
zIndex: 0,
width: "100%",
height: "100%",
}}
ref={viewer.canvasRef}
// Handle scene click events (onPointerDown, onPointerMove, onPointerUp)
onPointerDown={(e) => {
const pointerInfo = viewer.scenePointerInfo.current!;

// Only handle pointer events if enabled.
if (pointerInfo.enabled === false) return;

// Keep track of the first click position.
const canvasBbox = viewer.canvasRef.current!.getBoundingClientRect();
pointerInfo.dragStart = [
e.clientX - canvasBbox.left,
e.clientY - canvasBbox.top,
];
pointerInfo.dragEnd = pointerInfo.dragStart;

// Check if pointer position is in bounds.
if (ndcFromPointerXy(viewer, pointerInfo.dragEnd) === null) return;

// Only allow one drag event at a time.
if (pointerInfo.isDragging) return;
pointerInfo.isDragging = true;

// Disable camera controls -- we don't want the camera to move while we're dragging.
viewer.cameraControlRef.current!.enabled = false;

const ctx = viewer.canvas2dRef.current!.getContext("2d")!;
ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
}}
onPointerMove={(e) => {
const pointerInfo = viewer.scenePointerInfo.current!;

// Only handle if click events are enabled, and if pointer is down (i.e., dragging).
if (pointerInfo.enabled === false || !pointerInfo.isDragging) return;

// Check if pointer position is in boudns.
const canvasBbox = viewer.canvasRef.current!.getBoundingClientRect();
const pointerXy: [number, number] = [
e.clientX - canvasBbox.left,
e.clientY - canvasBbox.top,
];
if (ndcFromPointerXy(viewer, pointerXy) === null) return;

// Check if mouse position has changed sufficiently from last position.
// Uses 3px as a threshood, similar to drag detection in
// `SceneNodeClickMessage` from `SceneTree.tsx`.
pointerInfo.dragEnd = pointerXy;
if (
Math.abs(pointerInfo.dragEnd[0] - pointerInfo.dragStart[0]) <= 3 &&
Math.abs(pointerInfo.dragEnd[1] - pointerInfo.dragStart[1]) <= 3
)
return;

// If we're listening for scene box events, draw the box on the 2D canvas for user feedback.
if (pointerInfo.enabled === "rect-select") {
const ctx = viewer.canvas2dRef.current!.getContext("2d")!;
ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
ctx.beginPath();
ctx.fillStyle = theme.primaryColor;
ctx.strokeStyle = "blue";
ctx.globalAlpha = 0.2;
ctx.fillRect(
pointerInfo.dragStart[0],
pointerInfo.dragStart[1],
pointerInfo.dragEnd[0] - pointerInfo.dragStart[0],
pointerInfo.dragEnd[1] - pointerInfo.dragStart[1],
);
ctx.globalAlpha = 1.0;
ctx.stroke();
}
}}
onPointerUp={() => {
const pointerInfo = viewer.scenePointerInfo.current!;

// Re-enable camera controls! Was disabled in `onPointerDown`, to allow
// for mouse drag w/o camera movement.
viewer.cameraControlRef.current!.enabled = true;

// Only handle if click events are enabled, and if pointer was down (i.e., dragging).
if (pointerInfo.enabled === false || !pointerInfo.isDragging) return;

const ctx = viewer.canvas2dRef.current!.getContext("2d")!;
ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);

// If there's only one pointer, send a click message.
// The message will return origin/direction lists of length 1.
if (pointerInfo.enabled === "click") {
const raycaster = new THREE.Raycaster();

// Raycaster expects NDC coordinates, so we convert the click event to NDC.
const mouseVector = ndcFromPointerXy(viewer, pointerInfo.dragEnd);
if (mouseVector === null) return;
raycaster.setFromCamera(mouseVector, viewer.cameraRef.current!);
const ray = rayToViserCoords(viewer, raycaster.ray);

// Send OpenCV image coordinates to the server (normalized).
const mouseVectorOpenCV = opencvXyFromPointerXy(
viewer,
pointerInfo.dragEnd,
);
>
<Canvas
camera={{ position: [-3.0, 3.0, -3.0], near: 0.05 }}
gl={{ preserveDrawingBuffer: true }}
dpr={0.6 * window.devicePixelRatio /* Relaxed initial DPR. */}
style={{
width: "100%",
height: "100%",
}}
ref={viewer.canvasRef}
// Handle scene click events (onPointerDown, onPointerMove, onPointerUp)
onPointerDown={(e) => {
const pointerInfo = viewer.scenePointerInfo.current!;

// Only handle pointer events if enabled.
if (pointerInfo.enabled === false) return;

// Keep track of the first click position.
const canvasBbox = viewer.canvasRef.current!.getBoundingClientRect();
pointerInfo.dragStart = [
e.clientX - canvasBbox.left,
e.clientY - canvasBbox.top,
];
pointerInfo.dragEnd = pointerInfo.dragStart;

sendClickThrottled({
type: "ScenePointerMessage",
event_type: "click",
ray_origin: [ray.origin.x, ray.origin.y, ray.origin.z],
ray_direction: [ray.direction.x, ray.direction.y, ray.direction.z],
screen_pos: [[mouseVectorOpenCV.x, mouseVectorOpenCV.y]],
});
} else if (pointerInfo.enabled === "rect-select") {
// If the ScenePointerEvent had mouse drag movement, we will send a "box" message:
// Use the first and last mouse positions to create a box.
// Again, click should be in openCV image coordinates (normalized).
const firstMouseVector = opencvXyFromPointerXy(
viewer,
pointerInfo.dragStart,
);
const lastMouseVector = opencvXyFromPointerXy(
viewer,
pointerInfo.dragEnd,
);
// Check if pointer position is in bounds.
if (ndcFromPointerXy(viewer, pointerInfo.dragEnd) === null) return;

const x_min = Math.min(firstMouseVector.x, lastMouseVector.x);
const x_max = Math.max(firstMouseVector.x, lastMouseVector.x);
const y_min = Math.min(firstMouseVector.y, lastMouseVector.y);
const y_max = Math.max(firstMouseVector.y, lastMouseVector.y);
// Only allow one drag event at a time.
if (pointerInfo.isDragging) return;
pointerInfo.isDragging = true;

// Send the upper-left and lower-right corners of the box.
const screenBoxList: [number, number][] = [
[x_min, y_min],
[x_max, y_max],
// Disable camera controls -- we don't want the camera to move while we're dragging.
viewer.cameraControlRef.current!.enabled = false;

const ctx = viewer.canvas2dRef.current!.getContext("2d")!;
ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
}}
onPointerMove={(e) => {
const pointerInfo = viewer.scenePointerInfo.current!;

// Only handle if click events are enabled, and if pointer is down (i.e., dragging).
if (pointerInfo.enabled === false || !pointerInfo.isDragging) return;

// Check if pointer position is in boudns.
const canvasBbox = viewer.canvasRef.current!.getBoundingClientRect();
const pointerXy: [number, number] = [
e.clientX - canvasBbox.left,
e.clientY - canvasBbox.top,
];
if (ndcFromPointerXy(viewer, pointerXy) === null) return;

// Check if mouse position has changed sufficiently from last position.
// Uses 3px as a threshood, similar to drag detection in
// `SceneNodeClickMessage` from `SceneTree.tsx`.
pointerInfo.dragEnd = pointerXy;
if (
Math.abs(pointerInfo.dragEnd[0] - pointerInfo.dragStart[0]) <= 3 &&
Math.abs(pointerInfo.dragEnd[1] - pointerInfo.dragStart[1]) <= 3
)
return;

// If we're listening for scene box events, draw the box on the 2D canvas for user feedback.
if (pointerInfo.enabled === "rect-select") {
const ctx = viewer.canvas2dRef.current!.getContext("2d")!;
ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
ctx.beginPath();
ctx.fillStyle = theme.primaryColor;
ctx.strokeStyle = "blue";
ctx.globalAlpha = 0.2;
ctx.fillRect(
pointerInfo.dragStart[0],
pointerInfo.dragStart[1],
pointerInfo.dragEnd[0] - pointerInfo.dragStart[0],
pointerInfo.dragEnd[1] - pointerInfo.dragStart[1],
);
ctx.globalAlpha = 1.0;
ctx.stroke();
}
}}
onPointerUp={() => {
const pointerInfo = viewer.scenePointerInfo.current!;

// Re-enable camera controls! Was disabled in `onPointerDown`, to allow
// for mouse drag w/o camera movement.
viewer.cameraControlRef.current!.enabled = true;

// Only handle if click events are enabled, and if pointer was down (i.e., dragging).
if (pointerInfo.enabled === false || !pointerInfo.isDragging) return;

sendClickThrottled({
type: "ScenePointerMessage",
event_type: "rect-select",
ray_origin: null,
ray_direction: null,
screen_pos: screenBoxList,
});
}

// Release drag lock.
pointerInfo.isDragging = false;
}}
>
{children}
<BackgroundImage />
<AdaptiveDpr />
<SceneContextSetter />
<SynchronizedCameraControls />
<SplatRenderContext>
<SceneNodeThreeObject name="" parent={null} />
</SplatRenderContext>
<DefaultLights />
</Canvas>
const ctx = viewer.canvas2dRef.current!.getContext("2d")!;
ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);

// If there's only one pointer, send a click message.
// The message will return origin/direction lists of length 1.
if (pointerInfo.enabled === "click") {
const raycaster = new THREE.Raycaster();

// Raycaster expects NDC coordinates, so we convert the click event to NDC.
const mouseVector = ndcFromPointerXy(viewer, pointerInfo.dragEnd);
if (mouseVector === null) return;
raycaster.setFromCamera(mouseVector, viewer.cameraRef.current!);
const ray = rayToViserCoords(viewer, raycaster.ray);

// Send OpenCV image coordinates to the server (normalized).
const mouseVectorOpenCV = opencvXyFromPointerXy(
viewer,
pointerInfo.dragEnd,
);

sendClickThrottled({
type: "ScenePointerMessage",
event_type: "click",
ray_origin: [ray.origin.x, ray.origin.y, ray.origin.z],
ray_direction: [
ray.direction.x,
ray.direction.y,
ray.direction.z,
],
screen_pos: [[mouseVectorOpenCV.x, mouseVectorOpenCV.y]],
});
} else if (pointerInfo.enabled === "rect-select") {
// If the ScenePointerEvent had mouse drag movement, we will send a "box" message:
// Use the first and last mouse positions to create a box.
// Again, click should be in openCV image coordinates (normalized).
const firstMouseVector = opencvXyFromPointerXy(
viewer,
pointerInfo.dragStart,
);
const lastMouseVector = opencvXyFromPointerXy(
viewer,
pointerInfo.dragEnd,
);

const x_min = Math.min(firstMouseVector.x, lastMouseVector.x);
const x_max = Math.max(firstMouseVector.x, lastMouseVector.x);
const y_min = Math.min(firstMouseVector.y, lastMouseVector.y);
const y_max = Math.max(firstMouseVector.y, lastMouseVector.y);

// Send the upper-left and lower-right corners of the box.
const screenBoxList: [number, number][] = [
[x_min, y_min],
[x_max, y_max],
];

sendClickThrottled({
type: "ScenePointerMessage",
event_type: "rect-select",
ray_origin: null,
ray_direction: null,
screen_pos: screenBoxList,
});
}

// Release drag lock.
pointerInfo.isDragging = false;
}}
>
{inView ? null : <DisableRender />}
{children}
<BackgroundImage />
<AdaptiveDpr />
<SceneContextSetter />
{memoizedCameraControls}
<SplatRenderContext>
<SceneNodeThreeObject name="" parent={null} />
</SplatRenderContext>
<DefaultLights />
</Canvas>
</div>
);
}

Expand Down Expand Up @@ -513,10 +539,10 @@ function AdaptiveDpr() {
const setDpr = useThree((state) => state.setDpr);
return (
<PerformanceMonitor
factor={1.0}
factor={0.6}
ms={100}
iterations={5}
step={0.1}
step={0.2}
bounds={(refreshrate) => {
const max = Math.min(refreshrate * 0.75, 85);
const min = Math.max(max * 0.5, 38);
Expand Down
Loading

0 comments on commit 76f1ebf

Please sign in to comment.