Drei Hub disappears upon entering R3F XR session

Questions
, , ,
1

I tried to use a simple Hub from Drei with an OrthographicCamera with XR. The Hub shows up fine before I enter the XR session, but immediately disappears after entering the XR session. Could anyone help me figure out what’s going on? Also, I’m pretty new to R3F, what is the easiest way to to diagnose things like this? Very much appreciated!!!

Here’s the relevant code:

import { Hud, OrthographicCamera } from '@react-three/drei';

export default function GUI() {
    return (
        <Hud renderPriority={1}>
            <OrthographicCamera makeDefault position={[0, 0, 100]} />
            <mesh>
                <boxGeometry args={[50, 50, 50]} />
            </mesh>
        </Hud>
    );
}

import { Canvas } from '@react-three/fiber';
import Splat from './components/Splat';
import { createXRStore, XR } from '@react-three/xr';
import Capture from './components/Capture';
import { setSplatViewer } from './util/splatManager';
import DevControls from './components/DevControls';

//@ts-ignore
import devSplat from './assets/cabin.ply';
import GUI from './components/GUI';

export default function App() {
    const store = createXRStore({ emulate: false });

    const handleSplatReady = (viewer: any) => {
        setSplatViewer(viewer);
    };

    return (
        <>
            <div style={{ position: 'absolute', top: 0, left: 0, zIndex: 1000 }}>
                <button onClick={() => store.enterAR()}>Enter AR</button>
            </div>
            <Canvas>
                <DevControls />
                    <Splat file={devSplat} onSplatReady={handleSplatReady} />
                    <Capture />
                <XR store={store}>
                    <GUI />
                </XR>
            </Canvas>
        </>
    );
}
2 
import { OrthographicCamera } from '@react-three/drei';
import { createPortal, useFrame, useThree, type Camera } from '@react-three/fiber';
import { useEffect, useMemo, useRef, useState } from 'react';
import { Mesh, Scene, Object3D, Vector2 } from 'three';
import { forwardEvents, type GetCamera, type ForwardEventsOptions } from '@pmndrs/pointer-events';

function toCoords(e: unknown, target: Vector2): Vector2 {
    //@ts-ignore
    return target.set(e.pointer.x, e.pointer.y);
}

export function forwardEventsToPortal(
  fromObject: Object3D,
  getCamera: GetCamera,
  scene: Scene,
  options?: ForwardEventsOptions,
) {
  return forwardEvents(
    fromObject,
    getCamera,
    scene,
    toCoords as any,
    fromObject.setPointerCapture.bind(fromObject),
    fromObject.releasePointerCapture.bind(fromObject),
    options,
  );
}

type RenderHudProps = {
    defaultScene: THREE.Scene;
    defaultCamera: THREE.Camera;
    renderPriority?: number;
};

function RenderHud({ defaultScene, defaultCamera, renderPriority = 1 }: RenderHudProps) {
    const { gl, scene, camera } = useThree();

    useFrame(() => {
        const oldAutoClear = gl.autoClear;
        if (renderPriority === 1) {
            gl.autoClear = true;
            gl.render(defaultScene, defaultCamera);
        }
        gl.autoClear = false;
        gl.clearDepth();
        const oldXrEnabled = gl.xr.enabled;
        gl.xr.enabled = false;
        gl.render(scene, camera);
        gl.xr.enabled = oldXrEnabled;
        gl.autoClear = oldAutoClear;
    }, renderPriority);

    return <group onPointerOver={() => null} />;
}

export type HudProps = {
    children: React.ReactNode;
    renderPriority?: number;
};

export function XRHud({ children, renderPriority = 1 }: HudProps) {
    const { scene: defaultScene, camera: defaultCamera, size} = useThree();
    const { width, height } = size;
    const [hudScene] = useState(() => new Scene());

    const hudCamera = useRef<Camera>(null);
    const portalMesh = useRef<Mesh>(null);

    const update = useMemo(() => {
        const { update } = forwardEventsToPortal(
            defaultScene,
            () => hudCamera.current as Camera,
            hudScene,
        );
        return update;
    }, [defaultScene, hudCamera, hudScene]);

    useEffect(() => {
        if (portalMesh.current) {
            defaultCamera.add(portalMesh.current);
        }
    }, [portalMesh, defaultCamera]);

    useFrame(() => {
        update();
    });

    return (
        <>
            {createPortal(
                <>
                    {children}
                    <OrthographicCamera
                        ref={hudCamera as any}
                        makeDefault
                        left={-width / 2}
                        right={width / 2}
                        top={height / 2}
                        bottom={-height / 2}
                        near={0.1}
                        far={1000}
                    />
                    <RenderHud defaultScene={defaultScene} defaultCamera={defaultCamera} renderPriority={renderPriority} />
                </>,
                hudScene,
            )}

            {/*
                The portal mesh is used to capture pointer events and forward them to the HUD scene.
            */}
            <mesh ref={portalMesh} position={[0, 0, -defaultCamera.near - 0.001]}>
                <planeGeometry args={[100, 100]} />
                <meshBasicMaterial visible={false} />
            </mesh>
        </>
    );
}

This is what I ended up with, in case it helps someone.

1 Like
3

Mark yours as solution! :smiley:

1 Like
4

Just did, thanks! Some more info about this solution: the key to getting the portal rendering was disabling gl.xr.enabled before render, then re-enabling after. This is because there is custom logic in the three.js GL renderer class which messes with the render camera.

Events are also different in XR, so I needed to forward them the way that I did instead of using a custom compute function. Importing forwardEvents from @pmndrs/pointer-events requires patching the package because forwardEvents is not exported.

Hope this helps someone!

1 Like