Rotate object on drag (like orbitcontrol but only on specific object) react fiber

I try to have the ability to rotate the object on drag:

import React, { useRef, useState, useCallback, useEffect } from "react";
import { BoxGeometry, MeshBasicMaterial } from "three";
import { useTexture } from "@react-three/drei";

function Cube() {
  const meshRef = useRef();
  const [isMouseDown, setIsMouseDown] = useState(false);
  const [mouseCoords, setMouseCoords] = useState({ x: 0, y: 0 });
  const speed = 3;
  const texture = useTexture('http://i.imgur.com/CEGihbB.gif');
  const startRotation = useRef();

  useEffect(() => {
    startRotation.current = meshRef.current.rotation.clone();
  }, []);

  const handleRotation = useCallback((clientX, clientY) => {
    if (isMouseDown) {
      const temp_x = clientX - mouseCoords.x;
      const temp_y = clientY - mouseCoords.y;

      if (Math.abs(temp_y) < Math.abs(temp_x)) {
        meshRef.current.rotateY(temp_x * speed / 1000);
      } else {
        meshRef.current.rotateX(temp_y * speed / 1000);
      }

      setMouseCoords({ x: clientX, y: clientY });
    }
  }, [isMouseDown, mouseCoords.x, mouseCoords.y, speed]);

  return (
    <mesh
      ref={meshRef}
      onPointerDown={(event) => {
        setIsMouseDown(true);
        setMouseCoords({ x: event.clientX, y: event.clientY });
        event.stopPropagation();
        event.target.setPointerCapture(event.pointerId);
      }}
      onPointerMove={(event) => handleRotation(event.clientX, event.clientY)}
      onPointerUp={() => setIsMouseDown(false)}
      onPointerOut={() => setIsMouseDown(false)}
      geometry={new BoxGeometry(1, 1, 1)}
      material={new MeshBasicMaterial({ map: texture })}
    />
  );
}

export default Cube;

Strangely the onmousemove event stops after a second so that the box rotates just a tiny bit on every mousedown + drag. Drei has any helper functions but not this?

  1. Consider not updating state in onmousemove. Instead do the transforms outside of react scope - by assigning the mesh to useRef, and rotating it using just the ref - then commit the transformation only when the dragging finishes / in onpointerup event.
  2. The math part looks ok - try console logging the values of ex. rotation.x, temp_x, Math.abs(temp_y) < Math.abs(temp_x) and see if they look reasonable.