I have calculated a midpoint between the right and left shoulder of the human body detected through a webcam using Posnet. the detected coordinates are 2D pixel coordinates I need to find the exact location of the 3d dress to place it on the center of the human body in the webcam. I am trying to use Raycaster but as a newbie, I am not sure if I am doing it right or not currently I have calculated a midpoint from left shoulder to right shoulder and now I want to detect at that 2d pixel point the exact 3d point where thay ray intersect.
code with three.js fiber
function Model({…props}) {
const group = useRef()
const { nodes, materials } = useGLTF(‘/Red_Blouse_GLTF.txt’)
const { camera,raycaster, scene} = useThree();
const dressModelRef = dressMeshRef
//Update the position of the dress model based on the centerPoint
useFrame(() => {
if (dressModelRef.current) {
//console.log(‘hello’)
// Create a raycaster
const rayOrigin = new THREE.Vector3(camera.x, camera.y, camera.z); // Update with your camera position
const rayDirection = new THREE.Vector3(props.dimension.x - rayOrigin.x, props.dimension.y - rayOrigin.y, 0.5 - rayOrigin.z).normalize();
console.log(rayOrigin,rayDirection,"hello ray")
raycaster.set(rayOrigin, rayDirection);
// Perform raycasting to find intersections with objects in the scene
const intersections = raycaster.intersectObject(dressModelRef.current);
console.log('hello 1',intersections)
// Check if there are intersections
if (intersections.length > 0) {
const intersectionPoint = intersections[0].point;
// Place the dress at the intersection point or perform other actions
// dressModelRef.current.position.set(intersectionPoint.x, intersectionPoint.y, intersectionPoint.z);
// dressModelRef.current.scale.set(intersectionPoint.y-2)
console.log("hello 2",intersectionPoint)
}
}
});
return (
<group ref={group} {...props} dispose={null} scale={10} position={[0, -13, 0]} >
<directionalLight intensity={1} decay={4} color="#cacaca" position={[1.5, 3, 1.5]} rotation={[-1.107, 0.421, 0]} />
<mesh geometry={nodes.Cloth_mesh.geometry} material={materials.Cotton_Oxford_FRONT_299550} ref={dressMeshRef} />
<mesh geometry={nodes.Cloth_mesh_1.geometry} material={materials.Cotton_Oxford_FRONT_299550} ref={dressMeshRef} />
<mesh geometry={nodes.Cloth_mesh_2.geometry} material={materials.Cotton_Oxford_FRONT_299550} ref={dressMeshRef} />
<mesh geometry={nodes.Cloth_mesh_3.geometry} material={materials.Cotton_Oxford_FRONT_299550} ref={dressMeshRef} />
<mesh geometry={nodes.Cloth_mesh_4.geometry} material={materials.Cotton_Oxford_FRONT_299550} ref={dressMeshRef} />
<mesh geometry={nodes.Cloth_mesh_5.geometry} material={materials.Cotton_Oxford_FRONT_299550} ref={dressMeshRef} />
<mesh geometry={nodes.Cloth_mesh_6.geometry} material={materials.Cotton_Oxford_FRONT_299550}ref={dressMeshRef} />
<mesh geometry={nodes.Cloth_mesh_7.geometry} material={materials.Cotton_Oxford_FRONT_299550}ref={dressMeshRef} />
<mesh geometry={nodes.Cloth_mesh_8.geometry} material={materials.Cotton_Oxford_FRONT_299550}ref={dressMeshRef} />
<mesh geometry={nodes.Trim.geometry} material={materials.Material6175164} position={[0.001, 1.606, -0.107]} rotation={[-2.676, 0.001, -3.101]} />
<mesh geometry={nodes.Trim_1.geometry} material={materials.Material7217358} position={[0.001, 1.583, -0.123]} rotation={[0.574, 0, 0]} scale={[0.776, 0.617, 0.725]} />
<mesh geometry={nodes.BindedTrim_306379.geometry} material={materials.Material299764} />
<mesh geometry={nodes.BindedTrim_306448.geometry} material={materials.Material299764} />
<mesh geometry={nodes.BindedTrim_306466.geometry} material={materials.Material300015} />
<mesh geometry={nodes.BindedTrim_306580.geometry} material={materials.Material299764} />
<mesh geometry={nodes.BindedTrim_306649.geometry} material={materials.Material299764} />
<mesh geometry={nodes.BindedTrim_306667.geometry} material={materials.Material300015} />
<mesh geometry={nodes.BindedTrim_306936.geometry} material={materials.Material299764} />
<mesh geometry={nodes.BindedTrim_307205.geometry} material={materials.Material299764} />
<mesh geometry={nodes.BindedTrim_307401.geometry} material={materials.Material299764} />
<mesh geometry={nodes.BindedTrim_307481.geometry} material={materials.Material299889} />
<mesh geometry={nodes.BindedTrim_307555.geometry} material={materials.Material299889} />
<mesh geometry={nodes.BindedTrim_307619.geometry} material={materials.Material299889} />
<mesh geometry={nodes.BindedTrim_307683.geometry} material={materials.Material299889} />
<mesh geometry={nodes.BindedTrim_307760.geometry} material={materials.Material299889} />
<mesh geometry={nodes.BindedTrim_307834.geometry} material={materials.Material299889} />
<mesh geometry={nodes.BindedTrim_307908.geometry} material={materials.Material299889} />
<mesh geometry={nodes.BindedTrim_307982.geometry} material={materials.Material299889} />
<mesh geometry={nodes.BindedTrim_308046.geometry} material={materials.Material299889} />
<mesh geometry={nodes.BindedTrim_308110.geometry} material={materials.Material299889} />
<mesh geometry={nodes.BindedTrim_308174.geometry} material={materials.Material299889} />
<mesh geometry={nodes.BindedTrim_308238.geometry} material={materials.Material299889} />
<mesh geometry={nodes.BindedTrim_308481.geometry} material={materials.Material299889} />
<mesh geometry={nodes.BindedTrim_308721.geometry} material={materials.Material299889} />
</group>
)
}
Output :
props.dimension.x 328.7492550092628);
props.dimension.Y 483.5726960433992;
Vector3 {x: 0, y: 0, z: 0} Vector3 {x: 0.5622160497318541, y: 0.8269899529355352, z: 0.0008550833821904983} ‘hello ray’
hello 1 [{…}]
hello 2 Vector3 {x: 1.42366045690514, y: 2.094128929285631, z: 0.0021652679591091808}
the return 3d vector is very small
Vector3 {x: 1.42366045690514, y: 2.094128929285631, z: 0.0021652679591091808}
my 3d dress position to center on the cam scale={10} position={[0, -13, 0]}