Position a 3D dress model onto the center of left and right shoulder key points detected from the pose model.

Hi, I detect human body pose from a webcam using TensorFlow pose. I want to place my 3d dress in the center of the left and right shoulder. I’m converting the pixel X Y coordinates to 3D coordinates but the 3d dress in not positioned at the exact location.

My calculations:

let x_avg = (leftShoulderX + rightShoulderX) / 2

let y_avg = (leftShoulderY + rightShoulderY) / 2

mapWebcamTo3D(x_avg,y_avg,0)

function mapWebcamTo3D(webcamX, webcamY, depth) {

const videoFrameWidth = webcamRef.current.video.videoWidth;

const videoFrameHeight= webcamRef.current.video.videoHeight;

const desired3DWidth = 100; // The width of your 3D space in your desired unit (e.g., meters)

// Define the desired 3D space dimensions

const desired3DHeight = (desired3DWidth / videoFrameWidth) * videoFrameHeight;

// Calculate the scale factors for X and Y

const scaleX = desired3DWidth / videoFrameWidth;

const scaleY = desired3DHeight / videoFrameHeight;

// Calculate the 3D coordinates

const x3D = (webcamX - videoFrameWidth / 2) * scaleX;

const y3D = (videoFrameHeight / 2 - webcamY) * scaleY;

const z3D = depth;

}

My 3d Model:

function Model({…props}) {

const group = useRef()

const { nodes, materials } = useGLTF(‘/Red_Blouse_GLTF.txt’)

console.log("model: ", props.dimension.x, props.dimension.y )

return (

<group ref={group} {…props} dispose={null} scale={10} position={[props.dimension.x, -props.dimension.y, 0]} >

// //

}

<Webcam

ref={webcamRef}

style={{

position: “absolute”,

marginLeft: “auto”,

marginRight: “auto”,

left: 0,

right: 0,

textAlign: “center”,

zindex: 9,

width: 640,

height: 480,

}}

/>

<Model dimension={{x:XAvg, y:YAvg }}/>