Not sure if any help, but I managed to retarget some mixamo clips using the following. It was a while ago, but sharing in case it helps at all. This is for VRM characters encoded in GLB, so it uses some VRM specific data structures (extensions). It was also me hacking various approaches until it did something useful - not pretty code!!! There are things like the older VRM had characters standing backwards, but VRM1.0 had them right way around. But I would love to have a more standard and reusable way of doing this, so sharing in case any assistance to the efforts! (There are a few URLs in the code for sources that bits of the code were derived from as well.)
import * as THREE from 'three';
import { FBXLoader } from 'three/addons/loaders/FBXLoader.js';
import { FacialAnimationClip } from './FacialAnimationClip';
// https://pixiv.github.io/three-vrm/packages/three-vrm/examples/humanoidAnimation/mixamoVRMRigMap.js
/**
* A map from Mixamo rig name to VRM Humanoid bone name
*/
const mixamoVRMRigMap = {
mixamorigHips: "hips",
mixamorigSpine: "spine",
mixamorigSpine1: "chest",
mixamorigSpine2: "upperChest",
mixamorigNeck: "neck",
mixamorigHead: "head",
mixamorigLeftShoulder: "leftShoulder",
mixamorigLeftArm: "leftUpperArm",
mixamorigLeftForeArm: "leftLowerArm",
mixamorigLeftHand: "leftHand",
mixamorigLeftHandThumb1: "leftThumbMetacarpal",
mixamorigLeftHandThumb2: "leftThumbProximal",
mixamorigLeftHandThumb3: "leftThumbDistal",
mixamorigLeftHandIndex1: "leftIndexProximal",
mixamorigLeftHandIndex2: "leftIndexIntermediate",
mixamorigLeftHandIndex3: "leftIndexDistal",
mixamorigLeftHandMiddle1: "leftMiddleProximal",
mixamorigLeftHandMiddle2: "leftMiddleIntermediate",
mixamorigLeftHandMiddle3: "leftMiddleDistal",
mixamorigLeftHandRing1: "leftRingProximal",
mixamorigLeftHandRing2: "leftRingIntermediate",
mixamorigLeftHandRing3: "leftRingDistal",
mixamorigLeftHandPinky1: "leftLittleProximal",
mixamorigLeftHandPinky2: "leftLittleIntermediate",
mixamorigLeftHandPinky3: "leftLittleDistal",
mixamorigRightShoulder: "rightShoulder",
mixamorigRightArm: "rightUpperArm",
mixamorigRightForeArm: "rightLowerArm",
mixamorigRightHand: "rightHand",
mixamorigRightHandPinky1: "rightLittleProximal",
mixamorigRightHandPinky2: "rightLittleIntermediate",
mixamorigRightHandPinky3: "rightLittleDistal",
mixamorigRightHandRing1: "rightRingProximal",
mixamorigRightHandRing2: "rightRingIntermediate",
mixamorigRightHandRing3: "rightRingDistal",
mixamorigRightHandMiddle1: "rightMiddleProximal",
mixamorigRightHandMiddle2: "rightMiddleIntermediate",
mixamorigRightHandMiddle3: "rightMiddleDistal",
mixamorigRightHandIndex1: "rightIndexProximal",
mixamorigRightHandIndex2: "rightIndexIntermediate",
mixamorigRightHandIndex3: "rightIndexDistal",
mixamorigRightHandThumb1: "rightThumbMetacarpal",
mixamorigRightHandThumb2: "rightThumbProximal",
mixamorigRightHandThumb3: "rightThumbDistal",
mixamorigLeftUpLeg: "leftUpperLeg",
mixamorigLeftLeg: "leftLowerLeg",
mixamorigLeftFoot: "leftFoot",
mixamorigLeftToeBase: "leftToes",
mixamorigRightUpLeg: "rightUpperLeg",
mixamorigRightLeg: "rightLowerLeg",
mixamorigRightFoot: "rightFoot",
mixamorigRightToeBase: "rightToes",
};
export function convertFbxToVrmAnimation(animationGlb, characterGlb) {
console.log("convertFbxToVrmAnimation", animationGlb, characterGlb);
const clip = animationGlb.animations[0];
if (clip instanceof FacialAnimationClip) {
clip.setTargetCharacter(characterGlb);
return clip.toAnimationClip("facial_animation");
}
const vrm = characterGlb.vrm;
const asset = animationGlb.scene;
const tracks = []; // KeyframeTracks compatible with VRM will be added here
// TODO: At the moment, the following only supports mixamo clips.
if (asset.getObjectByName("mixamorigHips")) {
const restRotationInverse = new THREE.Quaternion();
const parentRestWorldRotation = new THREE.Quaternion();
const _quatA = new THREE.Quaternion();
const _quatB = new THREE.Quaternion();
const _vec3 = new THREE.Vector3();
// Adjust with reference to hips height.
const motionHipsHeight = asset.getObjectByName("mixamorigHips").position.y;
const vrmHipsY = vrm.humanoid?.getNormalizedBoneNode("hips").getWorldPosition(_vec3).y;
const vrmRootY = vrm.scene.getWorldPosition(_vec3).y;
const vrmHipsHeight = Math.abs(vrmHipsY - vrmRootY);
const hipsPositionScale = vrmHipsHeight / motionHipsHeight;
const n = asset.getObjectByName('mixamorigHips');
console.log("HIPS AT START", n.rotation, n.position);
clip.tracks.forEach((track) => {
// Convert each tracks for VRM use, and push to `tracks`
const trackSplitted = track.name.split(".");
const mixamoRigName = trackSplitted[0];
const vrmBoneName = mixamoVRMRigMap[mixamoRigName];
const vrmNormalizedNode = vrm.humanoid?.getNormalizedBoneNode(vrmBoneName);
const vrmNormalizedNodeName = vrmNormalizedNode?.name;
const vrmRawNode = vrm.humanoid?.getRawBoneNode(vrmBoneName);
const vrmRawNodeName = vrmRawNode?.name;
const mixamoRigNode = asset.getObjectByName(mixamoRigName);
if (vrmNormalizedNodeName != null) {
const propertyName = trackSplitted[1];
// Store rotations of rest-pose.
mixamoRigNode.getWorldQuaternion(restRotationInverse).invert();
mixamoRigNode.parent.getWorldQuaternion(parentRestWorldRotation);
if (track instanceof THREE.QuaternionKeyframeTrack) {
const newTrackValues = new Float32Array(track.values.length);
// Retarget rotation of mixamoRig to NormalizedBone.
for (let i = 0; i < track.values.length; i += 4) {
_quatA.fromArray(track.values, i);
// World rotation when resting of parent * Track rotation * Inverse of world rotation when resting
_quatA
.premultiply(parentRestWorldRotation)
.multiply(restRotationInverse);
// Apply normal bone rotation, then copy the raw bone rotation.
// See update() in https://github.com/pixiv/three-vrm/blob/15bf45f3399b6204db42601786c0612f13e985e3/packages/three-vrm-core/src/humanoid/VRMHumanoidRig.ts
// Get VRMHumanoidRig
const hr = vrm.humanoid._normalizedHumanBones;
const parentWorldRotation = hr._parentWorldRotations[vrmBoneName];
const invParentWorldRotation = _quatB.copy(parentWorldRotation).invert();
const boneRotation = hr._boneRotations[vrmBoneName];
_quatA
.multiply(parentWorldRotation)
.premultiply(invParentWorldRotation)
.multiply(boneRotation);
//// Move the mass center of the VRM
//if (vrmBoneName === 'hips') {
// const _boneWorldPos = new THREE.Vector3();
// const boneWorldPosition = vrmNormalizedNode.getWorldPosition(_boneWorldPos);
// vrmRawNode.parent.updateWorldMatrix(true, false);
// const parentWorldMatrix = vrmRawNode.parent.matrixWorld;
// const localPosition = boneWorldPosition.applyMatrix4(parentWorldMatrix.invert());
// vrmRawNode.position.copy(localPosition);
//}
// Copy into the new array (to not mess up the original).
_quatA.toArray(newTrackValues, i);
}
tracks.push(
new THREE.QuaternionKeyframeTrack(
`${vrmRawNodeName}.${propertyName}`,
track.times,
newTrackValues.map((v, i) =>
vrm.meta?.metaVersion === "0" && i % 2 === 0 ? -v : v
)
)
);
} else if (track instanceof THREE.VectorKeyframeTrack) {
const value = track.values.map(
(v, i) =>
(vrm.meta?.metaVersion === "0" && i % 3 !== 1 ? -v : v) *
hipsPositionScale
);
tracks.push(
new THREE.VectorKeyframeTrack(
`${vrmRawNodeName}.${propertyName}`,
track.times,
value
)
);
}
}
});
console.log("HIPS AT END", n.rotation, n.position);
}
//console.log(tracks)
const newClip = new THREE.AnimationClip("vrmAnimation", clip.duration, tracks);
newClip.retargetedFor = characterGlb;
console.log("Final retargeted clip", newClip);
return newClip;
}