I’ve been digging into animation code. One thing I’m interested in is animation retargeting.
SkeletonUtils has two methods retarget() and retargetClip() that are for retarting a single pose or all poses in an animation respectively.
These functions appear to be not used in any examples, so what I’d like to do is make examples for them to show on three.js examples, but we need to iron out some issues:
I was not able to get good results retargeting between two different models. As a sanity check I decided to try retargeting between the same exact models, but the results were not as expected. You’d expect that if you retarget an animation from a model to another model loaded from the same GLTF file that the result would be the exact same animation.
Here’s an HTML file you can copy to local to play with, showing that Michelle’s animation retargeted to a second Michelle model does not work as expected (it should work perfectly because there is no difference in the two models, right?):
webgl_animation_retargeting.html
<!DOCTYPE html>
<html lang="en">
<head>
<title>three.js webgl - animation - retargeting</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
<link type="text/css" rel="stylesheet" href="main.css">
<style>
body {
background-color: #bfe3dd;
color: #000;
}
a {
color: #2983ff;
}
</style>
</head>
<body>
<div id="container"></div>
<div id="info">
<a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> webgl - animation - keyframes<br/>
Model: <a href="https://artstation.com/artwork/1AGwX" target="_blank" rel="noopener">Littlest Tokyo</a> by
<a href="https://artstation.com/glenatron" target="_blank" rel="noopener">Glen Fox</a>, CC Attribution.
</div>
<!-- <script type="importmap">
{
"imports": {
"three": "../build/three.module.js",
"three/addons/": "./jsm/"
}
}
</script> -->
<script type="importmap">
{
"imports": {
"three": "https://unpkg.com/three@0.163.x/build/three.module.js",
"three/addons/": "https://unpkg.com/three@0.163.x/examples/jsm/"
}
}
</script>
<script type="module">
import * as THREE from 'three';
import Stats from 'three/addons/libs/stats.module.js';
import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
import { RoomEnvironment } from 'three/addons/environments/RoomEnvironment.js';
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
import { DRACOLoader } from 'three/addons/loaders/DRACOLoader.js';
import * as SkeletonUtils from 'three/addons/utils/SkeletonUtils.js';
const clock = new THREE.Clock();
const container = document.getElementById( 'container' );
const stats = new Stats();
container.appendChild( stats.dom );
const renderer = new THREE.WebGLRenderer( { antialias: true } );
renderer.setPixelRatio( window.devicePixelRatio );
renderer.setSize( window.innerWidth, window.innerHeight );
container.appendChild( renderer.domElement );
const pmremGenerator = new THREE.PMREMGenerator( renderer );
const scene = new THREE.Scene();
scene.background = new THREE.Color( 0xbfe3dd );
scene.environment = pmremGenerator.fromScene( new RoomEnvironment( renderer ), 0.04 ).texture;
const camera = new THREE.PerspectiveCamera( 40, window.innerWidth / window.innerHeight, 1, 100 );
camera.position.set( 5, 2, 8 );
const controls = new OrbitControls( camera, renderer.domElement );
controls.target.set( 0, 0.5, 0 );
controls.update();
controls.enablePan = true;
controls.enableDamping = true;
const [ sourceModel, targetModel, targetModel2 ] = await Promise.all([
new Promise( ( resolve, reject ) => {
// ours
// new GLTFLoader().load( 'https://assets.codepen.io/191583/Catwalk.gltf', resolve, undefined, reject );
// three's
new GLTFLoader().load( 'https://threejs.org/examples/models/gltf/Michelle.glb', resolve, undefined, reject );
} ),
new Promise( ( resolve, reject ) => {
// ours
new GLTFLoader().load( 'https://assets.codepen.io/191583/pete.gltf', resolve, undefined, reject );
// three's
// new GLTFLoader().load( 'https://threejs.org/examples/models/gltf/Soldier.glb', resolve, undefined, reject );
// new GLTFLoader().load( 'https://threejs.org/examples/models/gltf/Michelle.glb', resolve, undefined, reject );
} ),
// new Promise( ( resolve, reject ) => {
// new GLTFLoader().load( 'models/gltf/Michelle.glb', resolve, undefined, reject );
// } ),
]);
const playAnimations = true;
let targetMixer;
let sourceMixer;
let sourceClip
let sourceSkeletonHelper
let sourceSkeleton
handleSource(sourceModel)
// const target2Mixer = handlePete(targetModel)
const target2Mixer = handleMichelle(targetModel)
// const target2Mixer = handleSoldier(targetModel)
// handleManny()
function handleSource(sourceModel) {
console.log( 'source model:', sourceModel );
sourceClip = sourceModel.animations[ 0 ]
console.log( 'source clip:', sourceClip);
// sourceClip.tracks = sourceClip.tracks.filter(track => ['mixamorigHips', 'mixamorigSpine', 'mixamorigLeftArm'].some(name => track.name.startsWith(name)))
sourceModel.scene.position.x -= 1.5;
scene.add( sourceModel.scene );
const sourceSkin = sourceModel.scene.children[ 0 ].children[ 0 ];
sourceSkeletonHelper = new THREE.SkeletonHelper( sourceModel.scene );
scene.add( sourceSkeletonHelper );
sourceSkeleton = new THREE.Skeleton( sourceSkeletonHelper.bones );
console.log( sourceSkeleton );
sourceMixer = new THREE.AnimationMixer( sourceModel.scene );
sourceMixer.clipAction( sourceModel.animations[ 0 ] ).play();
}
function handlePete(targetModel) {
console.log( 'pete:', targetModel );
// remove pete's helmet, shield, and sword
targetModel.scene.remove( targetModel.scene.children[ 1 ] );
targetModel.scene.remove( targetModel.scene.children[ 1 ] );
targetModel.scene.remove( targetModel.scene.children[ 1 ] );
targetModel.scene.position.x += 1.5;
scene.add( targetModel.scene );
const targetSkin = targetModel.scene.children[ 0 ].children[ 0 ];
const targetSkelHelper = new THREE.SkeletonHelper( targetModel.scene );
scene.add( targetSkelHelper );
const retargetOptions = {
// specify the name of the source's hip bone.
hip: 'mixamorigHips',
// Map of target's bone names to source's bone names
names: {
hips: 'mixamorigHips',
spine: 'mixamorigSpine',
chest: 'mixamorigSpine2',
head: 'mixamorigHead',
upperarml: 'mixamorigLeftArm',
upperarmr: 'mixamorigRightArm',
lowerarml: 'mixamorigLeftForeArm',
lowerarmr: 'mixamorigRightForeArm',
upperlegl: 'mixamorigLeftUpLeg',
upperlegr: 'mixamorigRightUpLeg',
lowerlegl: 'mixamorigLeftLeg',
lowerlegr: 'mixamorigRightLeg',
footl: 'mixamorigLeftFoot',
footr: 'mixamorigRightFoot',
}
};
// const newNames = {}
// for (const name in retargetOptions.names) {
// if (['mixamorigHips', 'mixamorigSpine', 'mixamorigLeftArm'].some(name => name.startsWith(name)))
// newNames[name] = retargetOptions.names[name]
// }
// retargetOptions.names = newNames
let mixer;
if ( playAnimations ) {
const retargetedClip = SkeletonUtils.retargetClip( targetSkin, sourceSkeleton, sourceClip, retargetOptions );
console.log( 'retargeted clip', retargetedClip );
// Apply the mixer directly to the SkinnedMesh, not any
// ancestor node, because that's what
// SkeletonUtils.retargetClip outputs the clip to be
// compatible with.
mixer = new THREE.AnimationMixer( targetSkin );
mixer.clipAction( retargetedClip ).play();
}
return mixer;
}
function handleMichelle(targetModel) {
console.log( 'michelle:', targetModel );
targetModel.scene.position.x += 0;
scene.add( targetModel.scene );
const targetSkin = targetModel.scene.children[ 0 ].children[ 0 ];
const retargetOptions = {
// preservePosition: false,
// preserveHipPosition: false,
// specify the name of the target's hip bone.
hip: 'mixamorigHips',
// Map of target's bone names to source's bone names
names: {
mixamorigHips: 'mixamorigHips',
mixamorigSpine: 'mixamorigSpine',
mixamorigSpine1: 'mixamorigSpine1',
mixamorigSpine2: 'mixamorigSpine2',
mixamorigNeck: 'mixamorigNeck',
mixamorigHead: 'mixamorigHead',
mixamorigHeadTop_End: 'mixamorigHeadTop_End',
mixamorigLeftShoulder: 'mixamorigLeftShoulder',
mixamorigRightShoulder: 'mixamorigRightShoulder',
mixamorigLeftArm: 'mixamorigLeftArm',
mixamorigRightArm: 'mixamorigRightArm',
mixamorigLeftForeArm: 'mixamorigLeftForeArm',
mixamorigRightForeArm: 'mixamorigRightForeArm',
mixamorigLeftHand: 'mixamorigLeftHand',
mixamorigRightHand: 'mixamorigRightHand',
mixamorigLeftUpLeg: 'mixamorigLeftUpLeg',
mixamorigRightUpLeg: 'mixamorigRightUpLeg',
mixamorigLeftLeg: 'mixamorigLeftLeg',
mixamorigRightLeg: 'mixamorigRightLeg',
mixamorigLeftFoot: 'mixamorigLeftFoot',
mixamorigRightFoot: 'mixamorigRightFoot',
mixamorigLeftToeBase: 'mixamorigLeftToeBase',
mixamorigRightToeBase: 'mixamorigRightToeBase',
mixamorigLeftToe_End: 'mixamorigLeftToe_End',
mixamorigRightToe_End: 'mixamorigRightToe_End',
}
};
// const newNames = {}
// for (const name in retargetOptions.names) {
// if (['mixamorigHips', 'mixamorigSpine', 'mixamorigLeftArm'].some(name => name.startsWith(name)))
// newNames[name] = retargetOptions.names[name]
// }
// retargetOptions.names = newNames
let mixer
if ( playAnimations ) {
const retargetedClip = SkeletonUtils.retargetClip( targetSkin, sourceSkeleton, sourceClip, retargetOptions );
console.log( 'retargeted clip 2', retargetedClip );
// Apply the mixer directly to the SkinnedMesh, not any
// ancestor node, because that's what
// SkeletonUtils.retargetClip outputs the clip to be
// compatible with.
mixer = new THREE.AnimationMixer( targetSkin );
mixer.clipAction( retargetedClip ).play();
}
return mixer;
}
function handleSoldier(targetModel) {
console.log( 'soldier:', targetModel );
targetModel.scene.position.x += 0;
scene.add( targetModel.scene );
const targetSkin = targetModel.scene.children[ 0 ].children[ 0 ];
console.log('skin:', targetSkin)
const retargetOptions = {
// specify the name of the target's hip bone.
hip: 'mixamorigHips',
// Map of target's bone names to source's bone names
names: {
mixamorigHips: 'mixamorigHips',
mixamorigSpine: 'mixamorigSpine',
mixamorigSpine2: 'mixamorigSpine2',
// mixamorigHead: 'mixamorigHead',
mixamorigLeftArm: 'mixamorigLeftArm',
// mixamorigRightArm: 'mixamorigRightArm',
// mixamorigLeftForeArm: 'mixamorigLeftForeArm',
// mixamorigRightForeArm: 'mixamorigRightForeArm',
// mixamorigLeftUpLeg: 'mixamorigLeftUpLeg',
// mixamorigRightUpLeg: 'mixamorigRightUpLeg',
// mixamorigLeftLeg: 'mixamorigLeftLeg',
// mixamorigRightLeg: 'mixamorigRightLeg',
// mixamorigLeftFoot: 'mixamorigLeftFoot',
// mixamorigRightFoot: 'mixamorigRightFoot',
}
};
let mixer
if ( playAnimations ) {
const retargetedClip = SkeletonUtils.retargetClip( targetSkin, sourceSkeleton, sourceClip, retargetOptions );
console.log( 'retargeted clip 2', retargetedClip );
// Apply the mixer directly to the SkinnedMesh, not any
// ancestor node, because that's what
// SkeletonUtils.retargetClip outputs the clip to be
// compatible with.
mixer = new THREE.AnimationMixer( targetSkin );
mixer.clipAction( retargetedClip ).play();
}
return mixer;
}
const skelHelper = new THREE.SkeletonHelper( scene );
scene.add( skelHelper );
renderer.setAnimationLoop( animate );
window.onresize = function () {
camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();
renderer.setSize( window.innerWidth, window.innerHeight );
};
function animate() {
const delta = clock.getDelta();
if ( playAnimations ) {
sourceMixer.update( delta );
targetMixer?.update( delta );
target2Mixer?.update( delta );
}
controls.update();
stats.update();
renderer.render( scene, camera );
}
</script>
</body>
</html>
Here’s a codepen with the same example:
Why does the retarget from one model to the same exact model fail? We expect the retargeted animation to be identical right?
If we turn off the preservePosition and preserveHipPosition options that are passed to retargetClip (line 224), the orientation is better but the skin gets distorted and the whole model moves more than expected:
If we try the Pete model, with and without those two options, we get similar results as the last two examples with Michelle:
Other people are having related issues with this too:
- SkeletonUtils.retarget() doesn't work with mixamoRig skeleton (inverted feet, backward hands)
- Reposition BVH animation
- How to solve the problem of skin distortion caused by different orientations of model bones during animation migration?
- How to solve the skin deformation problem caused by different skin weight directions during animation migration? (looks like a duplicate of the previous one)
- Applying animations to character models of differing height
This unofficial example by @shootTheLuck uses SkeletonUtils.retargetClip() and somehow has no issues. Why? What is different in that example compared to others? Is it because the retarget(Clip) function was design for something specific that has to do with BVH format? Do you have any idea @shootTheLuck?
