Proposal: Post-instanceMatrix vertex transformation hook for InstancedMesh / BatchMesh

Resources
1

Hi Three.js team,

I’m new to the forum and I don’t know how to create a PR on GitHub, sorry.

I’d like to propose a feature to improve vertex transformations for InstancedMesh and BatchMesh in TSL / WebGPU.

Context

Currently in Three.js:

  • positionLocal is transformed by instanceMatrix inside InstanceNode.

  • Hooks like vertexNode and positionNode run before the instance matrix.

  • This makes it impossible to apply transformations that depend on vertex positions (wind, displacement, animations, global effects) in world space consistently across all instances.

  • Shadows and lights work correctly only in local / geometry space, not in world space.

Semantic Issue

Currently, in InstanceNode we have:

const instancePosition = instanceMatrixNode.mul(positionLocal).xyz
positionLocal.assign(instancePosition)

This is semantically confusing because positionLocal now actually contains the world-space position of the vertex, yet it is named positionLocal. Conceptually, This could instead be attributed to positionWorld or an intermediate position. This naming mismatch can mislead developers when writing vertex hooks or post-instance transformations.

Proposal

  1. Expose the instanceMatrixNode in both InstancedMesh and BatchMesh.

  2. Provide post-instanceMatrix hooks in the material, e.g.:

THREE.InstanceNode.prototype.setup=function(builder){
  const {instanceMatrix,instanceColor}=this
  let {instanceMatrixNode,instanceColorNode}=this
  const {count}=instanceMatrix,
        material=builder.object.material // < added
  if(instanceMatrixNode===null){
    if(count<=1000){instanceMatrixNode=buffer(instanceMatrix.array,'mat4',Math.max(count,1)).element(instanceIndex)}
    else{
      const buffer=new THREE.InstancedInterleavedBuffer(instanceMatrix.array,16,1)
      this.buffer=buffer
      const bufferFn=instanceMatrix.usage===THREE.DynamicDrawUsage?instancedDynamicBufferAttribute:instancedBufferAttribute,
            instanceBuffers=[bufferFn(buffer,'vec4',16,0),bufferFn(buffer,'vec4',16,4),bufferFn(buffer,'vec4',16,8),bufferFn(buffer,'vec4',16,12)]
      instanceMatrixNode=mat4(...instanceBuffers)}
    this.instanceMatrixNode=instanceMatrixNode
  }
  builder.instanceMatrixNode=instanceMatrixNode // < added

  if(instanceColor&&instanceColorNode===null){
    const buffer=new THREE.InstancedBufferAttribute(instanceColor.array,3),
          bufferFn=instanceColor.usage===THREE.DynamicDrawUsage?instancedDynamicBufferAttribute:instancedBufferAttribute
    this.bufferColor=buffer
    instanceColorNode=vec3(bufferFn(buffer,'vec3',3,0))
    this.instanceColorNode=instanceColorNode
  }

  const instancePosition=instanceMatrixNode.mul(positionLocal).xyz
  positionLocal.assign(instancePosition)

  // Post-instanceMatrix hooks
  if(material.afterInstancePositionNode){positionLocal.assign(material.afterInstancePositionNode)}
  if(builder.hasGeometryAttribute('normal')){
    const instanceNormal=transformNormal(normalLocal,instanceMatrixNode)
    normalLocal.assign(instanceNormal)
    if(material.afterInstanceNormalNode){normalLocal.assign(material.afterInstanceNormalNode)}
  }
  if(this.instanceColorNode!==null){varyingProperty('vec3','vInstanceColor').assign(this.instanceColorNode)}
}

THREE.BatchNode.prototype.setup=function(builder){
  const material=builder.object.material // < added
  if(this.batchingIdNode===null){
    if(builder.getDrawIndex()===null){this.batchingIdNode=instanceIndex}
    else{this.batchingIdNode=drawIndex}}
  const getIndirectIndex=Fn(([id])=>{
    const size=int(textureSize(textureLoad(this.batchMesh._indirectTexture),0).x),
          x=int(id).mod(size),
          y=int(id).div(size)
    return textureLoad(this.batchMesh._indirectTexture,ivec2(x,y)).x
  }).setLayout({name:'getIndirectIndex',type:'uint',inputs:[{name:'id',type:'int'}]})
  const indirectId=getIndirectIndex(int(this.batchingIdNode)),
        matricesTexture=this.batchMesh._matricesTexture,
        size=int(textureSize(textureLoad(matricesTexture),0).x),
        j=float(indirectId).mul(4).toInt().toVar(),
        x=j.mod(size),
        y=j.div(size),
        batchingMatrix=mat4(
          textureLoad(matricesTexture,ivec2(x,y)),
          textureLoad(matricesTexture,ivec2(x.add(1),y)),
          textureLoad(matricesTexture,ivec2(x.add(2),y)),
          textureLoad(matricesTexture,ivec2(x.add(3),y))
        ),
        colorsTexture=this.batchMesh._colorsTexture

  this.instanceMatrixNode=batchingMatrix // added
  builder.instanceMatrixNode=batchingMatrix // added

  if(colorsTexture!==null){
    const getBatchingColor=Fn(([id])=>{
      const size=int(textureSize(textureLoad(colorsTexture),0).x),
            j=id,
            x=j.mod(size),
            y=j.div(size)
      return textureLoad(colorsTexture,ivec2(x,y)).rgb
    }).setLayout({name:'getBatchingColor',type:'vec3',inputs:[{ name:'id',type:'int'}]})
    const color=getBatchingColor(indirectId)
    varyingProperty('vec3','vBatchColor').assign(color)
  }

  const bm=mat3(batchingMatrix)
  positionLocal.assign(batchingMatrix.mul(positionLocal))
  if(material.afterInstancePositionNode){positionLocal.assign(material.afterInstancePositionNode)}

  const transformedNormal=normalLocal.div(vec3(bm[0].dot(bm[0]),bm[1].dot(bm[1]),bm[2].dot(bm[2]))),
        batchingNormal=bm.mul(transformedNormal).xyz
  normalLocal.assign(batchingNormal)
  if(material.afterInstanceNormalNode){normalLocal.assign(material.afterInstanceNormalNode)}

  if(builder.hasGeometryAttribute('tangent')){
    tangentLocal.mulAssign(bm)
    if(material.afterInstanceTangentLocalNode){tangentLocal.assign(material.afterInstanceTangentLocalNode)}
  }
}
  1. Provide a helper to retrieve the instance matrix node in both cases:
const getInstanceMatrix=/*#__PURE__*/Fn((builder)=>{return builder.instanceMatrixNode})()
  1. Examples of use:
myMaterial.afterInstancePositionNode=/*#__PURE__*/Fn(()=>{
  const instancedPosition=positionLocal.toVar()
  instancedPosition.addAssign(someCustomOffsetPositionNode)
  return instancedPosition
})()

myMaterial.vertexNode=/*#__PURE__*/Fn(()=>{
  positionLocal.assign(getInstanceMatrix.mul(positionLocal).xyz)
  positionWorld.assign(modelWorldMatrix.mul(positionLocal).xyz)
  // or positionWorld.assign(positionLocal) if modelWorldMatrix is by default
  positionView.assign(modelViewMatrix.mul(positionWorld).xyz)
  modelViewProjection.assign(cameraProjectionMatrix.mul(positionView))
  return modelViewProjection
})()
// be careful, vertexNode is actually not compatible with shadows pipeline
  1. Example modifications for InstanceNode and BatchNode are include

Testing

  • I tested this in a complex animated scene with multiple lights, instanced and batched meshes (no skinning or morph targets yet).

  • Shadows and lights remain coherent.

  • Global post-instanceMatrix vertex transformations (wind, displacement, etc.) are now possible in world space.

  • The node getInstanceMatrix works equally well with InstancedMesh and BatchedMesh, without changing material,therefore, we can batch several instances of instancedMesh with the same material.
    I use it extensively in my project in an automated way. This allowed me to batch dozens of different trees with thousands of instances, and with consistent wind in the foliage regardless of their rotation, position, and scale. It works well with lights and shadows.

Suggestion

Consider adding official afterInstancePositionNode / afterInstanceNormalNode / afterInstanceTangentLocalNode hooks, and clarify the distinction between positionLocal and positionWorld for instance transformations, to allow safe and consistent post-instanceMatrix operations.

I understand that positionWorld is only truly global after being multiplied by modelWorldMatrix, but in the case of instances and batching, this multiplication is very rarely used. Therefore, we have a possible intermediate node `instancedPosition` after positionLocal and before positionWorld.

2

Note that currently, the shadows pipeline does not support modifying positionLocal in vertexNode:

vertexNode=/*#__PURE__*/Fn(()=>{
positionLocal.xz.addAssign(someOffset)
positionLocal.y.assign(someValue)
positionWorld.assign(positionLocal)
positionView.assign(modelViewMatrix.mul(positionLocal).xyz)
modelViewProjection.assign(cameraProjectionMatrix.mul(positionView))
return modelViewProjection})()

Has no effect on shadows.

3

Another example of the getInstanceMatrix method’s usefulness, helpful for correcting UVs on instances with a stretched scale, when applying UVs to the geometry:

// get pseudo-triplanar uv from position and normal
const makePseudoTriplanarUV=/#PURE/Fn(([position,normal,scale])=>{
const pXZ=position.xz.mul(vec2(scale.x,scale.z)),
pZY=position.zy.mul(vec2(scale.z,scale.y)),
pXY=position.xy.mul(vec2(scale.x,scale.y)),
ax=abs(normal.x),
ay=abs(normal.y),
zyxy=select(ax.greaterThanEqual(.5),pZY,pXY)
return select(ay.greaterThanEqual(.5),pXZ,zyxy)})

// get the instance scale from instance matrix
const getInstanceScale=vec3(length(getInstanceMatrix[0].xyz),length(getInstanceMatrix[1].xyz),length(getInstanceMatrix[2].xyz))

// get the object's scale from matrixWorld
const getMatrixWorldScale=vec3(length(modelWorldMatrix[0].xyz),length(modelWorldMatrix[1].xyz),length(modelWorldMatrix[2].xyz))

// get pseudo-triplanar uv from geometry normal according to the instance scale
const uvFromScaledGeometryNormal=makePseudoTriplanarUV(positionGeometry,normalGeometry,getInstanceScale).toVertexStage() // or getMatrixWorldScale.mul(getInstanceScale) if you need object's matrix world

// get pseudo-triplanar uv from world normal according to object's matrixWorld, if it's not by default (otherwise, getMatrixWorldScale could be replaced by vec3(1))
const uvFromScaledWorldNormal=makeTriplanarUV(positionWorld,normalWorldGeometry,getMatrixWorldScale)

// get pseudo-triplanar uv from geometry normal
const uvFromGeometryNormal=makeTriplanarUV(positionGeometry,normalGeometry,vec3(1))

// get pseudo-triplanar uv from world normal
const uvFromWorldNormal=makeTriplanarUV(positionWorld,normalWorldGeometry,vec3(1))

// example : apply regular uv from positionGeometry and normalGeometry in colorNode or outputNode
myUV.assign(uvFromScaledGeometryNormal)

The method uvFromScaledGeometryNormal allowed me to apply a repeating texture to an instanced wooden frame mesh while maintaining a regular UV scale along the xyz axes, even when the post is longer or shorter or wider.

This logic can also be applied to a custom real triplanarTexture node, if you have a geometry more complex than a simple cube.

I hope this can help other users.

4

I agree that this is a pain point.

But just nitpicking:

This is semantically confusing because positionLocal now actually contains the world-space position of the vertex, yet it is named positionLocal .

It is still in the local space relative of the InstancedMesh itself here, no?
i.e. It will only be in worldspace after going through modelViewMatrix of the instancedMesh?

5

“It is still in the local space relative of the InstancedMesh itself here, no?”

Yes, you’re right, and it’s not important; I’m the one who’s confused.

The main problem is controlling the nodes returned after instanceNode or batchNode.

Last night, I noticed that I was also missing a check on vInstanceColor after InstanceNode, which works fine with vertexNode but not with positionNode.

So I had to add the following to my hook, at the end of InstanceNode:

if(this.instanceColorNode!==null){
  const vColor=varyingProperty(‘vec3’,‘vInstanceColor’)
  vColor.assign(this.instanceColorNode)
  if(material.afterInstanceColorNode){
   vColor.assign(material.afterInstanceColorNode)
 }}

Multiplying conditions like afterInstancePositionNode, afterInstanceNormalNode, and afterInstanceColorNode is not practical at all.

Furthermore, my hack `material=builder.object.material` (I should have written builder.material, sorry) is absurd. I’m not sure InstancedMeshNode is supposed to know the material; it’s better to let it deal only with the attributes.

The ideal solution would be a stack that is independent of positionNode and InstanceNode but remains compatible with vertexNode (in setupVertex or setupPosition?); like this:

if ( this.afterInstanceNode !== null ) {
  positionLocal.assign( this.afterInstanceNode ) ;
}

We could then modify positionLocal, normalLocal, tangentLocal, vInstanceColor, and vBatchColor while maintaining consistency with lighting and shadows.

I don’t quite understand why instanceMatrixNode wasn’t exposed in the builder; it would be so convenient.

Finally, I think it would have been better to name vBatchColor as vInstanceColor. The same applies to batchingMatrix and instanceMatrixNode. I understand that semantically they’re not the same, but after all, vBatchColor and batchingMatrix are relatives to instances, if we consider that batching produces instances. This would improve material compatibility for InstancedMesh batching and wouldn’t break any of the engine’s logic.

6

Yeah. I think the webGPU stuff is still very WIP… so there are going to be rough edges and pain points.

I can’t speak to your specific issue personally.. but I’d remind you that “wouldn’t break any of the engine’s logic” might not be as simple as it sounds.

For instance shadowmapping need to be able to override certain parts of a shader to construct a shader capable of rendering to the shadowmap without incurring the cost of a full render (while then discarding the color and only using the depth).

Not saying you’re wrong, just that there are non-obvious complexities.. but, since the API’s are still in flux, this is probably a good time to raise your concerns and possibly help guide the direction of the API.
If you can boil this down to a specific, easily digestible pain point description, and propose a fix, it’s probably worth filing an issue on the threejs github and see if you can get some traction on it. If you do, feel free to link the issue back in this thread. I’d be interested in seeing it.

7

Yes, you’re right. And we also need to think about WebGL backward compatibility, which was never designed for that.

It’s great to see that there are such dedicated people willing to carefully answer questions from newcomers like me.
I’ll see soon if I can play around with GitHub and suggest a clearer improvement.

Thank you for your clear answers.

In the meantime, for those who might be interested, here’s a less disruptive temporary hack that works according to my latest tests, with lighting, shadows, reflection, refraction, fog, and postProcessing.

Custom InstanceNode, allow to get instanceMatrixNode and after-instance control:

THREE.InstanceNode.prototype.setup=function(builder){
const {instanceMatrix,instanceColor}=this
let {instanceMatrixNode,instanceColorNode}=this
const {count}=instanceMatrix
if(instanceMatrixNode===null){
if(count<=1000){instanceMatrixNode=buffer(instanceMatrix.array,'mat4',Math.max(count,1)).element(instanceIndex)}
else{
const buffer=new THREE.InstancedInterleavedBuffer(instanceMatrix.array,16,1)
this.buffer=buffer
const bufferFn=instanceMatrix.usage===THREE.DynamicDrawUsage?instancedDynamicBufferAttribute:instancedBufferAttribute,
instanceBuffers=[bufferFn(buffer,'vec4',16,0),bufferFn(buffer,'vec4',16,4),bufferFn(buffer,'vec4',16,8),bufferFn(buffer,'vec4',16,12)]
instanceMatrixNode=mat4(...instanceBuffers)}
this.instanceMatrixNode=instanceMatrixNode}

  // expose instanceMatrixNode in the builder
  builder.instanceMatrixNode=instanceMatrixNode

if(instanceColor&&instanceColorNode===null){
const buffer=new THREE.InstancedBufferAttribute(instanceColor.array,3),
bufferFn=instanceColor.usage===THREE.DynamicDrawUsage?instancedDynamicBufferAttribute:instancedBufferAttribute
this.bufferColor=buffer
instanceColorNode=vec3(bufferFn(buffer,'vec3',3,0))
this.instanceColorNode=instanceColorNode}
const instancePosition=instanceMatrixNode.mul(positionLocal).xyz
positionLocal.assign(instancePosition)
if(builder.hasGeometryAttribute('normal')){
const instanceNormal=transformNormal(normalLocal,instanceMatrixNode)
normalLocal.assign(instanceNormal)
if(this.instanceColorNode!==null){varyingProperty('vec3','vInstanceColor').assign(this.instanceColorNode)}

  // add a new stuff after instance work
  if(builder.material.instancePositionNode){
  positionLocal.assign(builder.material.instancePositionNode)
  }

}

Custom BatchNode

THREE.BatchNode.prototype.setup=function(builder){
if(this.batchingIdNode===null){
if(builder.getDrawIndex()===null){this.batchingIdNode=instanceIndex}
else{this.batchingIdNode=drawIndex}}
const getIndirectIndex=Fn(([id])=>{
const size=int(textureSize(textureLoad(this.batchMesh._indirectTexture),0).x),
x=int(id).mod(size),
y=int(id).div(size)
return textureLoad(this.batchMesh._indirectTexture,ivec2(x,y)).x}).setLayout({name:'getIndirectIndex',type:'uint',inputs:[{name:'id',type:'int'}]})
const indirectId=getIndirectIndex(int(this.batchingIdNode)),
matricesTexture=this.batchMesh._matricesTexture,
size=int(textureSize(textureLoad(matricesTexture),0).x),
j=float(indirectId).mul(4).toInt().toVar(),
x=j.mod(size),
y=j.div(size),
batchingMatrix=mat4(
textureLoad(matricesTexture,ivec2(x,y)),
textureLoad(matricesTexture,ivec2(x.add(1),y)),
textureLoad(matricesTexture,ivec2(x.add(2),y)),
textureLoad(matricesTexture,ivec2(x.add(3),y))),
colorsTexture=this.batchMesh._colorsTexture

  // expose batchingMatrix as instanceMatrixNode for better compatibility
  this.instanceMatrixNode=batchingMatrix
  builder.instanceMatrixNode=batchingMatrix

if(colorsTexture!==null){
const getBatchingColor=Fn(([id])=>{
const size=int(textureSize(textureLoad(colorsTexture),0).x),
j=id,
x=j.mod(size),
y=j.div(size)
return textureLoad(colorsTexture,ivec2(x,y)).rgb}).setLayout({name:'getBatchingColor',type:'vec3',inputs:[{ name:'id',type:'int'}]})
const color=getBatchingColor(indirectId)
varyingProperty('vec3','vBatchColor').assign(color)

  // add vInstanceColor for the InstancedMesh / BatchedMesh compatibility
  varyingProperty('vec3','vInstanceColor').assign(color) // < added

}
const bm=mat3(batchingMatrix)
positionLocal.assign(batchingMatrix.mul(positionLocal))
const transformedNormal=normalLocal.div(vec3(bm[0].dot(bm[0]),bm[1].dot(bm[1]),bm[2].dot(bm[2]))),
batchingNormal=bm.mul(transformedNormal).xyz
normalLocal.assign(batchingNormal)
if(builder.hasGeometryAttribute('tangent')){tangentLocal.mulAssign(bm)}

  // add a new stuff after batching work
  if(builder.material.instancePositionNode){
  positionLocal.assign(builder.material.instancePositionNode)
  }

}

Example of use

myMaterial.instancePositionNode=/*#__PURE__*/Fn((builder)=>{

  const {object,instanceMatrixNode}=builder

  // get the local position transformed by instanceMatrixNode
  const pos=positionLocal.toVar()

  // some deal with the position in instance space
  pos.yz.addAssign(myWindNode.yz)

  // some deal with the normal in instance space
  normalLocal.assign(normalLocal.mul(myPerturbNormalNode).normalize())

  // some deal with the color after the instance color has been applyed
  if(object.isInstancedMesh){
  varyingProperty('vec3','vInstanceColor').mulAssign(myCustomColor)
  }
  else if(object.isBatchedMesh){
  varyingProperty('vec3','vBatchColor').mulAssign(myCustomColor)
  }

  // some deal with uv and the instance matrix
  const getInstanceScale=vec3(length(instanceMatrixNode[0].xyz),length(instanceMatrixNode[1].xyz),length(instanceMatrixNode[2].xyz))
  myCustomUV.assign(makeTriplanarUV(positionGeometry,normalGeometry,getInstanceScale))

  // return the transformed positionLocal
  return pos

})()

Hope this helps

8

I posted a new issue: