How to select object InstancedMesh

I’m synchronizing the Three and Cesium camera. At the same time I set a click event to select objects. The problem is that it works on regular Object3D, but not on InstancedMesh. What am I doing wrong? I need to select more elements of InstancedMesh to return their index.

class Texture extends CanvasTexture {
 constructor(images) {
    let canvas = document.createElement('canvas');
    const columns = 98;
    const imageWidth = 48;
    const imageHeight = 48;
    canvas.width = 48 * 98
    canvas.height = 48

    let ctx = canvas.getContext('2d');

    images.forEach((image, index) => {
      const x = (index % columns) * imageWidth;
      const y = Math.floor(index / columns) * imageHeight;
      ctx.drawImage(image, x, y, imageWidth, imageHeight);
    });

    super(canvas);
    this.colorSpace = "srgb";
 }
}

const imageUrls = getUrlsFromObject(_URLS)

async function loadAirImages(imageUrls) {
  const loader = new ImageLoader();
  const promises = imageUrls.map(url => {
    return new Promise((resolve, reject) => {
      loader.load(url, resolve, undefined, reject);
    });
  });
  return Promise.all(promises);
}
async function loadEthalonImages(imageUrls) {
  const loader = new ImageLoader();
  const promises = imageUrls.map(url => {
    return new Promise((resolve, reject) => {
      loader.load(url, resolve, undefined, reject);
    });
  });
  return Promise.all(promises);
}

const ThreeSceneComponent = () => {

    const cesium = useCesium();
    const threeContainer = document.getElementById("threeContainer");
    const threeScene = new Scene();
    const threeCamera = new PerspectiveCamera(75, window.innerWidth / window.innerHeight, 1, 55000000);
    const threeRenderer = new WebGLRenderer({ alpha: true, antialias: true, depth: false, powerPreference: "high-performance", precision: 'lowp'});
    threeRenderer.setPixelRatio(window.devicePixelRatio);
    const airGroup = new Group();
    const ethalonGroup = new Group();
    threeScene.add(airGroup)
    threeScene.add(ethalonGroup)
    const threeGltfLoader = new GLTFLoader();
    const threeImageLoader = new TextureLoader();
    threeRenderer.setSize(window.innerWidth, window.innerHeight);
    threeContainer?.appendChild(threeRenderer.domElement);
    let geometry = new PlaneGeometry(35, 35);
    let etGeometry = new PlaneGeometry(35, 35);
    let material = new MeshBasicMaterial();
    let etMaterial = new MeshBasicMaterial();
    const raycaster = new Raycaster();
    const mouse = new Vector2();
loadAirImages(imageUrls).then(images => {
  material.side = DoubleSide
  material.transparent = true
  material.map = new Texture(images);
  material.depthTest = false;
  material.depthWrite = false;
  material.onBeforeCompile = shader => {
    shader.uniforms.atlasSize = { value: new Vector2(98, 1) };
    shader.vertexShader = `
    uniform vec2 atlasSize;
    attribute float cameraHeight;
    attribute vec2 uvOffset;
    ${shader.vertexShader}
  `.replace(
    `#include <uv_vertex>`,
    `#include <uv_vertex>
    vMapUv = ( mapTransform * vec3( (MAP_UV + uvOffset) / atlasSize, 2 ) ).xy;
    `
  ).replace(
    `#include <begin_vertex>`,
    `#include <begin_vertex>
    transformed *= cameraHeight; // Масштабируем вершины на основе высоты камеры
    `
  );
  };
  // Обновляем материал, чтобы изменения вступили в силу
  material.needsUpdate = true;
})

loadEthalonImages(imageUrls).then(images => {
  etMaterial.side = DoubleSide
  etMaterial.transparent = true
  etMaterial.map = new Texture(images);
  etMaterial.depthTest = false;
  etMaterial.depthWrite = false;
  etMaterial.onBeforeCompile = shader => {
    shader.uniforms.atlasSize = { value: new Vector2(98, 1) };
    shader.vertexShader = `
    uniform vec2 atlasSize;
    attribute float cameraHeight;
    attribute vec2 uvOffset;
    ${shader.vertexShader}
  `.replace(
    `#include <uv_vertex>`,
    `#include <uv_vertex>
    vMapUv = ( mapTransform * vec3( (MAP_UV + uvOffset) / atlasSize, 2 ) ).xy;
    `
  ).replace(
    `#include <begin_vertex>`,
    `#include <begin_vertex>
    transformed *= cameraHeight; // Масштабируем вершины на основе высоты камеры
    `
  );
  };
  // Обновляем материал, чтобы изменения вступили в силу
  etMaterial.needsUpdate = true;
})

    const etInstanceMesh = new InstancedMesh(etGeometry, etMaterial, 10000);
    const aInstanceMesh = new InstancedMesh(geometry, material, 10000);
    const three = {
      air: {
        geometry: geometry,
        material: material,
        group: aGroup,
        mesh: aInstanceMesh,
      },
      ethalon: {
        geometry: etGeometry,
        material: etMaterial,
        group: etGroup,
        mesh: etInstanceMesh
      },
      core: {
        scene: threeScene,
        imageLoader: threeImageLoader,
        gltfLoader: threeGltfLoader,
        camera: threeCamera
      }
    }
  // <------------------------------------------------------------->

    aController.initThreeController(three)
    etController.initThreeController(three)

    const updateCameraHeightAttribute = (instanceMesh, height) => {
      const attributes = instanceMesh.geometry.attributes;
      if (attributes.cameraHeight) {
        attributes.cameraHeight.array.fill(height / 1000); 
        attributes.cameraHeight.needsUpdate = true; 
      } else {
        const cameraHeightArray = new Float32Array(instanceMesh.count).fill(height / 1000);
        instanceMesh.geometry.setAttribute('cameraHeight', new BufferAttribute(cameraHeightArray, 1));
      }
    };

    function updateThreeJS() {
        threeCamera.fov = Cesium.Math.toDegrees(cesium.viewer.camera.frustum.fovy);
        threeCamera.updateProjectionMatrix();
        const cesiumCamera = cesium.viewer.camera;
        const cvm = cesiumCamera.viewMatrix;
        const civm = cesiumCamera.inverseViewMatrix;
      
        const cameraPosition = Cartesian3.fromElements(civm[12], civm[13], civm[14]);
        const cameraDirection = new Cartesian3(-cvm[2], -cvm[6], -cvm[10]);
        const cameraUp = new Cartesian3(cvm[1], cvm[5], cvm[9]);
      
        const cameraPositionVec3 = new Vector3(cameraPosition.x, cameraPosition.y, cameraPosition.z);
        const cameraDirectionVec3 = new Vector3(cameraDirection.x, cameraDirection.y, cameraDirection.z);
        const cameraUpVec3 = new Vector3(cameraUp.x, cameraUp.y, cameraUp.z);
      
        threeCamera.position.copy(cameraPositionVec3);
        threeCamera.up.copy(cameraUpVec3);
        threeCamera.lookAt(cameraPositionVec3.clone().add(cameraDirectionVec3));
        const height = cesium.viewer.camera.positionCartographic.height;
        updateCameraHeightAttribute(etInstanceMesh, height);
        updateCameraHeightAttribute(aInstanceMesh, height)

        threeRenderer.render(threeScene, threeCamera);
      }
      window.addEventListener('resize', () => {
        const width = window.innerWidth;
        const height = window.innerHeight;
        threeRenderer.setSize(width, height);
        threeCamera.aspect = width / height;
        threeCamera.updateProjectionMatrix();
      });

      const onClick = (event) => {
        const rect = threeRenderer.domElement.getBoundingClientRect();
        mouse.x = ((event.clientX - rect.left) / rect.width) * 2 - 1;
        mouse.y = -((event.clientY - rect.top) / rect.height) * 2 + 1;
    
        raycaster.setFromCamera(mouse, threeCamera);
        const intersects = raycaster.intersectObject(threeScene.children[2], false);
        if (intersects.length > 0) {
          const instanceId = intersects[0].instanceId;
          console.log(`Clicked on instance ${instanceId}`);
        }
      };
      document.addEventListener('click', onClick);
      function renderLoop() {
        requestAnimationFrame(renderLoop);
        // cesium.viewer.render();
        if(three.core.scene.children.length > 3) {
          if(cesium.viewer.camera.positionCartographic.height < 1000000) {
            threeScene.children[0].visible = true
            threeScene.children[1].visible = true
            threeScene.children[2].visible = false
            threeScene.children[3].visible = false
          } else {
            threeScene.children[0].visible = false
            threeScene.children[1].visible = false
            threeScene.children[2].visible = true
            threeScene.children[3].visible = true
          }
        }
        updateThreeJS();
      }
      
      renderLoop();

      return <></>
};

export default ThreeSceneComponent;
  1. threeScene.children[2] don’t do this - relying on the order of children in parent is risky and incredibly easy to break by accident. Use scene.getObjectByName instead, for example.

  2. Can you create a codepen or codesandbox replicating the issue? The code looks ok, so it’s hard to debug just by reading it (there’s also a chance you’ll find the issue yourself when writing a smaller replicate of your code - isolation makes debugging way faster and simpler.)

I found out that it is because of scaling in the shader: transformed *= cameraHeight. When I remove this it is selected normally, but I need scaling constant, how can I be?

Either scale objects via instanceMatrix (raycast() operates with the data about vertices at the level of JS and knows nothing of what you do to vertices with shaders on GPU), or try to implement GPU picking :thinking: