Hello to all,
I am relatively new to Three.js and still learning a lot of new things.
I have in a scene a plane with a texture on it, and raycast by clicking from the camera to the plane. Now I need the color at the point of the intersection. How do I get the texture color at the intersection, since raycasting only happens at geometry level?
This is… complicated. The raycast happens on the CPU - meaning that you don’t HAVE the texture information that the GPU does. But you could theoretically read the texture onto the CPU, then use the barycoords of the intersection point to find the UV of the texture, and use that. Or you could use something like three-mesh-bvh to do the raycast on the gpu, and then read it back to the CPU. Lots of options.
2 Likes
I would raycast to get the uv and then with a custom shader sample the texture at the uv and output that color to a 1px render target. From there you can copy the color back to the cpu to get the value. This also gets you get the gpu interpolated texture value, as well, without having to write that yourself.
Definitely a bit tricky but I think that’s how I’d do it.
2 Likes
Great, that has already brought my understanding a little further! With N8Threes answer I am a bit overwhelmed.
I think @gkjohnson and @Chaser_Code approaches are similar. I tried to implement these and this is what came out, which unfortunately doesn’t work. Does anyone have any ideas as to what is causing this?
I have written a minimal example here.
import {
WebGLRenderer,
PerspectiveCamera,
Scene,
Vector3,
Mesh,
MeshBasicMaterial,
TextureLoader,
PlaneGeometry,
Raycaster,
Vector2,
WebGLRenderTarget,
} from "three";
import { OrbitControls } from "three/examples/jsm/controls/OrbitControls";
let width = window.innerWidth,
height = window.innerHeight;
//initialize renderer
let renderer = new WebGLRenderer();
renderer.setSize(width, height);
document.body.appendChild(renderer.domElement);
//scene / raycaster / camera / controls
let scene = new Scene();
let rayCaster = new Raycaster();
let camera = new PerspectiveCamera(45, width / height, 0.1, 10000);
camera.position.copy(new Vector3(100, 100, 100));
camera.lookAt(new Vector3(0, 0, 0));
scene.add(camera);
let controls = new OrbitControls(camera, renderer.domElement);
// global variables
let mouse = new Vector2();
let planeMesh;
let textureData;
let renderTargetTexture;
let imgPath =
"https://cdn.pixabay.com/photo/2022/05/23/13/16/bird-7216181_1280.jpg";
//load texture and initialize planeMesh, textureData and renderTargetTexture
new TextureLoader().load(imgPath, function (texture) {
textureData = texture;
planeMesh = new Mesh(
new PlaneGeometry(texture.image.width, texture.image.height),
new MeshBasicMaterial({ map: texture })
);
scene.add(planeMesh);
renderTargetTexture = new WebGLRenderTarget(
texture.image.width,
texture.image.height
);
});
resize();
animate();
function resize() {
renderer.setSize(width, height);
camera.aspect = width / height;
camera.updateProjectionMatrix();
}
function animate() {
renderer.render(scene, camera);
requestAnimationFrame(animate);
controls.update();
}
window.addEventListener("resize", resize);
window.addEventListener("pointermove", function (event) {
mouse.x =
((event.clientX - renderer.domElement.offsetLeft) /
renderer.domElement.width) *
2 -
1;
mouse.y =
-(
(event.clientY - renderer.domElement.offsetTop) /
renderer.domElement.height
) *
2 +
1;
});
window.addEventListener("pointerup", (event) => {
rayCaster.setFromCamera(mouse, camera);
let intersections = rayCaster.intersectObject(planeMesh);
if (intersections.length > 0) {
// UV at plane-intersection-point
let uv = intersections[0].uv;
// from uv position to discrete Pixel coords
let x = Math.floor(textureData.image.width * uv.x);
let y = Math.floor(textureData.image.height * uv.y);
//buffer for pixel values -- Format RGBA
const read = new Uint8Array(4);
//load Texture into buffer
renderer.setRenderTarget(renderTargetTexture);
// read targetpixels from target Texture
renderer.readRenderTargetPixels(
renderTargetTexture,
x,
y,
textureData.image.width,
textureData.image.height,
read
);
// ---------------- E R R O R ----------------
// read is always [0,0,0,0]
console.log(read);
// set renderer to default canvas
renderer.setRenderTarget(null);
}
});
Tested at local server and THREE examples folder
<!DOCTYPE html>
<html lang="en">
<head>
<title>three.js webgl - cameras</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
</head>
<body>
<script type="importmap">
{
"imports": {
"three": "../build/three.module.js"
}
}
</script>
<script id="fragment_shader_screen" type="x-shader/x-fragment">
varying vec2 vUv;
uniform sampler2D tDiffuse;
void main() {
gl_FragColor = texture2D( tDiffuse, vUv );
}
</script>
<script id="vertexShader" type="x-shader/x-vertex">
varying vec2 vUv;
void main() {
vUv = uv;
gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );
}
</script>
<script type="module">
import * as THREE from 'three';
import { OrbitControls } from "./jsm/controls/OrbitControls.js";
let width = window.innerWidth,
height = window.innerHeight;
//initialize renderer
let renderer = new THREE.WebGLRenderer();
renderer.setSize(width, height);
renderer.autoClear = false;
document.body.appendChild(renderer.domElement);
//scene / raycaster / camera / controls
let scene = new THREE.Scene();
let cameraRTT = new THREE.OrthographicCamera( window.innerWidth / - 2, window.innerWidth / 2, window.innerHeight / 2, window.innerHeight / - 2, - 10000, 10000 );
cameraRTT.position.z = 100;
let sceneScreen = new THREE.Scene();
let renderTargetTexture = new THREE.WebGLRenderTarget(
window.innerWidth, window.innerHeight, { minFilter: THREE.LinearFilter, magFilter: THREE.NearestFilter, format: THREE.RGBAFormat, type: THREE.FloatType }
);
const materialScreen = new THREE.ShaderMaterial( {
uniforms: { 'tDiffuse': { value: renderTargetTexture.texture } },
vertexShader: document.getElementById( 'vertexShader' ).textContent,
fragmentShader: document.getElementById( 'fragment_shader_screen' ).textContent,
depthWrite: false
} );
const plane = new THREE.PlaneGeometry( window.innerWidth, window.innerHeight );
let quad = new THREE.Mesh(plane,materialScreen );
quad.position.z = - 100;
sceneScreen.add( quad );
let rayCaster = new THREE.Raycaster();
let camera = new THREE.PerspectiveCamera(45, width / height, 0.1, 10000);
camera.position.copy(new THREE.Vector3(100, 100, 100));
camera.lookAt(new THREE.Vector3(0, 0, 0));
scene.add(camera);
let controls = new OrbitControls(camera, renderer.domElement);
// global variables
let mouse = new THREE.Vector2();
let planeMesh;
let textureData;
let imgPath =
"https://cdn.pixabay.com/photo/2022/05/23/13/16/bird-7216181_1280.jpg";
//load texture and initialize planeMesh, textureData and renderTargetTexture
new THREE.TextureLoader().load(imgPath, function (texture) {
textureData = texture;
planeMesh = new THREE.Mesh(
new THREE.PlaneGeometry(texture.image.width, texture.image.height),
new THREE.MeshBasicMaterial({ map: texture })
);
scene.add(planeMesh);
});
resize();
animate();
function resize() {
renderer.setSize(width,height);
camera.aspect=width/height;
camera.updateProjectionMatrix();
}
function animate() {
renderer.clear();
renderer.render(scene,camera);
requestAnimationFrame(animate);
controls.update();
}
window.addEventListener("resize", resize);
window.addEventListener("pointerup", (event) => {
mouse.x =((event.clientX-renderer.domElement.offsetLeft)/renderer.domElement.width)*2-1;
mouse.y =-((event.clientY-renderer.domElement.offsetTop)/renderer.domElement.height)*2+1;
rayCaster.setFromCamera(mouse, camera);
let intersections = rayCaster.intersectObject(planeMesh);
if(intersections.length>0){
mouse.x=parseInt(window.innerWidth/2+mouse.x*window.innerWidth/2);
mouse.y=parseInt(window.innerHeight/2+mouse.y*window.innerHeight/2);
renderer.clear();
renderer.setRenderTarget(renderTargetTexture);
renderer.clear();
renderer.render(scene,camera);
renderer.setRenderTarget(null);
renderer.render(sceneScreen,cameraRTT);
const read=new Float32Array(4);
renderer.readRenderTargetPixels(renderTargetTexture,mouse.x,mouse.y,1,1,read);
let r=parseInt(read[0]*255);
let g=parseInt(read[1]*255);
let b=parseInt(read[2]*255);
console.log("X:"+mouse.x+" Y:"+mouse.y+" RGB: ["+r+","+g+","+b+"] Color: %c ","background:rgb("+r+","+g+","+b+");");
}
});
</script>
</body>
</html>
1 Like
Maybe try rendering your texture into the render target first?
New code. To calculate pages scrolling coordinates:
<!DOCTYPE html>
<html lang="en">
<head>
<title>three.js webgl - cameras</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
</head>
<body>
<script type="importmap">
{
"imports": {
"three": "../build/three.module.js"
}
}
</script>
<script id="fragment_shader_screen" type="x-shader/x-fragment">
varying vec2 vUv;
uniform sampler2D tDiffuse;
void main() {
gl_FragColor = texture2D( tDiffuse, vUv );
}
</script>
<script id="vertexShader" type="x-shader/x-vertex">
varying vec2 vUv;
void main() {
vUv = uv;
gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );
}
</script>
<script type="module">
import * as THREE from 'three';
import { OrbitControls } from "./jsm/controls/OrbitControls.js";
let width = window.innerWidth,
height = window.innerHeight;
//initialize renderer
let renderer = new THREE.WebGLRenderer();
renderer.setSize(width, height);
renderer.autoClear = false;
document.body.appendChild(renderer.domElement);
//scene / raycaster / camera / controls
let scene = new THREE.Scene();
let cameraRTT = new THREE.OrthographicCamera( window.innerWidth / - 2, window.innerWidth / 2, window.innerHeight / 2, window.innerHeight / - 2, - 10000, 10000 );
cameraRTT.position.z = 100;
let sceneScreen = new THREE.Scene();
let renderTargetTexture = new THREE.WebGLRenderTarget(
window.innerWidth, window.innerHeight, { minFilter: THREE.LinearFilter, magFilter: THREE.NearestFilter, format: THREE.RGBAFormat, type: THREE.FloatType }
);
const materialScreen = new THREE.ShaderMaterial( {
uniforms: { 'tDiffuse': { value: renderTargetTexture.texture } },
vertexShader: document.getElementById( 'vertexShader' ).textContent,
fragmentShader: document.getElementById( 'fragment_shader_screen' ).textContent,
depthWrite: false
} );
const plane = new THREE.PlaneGeometry( window.innerWidth, window.innerHeight );
let quad = new THREE.Mesh(plane,materialScreen );
quad.position.z = - 100;
sceneScreen.add( quad );
let rayCaster = new THREE.Raycaster();
let camera = new THREE.PerspectiveCamera(45, width / height, 0.1, 10000);
camera.position.copy(new THREE.Vector3(100, 100, 100));
camera.lookAt(new THREE.Vector3(0, 0, 0));
scene.add(camera);
let controls = new OrbitControls(camera, renderer.domElement);
// global variables
let mouse = new THREE.Vector2();
let planeMesh;
let textureData;
let imgPath =
"https://cdn.pixabay.com/photo/2022/05/23/13/16/bird-7216181_1280.jpg";
//load texture and initialize planeMesh, textureData and renderTargetTexture
new THREE.TextureLoader().load(imgPath, function (texture) {
textureData = texture;
planeMesh = new THREE.Mesh(
new THREE.PlaneGeometry(texture.image.width, texture.image.height),
new THREE.MeshBasicMaterial({ map: texture })
);
scene.add(planeMesh);
});
resize();
animate();
function resize() {
renderer.setSize(width,height);
camera.aspect=width/height;
camera.updateProjectionMatrix();
}
function animate() {
renderer.clear();
renderer.render(scene,camera);
requestAnimationFrame(animate);
controls.update();
}
window.addEventListener("resize", resize);
window.addEventListener("pointerup", (event) => {
var scrollLeft=window.pageXOffset || document.documentElement.scrollLeft;
var scrollTop=window.pageYOffset || document.documentElement.scrollTop;
mouse.x =((event.clientX-renderer.domElement.offsetLeft+scrollLeft)/renderer.domElement.width)*2-1;
mouse.y =-((event.clientY-renderer.domElement.offsetTop+scrollTop)/renderer.domElement.height)*2+1;
rayCaster.setFromCamera(mouse, camera);
let intersections = rayCaster.intersectObject(planeMesh);
if(intersections.length>0){
mouse.x=parseInt(renderer.domElement.width/2+mouse.x*renderer.domElement.width/2);
mouse.y=parseInt(renderer.domElement.height/2+mouse.y*renderer.domElement.height/2);
renderer.clear();
renderer.setRenderTarget(renderTargetTexture);
renderer.clear();
renderer.render(scene,camera);
renderer.setRenderTarget(null);
renderer.render(sceneScreen,cameraRTT);
const read=new Float32Array(4);
renderer.readRenderTargetPixels(renderTargetTexture,mouse.x,mouse.y,1,1,read);
let r=parseInt(read[0]*255);
let g=parseInt(read[1]*255);
let b=parseInt(read[2]*255);
console.log("X:"+mouse.x+" Y:"+mouse.y+" RGB: ["+r+","+g+","+b+"] Color: %c ","background:rgb("+r+","+g+","+b+");");
}
});
</script>
</body>
</html>
hey N8Three can you pm me