Production-ready green screen with three.js

I trying to port this webgl example Production-ready green screen in the browser to three.js
putting the chroma key shader in a 3d object into the scene, because the idea is interact with camera and lights.This green screen algorithm is derived from the Chroma Key filter in OBS Studio that give us a very good effect result.
Here is my code
I don’t know what is missing or wrong in my code, as the browser inspector is not showing anything

I’ve quickly ported the code to the latest version of three.js: https://jsfiddle.net/mpkw8yse/

But without the video texture for simplicity.

2 Likes

Hey, I’ve created an OBS inspired webcam greenscreen example as well,
for extra reference in case you need.
Webcam - Three.js Tutorials (sbcode.net)

6 Likes

Hi ,Thanks !

Hi, Seanwasere. It’s exactly what I was trying to do ! Thanks !! :slightly_smiling_face:

Hello,
First, sorry for my english :sweat:
I’m trying now that the chroma key object is affected by light. I spent a lot of time studying examples of how to use onBeforeCompile, in this forum and this guide (Extending three.js materials with GLSL), but I still don’t quite understand how the Replace works.
I made this fiddle using a phongMaterial to demonstrate the expected result
chromakey shader from webcam

Can you please share the working code with me which is fetching video from a url instead of webcam

Hey, sorry for delay. I Create an example based on seanwasere (Profile - seanwasere - three.js forum) code using THREE.VideoTexture
Becouse security reason you should use this video in your server or localhost

Download the video here

<!DOCTYPE html>
<html>
<head>
<title>Three.js Chromakey shader</title>
<meta charset="utf-8">
	<meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">

<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r123/three.min.js"></script>
<script src="https://unpkg.com/three@0.85.0/examples/js/controls/OrbitControls.js"></script>

<script src="https://cdnjs.cloudflare.com/ajax/libs/dat-gui/0.7.7/dat.gui.min.js" integrity="sha512-LF8ZB1iTwi4Qvkm4pekHG4a437Y9Af5ZuwbnW4GTbAWQeR2E4KW8WF+xH8b9psevV7wIlDMx1MH9YfPqgKhA/Q==" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/stats.js/7/Stats.js" integrity="sha512-+nNqDCQd6FiN/mQKnpfI/UswnfuG/cjHZbv+amV/2PLiuGgN8+D5dKzR4PqnSY5pZBExGavZz+FdjIeI9WgpKQ==" crossorigin="anonymous"></script>

<style>
    body {
        overflow: hidden;
        margin: 0px;
    }
</style>
</head>

<body>

<video id="video" width="320" height="240" preload autoplay loop muted src="171003D_002_2K_preview.webm"></video>

<script id="vertexShader" type="glsl">

  varying vec2 vUv;
  void main( void ) {
      vUv = uv;
      gl_Position = projectionMatrix * modelViewMatrix * vec4(position,1.0);
  }

</script>


<script id="fragmentShader" type="glsl">

  uniform vec3 keyColor;
  uniform float similarity;
  uniform float smoothness;
  varying vec2 vUv;
  uniform sampler2D map;
  void main() {

      vec4 videoColor = texture2D(map, vUv);

      float Y1 = 0.299 * keyColor.r + 0.587 * keyColor.g + 0.114 * keyColor.b;
      float Cr1 = keyColor.r - Y1;
      float Cb1 = keyColor.b - Y1;

      float Y2 = 0.299 * videoColor.r + 0.587 * videoColor.g + 0.114 * videoColor.b;
      float Cr2 = videoColor.r - Y2;
      float Cb2 = videoColor.b - Y2;

      float blend = smoothstep(similarity, similarity + smoothness, distance(vec2(Cr2, Cb2), vec2(Cr1, Cb1)));
      gl_FragColor = vec4(videoColor.rgb, videoColor.a * blend);
  }

</script>


<script>

let camera, controls, gridHelper, scene, renderer, stats;
let container, urlVideo, urlVideoTexture, chromakeyMaterial;

window.onload = init();

function init() {

    container = document.createElement( 'div' );
    document.body.appendChild( container );

    scene = new THREE.Scene();

    renderer = new THREE.WebGLRenderer();
    renderer.setPixelRatio( window.devicePixelRatio );
    renderer.setSize(window.innerWidth, window.innerHeight);
    document.body.appendChild(renderer.domElement);

    camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 1000);

    controls = new THREE.OrbitControls( camera, renderer.domElement );
    controls.minDistance = 5;
    controls.maxDistance = 20;
    controls.enableDamping = true;
    controls.dampingFactor = 0.5;

    gridHelper = new THREE.GridHelper(10, 10);

    gridHelper.position.y = -1.5;
    scene.add(gridHelper);

    camera.position.z = 5;

    // urlVideo

    urlVideo = document.getElementById("video");

    urlVideo.onplaying = function() {

      urlVideoTexture = new THREE.VideoTexture(urlVideo);
      urlVideoTexture.minFilter = THREE.LinearFilter;
      urlVideoTexture.magFilter = THREE.LinearFilter;


      const vertexShader = document.getElementById("vertexShader").textContent;
      const fragmentShader = document.getElementById("fragmentShader").textContent;

      // Cria o material usando a urlVideoTexture

      chromakeyMaterial = new THREE.ShaderMaterial({
          transparent: true,
          uniforms: {
              map: { value: urlVideoTexture },
              keyColor: { value: [0.0, 1.0, 0.0] },
              similarity: { value: 0.74 },
              smoothness: { value: 0.0 }
          },
          vertexShader: vertexShader,
          fragmentShader: fragmentShader
      });

      const geometry = new THREE.PlaneGeometry( 16, 9 );
      const mesh = new THREE.Mesh( geometry, chromakeyMaterial );
			geometry.scale( 0.4, 0.5, 0.5 );
      scene.add(mesh);

  }


    var data = {
        keyColor: [0, 255, 0],
        similarity: 0.74,
        smoothness: 0.0
    };

    var gui = new dat.GUI( { width: 300 } );
    gui.addColor(data, 'keyColor').onChange(() => updateKeyColor(data.keyColor));
    gui.add(data, 'similarity', 0.0, 1.0).onChange(() => updateSimilarity(data.similarity));
    gui.add(data, 'smoothness', 0.0, 1.0).onChange(() => updateSmoothness(data.smoothness));

    stats = new Stats();
    container.appendChild( stats.domElement );

    window.addEventListener('resize', onWindowResize, false);

}

function onWindowResize() {
    camera.aspect = window.innerWidth / window.innerHeight;
    camera.updateProjectionMatrix();
    renderer.setSize(window.innerWidth, window.innerHeight);
    render();
}

function updateKeyColor(v) {
    chromakeyMaterial.uniforms.keyColor.value = [v[0] / 255, v[1] / 255, v[2] / 255];
}

function updateSimilarity(v) {
    chromakeyMaterial.uniforms.similarity.value = v;
}

function updateSmoothness(v) {
    chromakeyMaterial.uniforms.smoothness.value = v;
}

function animate() {
    requestAnimationFrame(animate);

    //if (urlVideo.readyState === urlVideo.HAVE_ENOUGH_DATA) {
    if (urlVideoTexture)
        urlVideoTexture.needsUpdate = true;
    //}
    controls.update();
    stats.update();
    render();

};

function render() {
    renderer.render(scene, camera);
}
animate();
2 Likes

Thanks for your help, I will test out the same