Make a custom Pass to render a 360 degree equirectangular panorama image from a scene

Hello everyone,

First, thank you so much for this awesome three.js library and this awesome community. I really appreciate your support.

Now, I am trying to make a Pass to render a 360 degree equirectangular panorama image from a scene. My goal is as follow,

  1. Make a WebGLRenderTarget to render the 360 degree equirectangular image as a texture on a PlaneBufferGeometry.
  2. Then prepare a EffectComposer using the WebGLRenderTarget from point 1 as the renderTarget.
  3. In the EffectComposer I want to use a chain of post-processing passes to capture, process and render the 360 degree equirectangular image.

Here if we put the requirements of the 360 degree equirectangular image, it is easy to render the scene using the RenderPass. If I am not wrong, the RenderPass renders the scene and provides the result image to the post-processing pipeline and I need to work here.

So I planned to make a custom Render360Pass by following the implementation of the RenderPass class. The task of the Render360Pass is to take a 360 degree equirectangular panorama image of the scene and pass the result to the post-processing pipeline.

Here is what i tried so far,

(function () {

	class Render360Pass extends THREE.Pass {

		vertexShader = `attribute vec3 position;
						attribute vec2 uv;
						uniform mat4 projectionMatrix;
						uniform mat4 modelViewMatrix;
						varying vec2 vUv;
						void main()  {
							vUv = vec2( 1.- uv.x, uv.y );
							gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );
						}`;
		fragmentShader = `precision mediump float;
						uniform samplerCube map;
						varying vec2 vUv;
						#define M_PI 3.1415926535897932384626433832795
						void main()  {
							vec2 uv = vUv;
							float longitude = uv.x * 2. * M_PI - M_PI + M_PI / 2.;
							float latitude = uv.y * M_PI;
							vec3 dir = vec3(
								- sin( longitude ) * sin( latitude ),
								cos( latitude ),
								- cos( longitude ) * sin( latitude )
							);
							normalize( dir );
							gl_FragColor = textureCube( map, dir );
						}`;

		constructor(scene, camera, overrideMaterial, clearColor, clearAlpha) {
			super();
			this.scene = scene;
			this.camera = camera;
			this.overrideMaterial = overrideMaterial;
			this.clearColor = clearColor;
			this.clearAlpha = clearAlpha !== undefined ? clearAlpha : 0;
			this.clear = true;
			this.clearDepth = false;
			this.needsSwap = false;
			this._oldClearColor = new THREE.Color();

			this.width = 1;
			this.height = 1;
			this.material = new THREE.RawShaderMaterial({
				uniforms: {
					map: { type: 't', value: null }
				},
				vertexShader: this.vertexShader,
				fragmentShader: this.fragmentShader,
				side: THREE.DoubleSide,
				transparent: true
			});
			this.quad = new THREE.Mesh(
				new THREE.PlaneBufferGeometry(1, 1),
				this.material
			);
			this.scene.add(this.quad);
			this.orthoCamera = new THREE.OrthographicCamera(1 / - 2, 1 / 2, 1 / 2, 1 / - 2, -10000, 10000);

			this.cubeCamera = null;
			this.attachedCamera = null;
			this.setSize(4096, 2048);
		}

		setSize(width, height) {
			this.width = width;
			this.height = height;

			this.quad.scale.set(this.width, this.height, 1);

			this.orthoCamera.left = this.width / - 2;
			this.orthoCamera.right = this.width / 2;
			this.orthoCamera.top = this.height / 2;
			this.orthoCamera.bottom = this.height / - 2;

			this.orthoCamera.updateProjectionMatrix();

			// we are not using this.output anywhere as the target should come from EffectComposer 
			this.output = new THREE.WebGLRenderTarget(this.width, this.height, {
				minFilter: THREE.LinearFilter,
				magFilter: THREE.LinearFilter,
				wrapS: THREE.ClampToEdgeWrapping,
				wrapT: THREE.ClampToEdgeWrapping,
				format: THREE.RGBAFormat,
				type: THREE.UnsignedByteType
			});
			// we are not using this.output anywhere as the target should come from EffectComposer 
		}

		getCubeCamera(size, renderer) {
			let gl = renderer.getContext();
			let cubeMapSize = Math.min(gl.getParameter(gl.MAX_CUBE_MAP_TEXTURE_SIZE), size);
			let options = { format: THREE.RGBAFormat, magFilter: THREE.LinearFilter, minFilter: THREE.LinearFilter };
			const cubeRenderTarget = new THREE.WebGLCubeRenderTarget(cubeMapSize, options);
			this.cubeCamera = new THREE.CubeCamera(.1, 1000, cubeRenderTarget);
			return this.cubeCamera;
		}

		convert(cubeCamera, renderer) {
			this.quad.material.uniforms.map.value = cubeCamera.renderTarget.texture;
			renderer.render(this.scene, this.orthoCamera);//, this.output, true ); // we are not using this.output anywhere as the target should come from EffectComposer 		
		};

		update(camera, scene, renderer) {
			let autoClear = renderer.autoClear;
			renderer.autoClear = true;
			this.cubeCamera.position.copy(camera.position);
			this.cubeCamera.update(renderer, scene);
			renderer.autoClear = autoClear;
		}

		prepareAndRender(renderer) {
			this.getCubeCamera(2048, renderer);
			this.update(this.camera, this.scene, renderer);
			this.convert(this.cubeCamera, renderer);
		}

		render(renderer, writeBuffer, readBuffer
			/*, deltaTime, maskActive */
		) {
			const oldAutoClear = renderer.autoClear;
			renderer.autoClear = false;
			let oldClearAlpha, oldOverrideMaterial;

			if (this.overrideMaterial !== undefined) {

				oldOverrideMaterial = this.scene.overrideMaterial;
				this.scene.overrideMaterial = this.overrideMaterial;

			}

			if (this.clearColor) {

				renderer.getClearColor(this._oldClearColor);
				oldClearAlpha = renderer.getClearAlpha();
				renderer.setClearColor(this.clearColor, this.clearAlpha);

			}

			if (this.clearDepth) {

				renderer.clearDepth();

			}

			renderer.setRenderTarget(this.renderToScreen ? null : readBuffer); // TODO: Avoid using autoClear properties, see https://github.com/mrdoob/three.js/pull/15571#issuecomment-465669600

			if (this.clear) renderer.clear(renderer.autoClearColor, renderer.autoClearDepth, renderer.autoClearStencil);
			// renderer.render( this.scene, this.camera );
			this.prepareAndRender(renderer);

			if (this.clearColor) {

				renderer.setClearColor(this._oldClearColor, oldClearAlpha);

			}

			if (this.overrideMaterial !== undefined) {

				this.scene.overrideMaterial = oldOverrideMaterial;

			}

			renderer.autoClear = oldAutoClear;
		}

	}

	THREE.Render360Pass = Render360Pass;

})();

But it is not working as expected and throwing many console errors.

The errors are saying something about shaders but I think the issue is in the Render360Pass implementation. I know the Render360Pass implementation is missing something but I could not figure out the problem as I am new to this domain.

Any suggestion and help regarding this Render360Pass would be a great help to achieve my goal. Thank you so much.

Notes:

  1. There is a library that can take equirectangular panorama PNG images, GitHub - spite/THREE.CubemapToEquirectangular: Export an equirectangular panorama image from a three.js scene. I was following this to implement the Render360Pass.
  2. Here is another project that uses the same library to capture 360 videos, GitHub - imgntn/j360: 360 Video and Photo Capture in 4K for Three.js. Here, the library is slightly modified.
  3. I have to modify some codes to fit my local three.js revision and RenderPass class. Here is an open pull request regarding this case, update-to-r127 by nakamura2000 · Pull Request #11 · spite/THREE.CubemapToEquirectangular · GitHub.
  4. I am using three.js r141. Due to some issues I can’t upgrade or downgrade it.
  5. I am always available to provide more information if required. Please let me know if needed.

Iirc RawShaderMaterial does not include absolutely anything from three environment - try using ShaderMaterial instead.

2 Likes

Hello @mjurczyk,
Thanks for your response.

I am not sure if I did it right. According to your suggestion what I did is updated the RawShaderMaterial call with ShaderMaterial call. So it is now as follow,

(function () {

	class Render360Pass extends THREE.Pass {

		vertexShader = `attribute vec3 position;
						attribute vec2 uv;
						uniform mat4 projectionMatrix;
						uniform mat4 modelViewMatrix;
						varying vec2 vUv;
						void main()  {
							vUv = vec2( 1.- uv.x, uv.y );
							gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );
						}`;
		fragmentShader = `precision mediump float;
						uniform samplerCube map;
						varying vec2 vUv;
						#define M_PI 3.1415926535897932384626433832795
						void main()  {
							vec2 uv = vUv;
							float longitude = uv.x * 2. * M_PI - M_PI + M_PI / 2.;
							float latitude = uv.y * M_PI;
							vec3 dir = vec3(
								- sin( longitude ) * sin( latitude ),
								cos( latitude ),
								- cos( longitude ) * sin( latitude )
							);
							normalize( dir );
							gl_FragColor = textureCube( map, dir );
						}`;

		constructor(scene, camera, overrideMaterial, clearColor, clearAlpha) {
			super();
			this.scene = scene;
			this.camera = camera;
			this.overrideMaterial = overrideMaterial;
			this.clearColor = clearColor;
			this.clearAlpha = clearAlpha !== undefined ? clearAlpha : 0;
			this.clear = true;
			this.clearDepth = false;
			this.needsSwap = false;
			this._oldClearColor = new THREE.Color();

			this.width = 1;
			this.height = 1;
			this.material = new THREE.ShaderMaterial({
				uniforms: {
					map: { type: 't', value: null }
				},
				vertexShader: this.vertexShader,
				fragmentShader: this.fragmentShader,
				side: THREE.DoubleSide,
				transparent: true
			});
			this.quad = new THREE.Mesh(
				new THREE.PlaneBufferGeometry(1, 1),
				this.material
			);
			this.scene.add(this.quad);
			this.orthoCamera = new THREE.OrthographicCamera(1 / - 2, 1 / 2, 1 / 2, 1 / - 2, -10000, 10000);

			this.cubeCamera = null;
			this.attachedCamera = null;
			this.setSize(4096, 2048);
		}

		setSize(width, height) {
			this.width = width;
			this.height = height;

			this.quad.scale.set(this.width, this.height, 1);

			this.orthoCamera.left = this.width / - 2;
			this.orthoCamera.right = this.width / 2;
			this.orthoCamera.top = this.height / 2;
			this.orthoCamera.bottom = this.height / - 2;

			this.orthoCamera.updateProjectionMatrix();

			// we are not using this.output anywhere as the target should come from EffectComposer 
			this.output = new THREE.WebGLRenderTarget(this.width, this.height, {
				minFilter: THREE.LinearFilter,
				magFilter: THREE.LinearFilter,
				wrapS: THREE.ClampToEdgeWrapping,
				wrapT: THREE.ClampToEdgeWrapping,
				format: THREE.RGBAFormat,
				type: THREE.UnsignedByteType
			});
			// we are not using this.output anywhere as the target should come from EffectComposer 
		}

		getCubeCamera(size, renderer) {
			let gl = renderer.getContext();
			let cubeMapSize = Math.min(gl.getParameter(gl.MAX_CUBE_MAP_TEXTURE_SIZE), size);
			let options = { format: THREE.RGBAFormat, magFilter: THREE.LinearFilter, minFilter: THREE.LinearFilter };
			const cubeRenderTarget = new THREE.WebGLCubeRenderTarget(cubeMapSize, options);
			this.cubeCamera = new THREE.CubeCamera(.1, 1000, cubeRenderTarget);
			return this.cubeCamera;
		}

		convert(cubeCamera, renderer) {
			this.quad.material.uniforms.map.value = cubeCamera.renderTarget.texture;
			renderer.render(this.scene, this.orthoCamera);//, this.output, true ); // we are not using this.output anywhere as the target should come from EffectComposer 		
		};

		update(camera, scene, renderer) {
			let autoClear = renderer.autoClear;
			renderer.autoClear = true;
			this.cubeCamera.position.copy(camera.position);
			this.cubeCamera.update(renderer, scene);
			renderer.autoClear = autoClear;
		}

		prepareAndRender(renderer) {
			this.getCubeCamera(2048, renderer);
			this.update(this.camera, this.scene, renderer);
			this.convert(this.cubeCamera, renderer);
		}

		render(renderer, writeBuffer, readBuffer
			/*, deltaTime, maskActive */
		) {
			const oldAutoClear = renderer.autoClear;
			renderer.autoClear = false;
			let oldClearAlpha, oldOverrideMaterial;

			if (this.overrideMaterial !== undefined) {

				oldOverrideMaterial = this.scene.overrideMaterial;
				this.scene.overrideMaterial = this.overrideMaterial;

			}

			if (this.clearColor) {

				renderer.getClearColor(this._oldClearColor);
				oldClearAlpha = renderer.getClearAlpha();
				renderer.setClearColor(this.clearColor, this.clearAlpha);

			}

			if (this.clearDepth) {

				renderer.clearDepth();

			}

			renderer.setRenderTarget(this.renderToScreen ? null : readBuffer); // TODO: Avoid using autoClear properties, see https://github.com/mrdoob/three.js/pull/15571#issuecomment-465669600

			if (this.clear) renderer.clear(renderer.autoClearColor, renderer.autoClearDepth, renderer.autoClearStencil);
			// renderer.render( this.scene, this.camera );
			this.prepareAndRender(renderer);

			if (this.clearColor) {

				renderer.setClearColor(this._oldClearColor, oldClearAlpha);

			}

			if (this.overrideMaterial !== undefined) {

				this.scene.overrideMaterial = oldOverrideMaterial;

			}

			renderer.autoClear = oldAutoClear;
		}

	}

	THREE.Render360Pass = Render360Pass;

})();

But the errors are not fixed and I am seeing the same kind of errors.

Do I have to change and fix some other things also? And Is there anything else I need to provide?

Please let me know your feedback.