Server-side (node.js) browserless texture rendering

mark · May 4, 2018, 10:38am

I’m trying to do a fairly basic server-side render (browserless, in node.js) of a textured plane to an equirectangular (360º format) .png image.

So far I’m managing to generate .png files from a THREE scene (cameras, meshes, basic solid color materials and shaders seem to work fine) but I’m having trouble with textures.

Below is the full script. I didn’t include the local CubemapToEquirectangular which I stole from this repo and made some minor changes but that part seems to be working fine.

This is what my output image currently looks like. The top left image is drawn as an overlay on top of the canvas (to verify that the image is loaded properly), but the transparent plane in front of the magenta plane should have the same image as a texture in its material. Instead it appears completely transparent.

three-plane-equi

Note that when I load the texture using an HTTP request (instead of reading directly from the filesystem), the textured plane shows up opaque black instead of transparent.

Any suggestions are welcome!

var fs = require("fs");
var path = require("path");
var Canvas = require("canvas");
var glContext = require('gl')(1,1); //headless-gl
var THREE = require("three");
var CubemapToEquirectangular = require('../lib/three-CubemapToEquirectangular');

var equi, camera, scene, renderer, teximage;

var window = {innerWidth: 800, innerHeight: 600};
var LOAD_TEXTURE_USING_HTTP = false;

// http://stackoverflow.com/a/14855016/2207790
var loadTextureHTTP = function (url, callback) {
  require('request')({
    method: 'GET', url: url, encoding: null
  }, function(error, response, body) {
    if(error) throw error;

    console.log('body:', body.length);

    var image = new Canvas.Image;
    image.src = body;

    var texture = new THREE.Texture(image);
    texture.needsUpdate = true;

    teximage = image;
    if (callback) callback(texture);
  });
};

function init() {
  // GL scene renderer
  var canvasGL = new Canvas(window.innerWidth, window.innerHeight);
  canvasGL.addEventListener = function(event, func, bind_) {}; // mock function to avoid errors inside THREE.WebGlRenderer()
  renderer = new THREE.WebGLRenderer( { context: glContext, antialias: true, canvas: canvasGL });

  // Equirectangular renderer
  var canvasEqui = new Canvas(4096, 2048);
  equi = new CubemapToEquirectangular( renderer, true, { canvas: canvasEqui} );

  // camera
  camera = new THREE.PerspectiveCamera( 70, window.innerWidth / window.innerHeight, 1, 10000 );
  camera.position.set( 1,1,1 );
  camera.aspect = window.innerWidth / window.innerHeight;
  camera.updateProjectionMatrix();

  // load image from filesystem
  var imgData = fs.readFileSync(path.join(__dirname, 'UV_Grid_Sm.jpg'));
  teximage = new Canvas.Image();
  teximage.src = imgData;

  // scene
  scene = new THREE.Scene();

  // untextured purple plane
  var geometry = new THREE.PlaneGeometry( 10, 20, 32 );
  var material = new THREE.MeshBasicMaterial( {color: 0xff00f0, side: THREE.DoubleSide} );
  var plane = new THREE.Mesh( geometry, material );
  plane.position.z = -3;
  scene.add( plane );

  // texture
  var texture = new THREE.Texture(teximage);
  texture.wrapS = THREE.RepeatWrapping;
  texture.wrapT = THREE.RepeatWrapping;
  // texture.repeat.set( 4, 4 );
  // texture.matrixAutoUpdate = false; // set this to false to update texture.matrix manually

  // textured plane (shows up transparent)
  material = new THREE.MeshBasicMaterial({ map: texture });
  // material = new THREE.MeshBasicMaterial();
  plane = new THREE.Mesh(geometry, material );
  plane.position.z = -2.8;
  plane.scale.set(0.5,0.5,0.5);
  scene.add( plane );

  // // load texture using HTTP request, also doesn't work but makes the textured plane white
  if (LOAD_TEXTURE_USING_HTTP) {
    loadTextureHTTP('http://localhost:8000/UV_Grid_Sm.jpg', function(tex) {
      console.log('http done');
      material.map = tex;
      tex.matrix.identity().translate(-0.435, -0.235).scale(2.2,2.2);

      renderAndExport("./three-plane-equi.png", 3000);
    });
  }

  // light
  scene.add( new THREE.HemisphereLight( 0x443333, 0x222233, 4 ) );
}

function render() {
  // renderer.render( scene, camera );
  var canv = equi.updateAndGetCanvas( camera, scene );

  // overlay texture's source image on the canvas to verify image was loaded properly
  canv.getContext('2d').drawImage(teximage, 0, 0, 1024, 512);
}

function exportImage(exportPath) {
  var out = fs.createWriteStream(exportPath);
  var canvasStream = equi.canvas.pngStream();
  canvasStream.on("data", function (chunk) { out.write(chunk); });
  canvasStream.on("end", function () { console.log("done"); });
}

function renderAndExport(exportPath, delay) {
  var func = function() {
    console.log('rendering...');
    render();
    console.log('exporting...');
    exportImage(exportPath);
    console.log('done');
  };

  if (delay !== undefined) {
    console.log('waiting '+delay+' ms in case texture initialization takes time...');
    setTimeout(function(){ func(); }, delay);
  } else {
    func();
  }
}

init();
if (!LOAD_TEXTURE_USING_HTTP) {
  renderAndExport("./three-plane-equi.png", 3000);
}

For completeness sake, the dependencies from my package.json:

  "dependencies": {
    "canvas": "1.6.10",
    "gl": "^4.0.4",
    "three": "^0.92.0"
  },

Execution log when loading image directly from filesystem:

THREE.WebGLRenderer 92
THREE.WebGLRenderer: WEBGL_depth_texture extension not supported.
THREE.WebGLRenderer: OES_texture_float extension not supported.
THREE.WebGLRenderer: OES_texture_float_linear extension not supported.
THREE.WebGLRenderer: OES_texture_half_float extension not supported.
THREE.WebGLRenderer: OES_texture_half_float_linear extension not supported.
THREE.WebGLRenderer: OES_standard_derivatives extension not supported.
THREE.WebGLRenderer: OES_element_index_uint extension not supported.
waiting 3000 ms in case texture initialization takes time...
rendering...
THREE.WebGLRenderer: EXT_texture_filter_anisotropic extension not supported.
exporting...
done
done

Execution log when loading image over HTTP:

THREE.WebGLRenderer 92
THREE.WebGLRenderer: WEBGL_depth_texture extension not supported.
THREE.WebGLRenderer: OES_texture_float extension not supported.
THREE.WebGLRenderer: OES_texture_float_linear extension not supported.
THREE.WebGLRenderer: OES_texture_half_float extension not supported.
THREE.WebGLRenderer: OES_texture_half_float_linear extension not supported.
THREE.WebGLRenderer: OES_standard_derivatives extension not supported.
THREE.WebGLRenderer: OES_element_index_uint extension not supported.
body: 296346
http done
waiting 3000 ms in case texture initialization takes time...
rendering...
THREE.WebGLRenderer: EXT_texture_filter_anisotropic extension not supported.
THREE.WebGLState: TypeError: texImage2D(GLenum, GLint, GLenum, GLint, GLenum, GLenum, ImageData)
    at WebGLRenderingContext.texImage2D (/Users/mark/code/node-equirectangular-renderer/node_modules/gl/webgl.js:3495:13)
    at Object.texImage2D (/Users/mark/code/node-equirectangular-renderer/node_modules/three/build/three.js:19534:19)
    at uploadTexture (/Users/mark/code/node-equirectangular-renderer/node_modules/three/build/three.js:20247:12)
    at WebGLTextures.setTexture2D (/Users/mark/code/node-equirectangular-renderer/node_modules/three/build/three.js:19881:6)
    at WebGLRenderer.setTexture2D (/Users/mark/code/node-equirectangular-renderer/node_modules/three/build/three.js:23464:14)
    at SingleUniform.setValueT1 [as setValue] (/Users/mark/code/node-equirectangular-renderer/node_modules/three/build/three.js:15699:12)
    at Function.WebGLUniforms.upload (/Users/mark/code/node-equirectangular-renderer/node_modules/three/build/three.js:16039:7)
    at setProgram (/Users/mark/code/node-equirectangular-renderer/node_modules/three/build/three.js:23023:19)
    at WebGLRenderer.renderBufferDirect (/Users/mark/code/node-equirectangular-renderer/node_modules/three/build/three.js:21797:18)
    at renderObject (/Users/mark/code/node-equirectangular-renderer/node_modules/three/build/three.js:22556:11)
exporting...
done
done

Mugen87 · May 4, 2018, 1:49pm

You definitely have to set texture.needsUpdate = true; after this section. Can you apply the change and make a test?

mark · May 4, 2018, 5:39pm

@Mugen87, thanks for the suggestion. I added the needsUpdate = true line and now the plane shows up black instead of transparent (just like when I load using HTTP, where I was actually setting the needsUpdate, so that makes sense now).

Unfortunately no texture yet. This is what my output looks like now (manually downscaled the image a bit);

three-plane-equi

mark · May 4, 2018, 5:45pm

Note that I googled around for the THREE,WebGLRenderer’s unsupported gl extension warnings in my execution logs and found this issue in the headless-gl package I’m using.

Sounds like headless-gl might not support the necessary GL extensions for loading a texture? I know nothing about OpenGL extensions, so I might be talking out of my ass here… Can anybody confirm or refute?

Mugen87 · May 4, 2018, 5:55pm

I don’t think these extension are relevant for your case.

I think the problem is that Canvas.Image is no valid parameter for the WebGL texImage2D call. Read the following for more information.

mark · May 7, 2018, 8:54am

Alright! Thanks @Mugen87 you were right. I was able to get it working with get-pixels and THREE.DataTexture.

My image loading code looks like this:

  getPixels(__dirname+"/UV_Grid_Sm.jpg", function(err, pixels) {
      if(err) {
        console.log("Failed to load texture using get-pixels:", err);
        return;
      }

      var texture = new THREE.DataTexture( new Uint8Array(pixels.data), pixels.shape[0], pixels.shape[1], THREE.RGBAFormat );
      texture.needsUpdate = true;
      material.map = texture;

      renderAndExport("./output.png");
    });

Proof:
three-plane-equi

dukuo · August 30, 2018, 7:33pm

I know this may be an old thread @mark but could you post a working example of what you achieved? Thanks!

mark · August 31, 2018, 9:17am

@dukuo, you can find my repo here: https://gitlab.com/shortnotion/node-equirectangular-renderer

I haven’t cleaned it up, as it was basically a proof-of-concept and I hope the dependencies are up to date :]

dukuo · August 31, 2018, 7:40pm

thanks @mark, i’m inspecting the code and so far it runs with headless-gl flawlessly! Thanks a lot, really appreciate it

Topic		Replies	Views
Server-side rendering and sending data to the client as JPEG (or PNG) - with Client side interaction data Questions node-js	14	6708	April 30, 2020
Transparency in nodejs version of three js (headless server) Questions	2	533	July 1, 2022
renderer.domElement.toDataURL() returns transparent image with headless-gl but works in browser Questions headless-render	4	904	February 22, 2021
A working example that renders a .glb file on the server side (node) Questions loaders , node-js , server-side-rendering , puppeteer , headless-render	8	5429	October 4, 2024
Using GLB with textures on node Questions node-js , gltf	6	3106	July 30, 2021

Server-side (node.js) browserless texture rendering

Related topics