THREE.DataTexture - data manipulation and texture resizing

Resources
, , , ,
1

For those who might ever have a need to resize textures and / or convert texture data, here is one possible way of doing this.

Some might find handling of Uint16Array and Float32Array somewhat interesting.

This is not necessarily limited to THREE.DataTexture and the code can probably be simplified and improved. Try passing any texture.image to the function to see what it does.

The whole code can be seen in my repository and can be tested with my IMG2MESH viewer / converter.

Don’t hesitate to try this and if you manage to improve it then post your code here.

      async function resize_texture( image, res = 2048 ) {
        let tex;

        await new Promise( resolve => {
          let canvas = document.createElement('canvas');

          let scale = res / Math.max( image.width, image.height );

          canvas.width = image.width * Math.min( 1, scale );
          canvas.height = image.height * Math.min( 1, scale );

          let ctx = canvas.getContext( '2d', { willReadFrequently: true } );

          if ( image instanceof ImageData ) {
            ctx.putImageData( image, 0, 0 );
          } else if ( image.data && image.data.constructor === Float32Array ) {
            let u8 = new Uint8Array( image.data.length );

            for ( let i = 0; i < image.data.length; i ++ ) {
              let tmp = Math.max( -1, Math.min( 1, image.data[ i ] ) );
              tmp = tmp < 0 ? ( tmp * 0x8000 ) : ( tmp * 0x7FFF );
              u8[ i ] = tmp / 128.0;
            }

            let imgData = new ImageData( new Uint8ClampedArray( u8.buffer ), image.width, image.height );
            ctx.putImageData( imgData, 0, 0 );
          } else if ( image.data && image.data.constructor === Uint16Array ) {
            let u8 = new Uint8Array( image.data.length );

            for ( let i = 0; i < image.data.length; i ++ ) {
              let tmp = Math.max( -1, Math.min( 1, THREE.DataUtils.fromHalfFloat( image.data[ i ] ) ) );
              tmp = tmp < 0 ? ( tmp * 0x8000 ) : ( tmp * 0x7FFF );
              u8[ i ] = tmp / 128.0;
            }

            let imgData = new ImageData( new Uint8ClampedArray( u8.buffer ), image.width, image.height );
            ctx.putImageData( imgData, 0, 0 );
          } else if ( image.data && image.data.constructor === Uint8Array ) {
            let imgData = new ImageData( new Uint8ClampedArray( image.data.buffer ), image.width, image.height );
            ctx.putImageData( imgData, 0, 0 );
          } else {
            ctx.drawImage( image, 0, 0, canvas.width, canvas.height );
          }

          resolve( tex = new THREE.CanvasTexture( canvas ) );
        });

        return tex;
      }
2

Here is an updated version of the code, which speeds up resizing common image formats by using the Image element. It also passes texture instead of texture.image to this function.

Maybe try eventually to understand the whole code of the viewer since it might make it easier to grasp certain variables / parameters.

I would not be looking to make any further code updates in this topic so check my repository if it interests you.

      async function resize_texture( texture, res = 2048, tex_flip = false, m_type = 'image/png', isMeshTex = false ) {
        const image = texture.image;

        let image_types = [ 'avif', 'bmp', 'gif', 'jpeg', 'png', 'svg+xml', 'webp' ];
        let image_hd_types = [ 'x.hdr', 'x.exr', 'ktx2' ];

        let img_ext = image_types.some( ext => m_type.endsWith( ext ) );
        let img_hd_ext = image_hd_types.some( ext => m_type.endsWith( ext ) );
        let mesh_texture = ( isMeshTex === true && ( img_ext === true || img_hd_ext === true ) );

        let tex, img = new Image();

        await new Promise( resolve => {
          img.onload = function() {
            let canvas2 = document.createElement('canvas');

            let scale = res / Math.max( img.naturalWidth, img.naturalHeight );

            canvas2.width = img.naturalWidth * Math.min( 1, scale );
            canvas2.height = img.naturalHeight * Math.min( 1, scale );

            let ctx2 = canvas2.getContext( '2d', { willReadFrequently: true } );

            // Flip image vertically
            if (tex_flip === true) {
              ctx2.translate( 0, canvas2.height );
              ctx2.scale( 1, -1 );
            }

            ctx2.drawImage( img, 0, 0, canvas2.width, canvas2.height );

            resolve( tex = new THREE.CanvasTexture( canvas2 ) );
          }

          if ( image.src ) {
            img.src = image.src;
          } else if ( image.data && ( img_ext === true || mesh_texture === true ) ) {

            let blob = new Blob( [ image.data ], { type: m_type } );

            img.src = URL.createObjectURL( blob );
            URL.revokeObjectURL( blob );
          } else {
            let canvas1 = document.createElement('canvas');

            canvas1.width = image.width;
            canvas1.height = image.height;

            let ctx1 = canvas1.getContext( '2d', { willReadFrequently: true } );

            if ( image instanceof ImageData ) {
              ctx1.putImageData( image, 0, 0 );
            } else if ( image.data && image.data.constructor === Float32Array ) {
              let u8 = new Uint8Array( image.data.length );

              for ( let i = 0; i < image.data.length; i ++ ) {
                let tmp = Math.max( -1, Math.min( 1, image.data[ i ] ) );
                tmp = tmp < 0 ? ( tmp * 0x8000 ) : ( tmp * 0x7FFF );
                u8[ i ] = tmp / 128.0;
              }

              let imgData = new ImageData( new Uint8ClampedArray( u8.buffer ), image.width, image.height );
              ctx1.putImageData( imgData, 0, 0 );
            } else if ( image.data && image.data.constructor === Uint16Array ) {
              let u8 = new Uint8Array( image.data.length );

              for ( let i = 0; i < image.data.length; i ++ ) {
                let tmp = Math.max( -1, Math.min( 1, THREE.DataUtils.fromHalfFloat( image.data[ i ] ) ) );
                tmp = tmp < 0 ? ( tmp * 0x8000 ) : ( tmp * 0x7FFF );
                u8[ i ] = tmp / 128.0;
              }

              let imgData = new ImageData( new Uint8ClampedArray( u8.buffer ), image.width, image.height );
              ctx1.putImageData( imgData, 0, 0 );
            } else if ( image.data && image.data.constructor === Uint8Array ) {
              let imgData = new ImageData( new Uint8ClampedArray( image.data.buffer ), image.width, image.height );
              ctx1.putImageData( imgData, 0, 0 );
            } else {
              ctx1.drawImage( image, 0, 0, canvas1.width, canvas1.height );
            }

            let base64data = canvas1.toDataURL( 'image/png', 1 ).replace( /^data:image\/(avif|png|gif|ktx2|jpeg|webp);base64,/, '' );
            let a2b = atob( base64data );
            let buff = new Uint8Array( a2b.length );

            for ( let i = 0, l = buff.length; i < l; i ++ ) {
              buff[ i ] = a2b.charCodeAt( i );
            }

            let blob = new Blob( [ buff ], { type: 'image/png' } );

            img.src = URL.createObjectURL( blob );
            URL.revokeObjectURL( blob );
          }
        });

        // Dispose of current texture and return a new one

        if (texture.isTexture) texture.dispose();

        return tex;
      }
3

Is there a reason why you don’t do this in the webgl context?

4

If you could provide a specific example of what you mean by that then I could look into it and see if it could be included in my viewer. Don’t hesitate to even use the function I provided and modify it any way you find better.

Things do work properly as they are and resizing is only done if the device this app runs on might have image resolution limitations.

5

Well, if you are going to use it as a texture eventually, I think loading it into the 2d canvas and then the Webgl one is going to be pretty expensive. You could do all this processing in the Webgl canvas and thus pay the penalty of loading it once.

6

No offence, but your statement is kind of hypothetical and without any code example to back it up.

Since my app does 1-time 1-image resize operation (and this is only when needed), then disposing of an old texture and replacing it with new should not be too taxing.

I did make slight improvements to my code and, just for the reference, the code in my initial post is passing texture.image to the resize function while in my last code I opted to pass the texture instead so it can be disposed of and a new texture returned.

7

None taken but no offence your use case then is extremely specific and you didn’t clarify that.

I imagined something where you can drag the corners of an image and resize it in real time.

In any case, you are using a 2d canvas and it seems you want to use it as a texture in a 3D one after the operation. You pay an extremely high price to upload the image into the first canvas. Then another high price to read it out of the canvas. And then another to upload it to the Webgl canvas.

You could just upload it to the Webgl canvas, and pay that price once.

Also, don’t take this the wrong way, but this is a threejs forum, it would be slightly more consistent to give an example of how to do this with threejs.

But yeah, if you have to do this once I guess it’s fine. But then also conceptually, why don’t you just resize the image offline? Why would you need to resize it, even once, but every time you load the page? Just resize it once in your life, put it on a server, and now everyone just downloads the right size.

:person_shrugging: