Issues in my portal game implementation (stencil buffer)

Guys I need help i’ve been working on this a lot but nothing solved the problem

class Portal {
    constructor(scene, renderer) {
        this.scene = scene;
        this.renderer = renderer;
        this.tmpScene = new THREE.Scene();

        this.rotationYMatrix = new THREE.Matrix4().makeRotationY(Math.PI);
        this.inverse = new THREE.Matrix4();
        this.dstInverse = new THREE.Matrix4();
        this.srcToCam = new THREE.Matrix4();
        this.srcToDst = new THREE.Matrix4();
        this.result = new THREE.Matrix4();

        this.dstRotationMatrix = new THREE.Matrix4();
        this.normal = new THREE.Vector3();
        this.clipPlane = new THREE.Plane();
        this.clipVector = new THREE.Vector4();
        this.q = new THREE.Vector4();
        this.projectionMatrix = new THREE.Matrix4();
        this.cameraInverseViewMat = new THREE.Matrix4();

        this.originalCameraMatrixWorld = new THREE.Matrix4();
        this.originalCameraProjectionMatrix = new THREE.Matrix4();

        this.maxRecursion = 2;
    }

    renderScene(camera, children) {
        this.tmpScene.children = children;
        this.renderer.render(this.tmpScene, camera);
    }

    calculateObliqueMatrix(projMatrix, clipPlane) {
        const q = new THREE.Vector4(
            (Math.sign(clipPlane.x) + projMatrix.elements[8]) / projMatrix.elements[0],
            (Math.sign(clipPlane.y) + projMatrix.elements[9]) / projMatrix.elements[5],
            -1.0,
            (1.0 + projMatrix.elements[10]) / projMatrix.elements[14]
        );
    
        const c = clipPlane.multiplyScalar(2.0 / clipPlane.dot(q));
    
        projMatrix.elements[2] = c.x;
        projMatrix.elements[6] = c.y;
        projMatrix.elements[10] = c.z + 1.0;
        projMatrix.elements[14] = c.w;
    
        return projMatrix;
    }

    render(camera, recursionLevel = 0, virtualCamera, portals) {
        if (recursionLevel > this.maxRecursion) return;

        const gl = this.renderer.getContext();

        for (let i = 0; i < portals.length; i++) {
            let portal = portals[i];

            gl.colorMask(false, false, false, false);
            gl.depthMask(false);
            gl.disable(gl.DEPTH_TEST);
            gl.enable(gl.STENCIL_TEST);
            gl.stencilFunc(gl.NOTEQUAL, recursionLevel, 0xff);
            gl.stencilOp(gl.INCR, gl.KEEP, gl.KEEP);
            gl.stencilMask(0xff);

            this.renderScene(camera, [portal, camera]);

            // Get the inverse of the inPortal's transformation matrix
            const inverseInMatrix = new THREE.Matrix4().copy(portal.matrixWorld).invert();

            // Calculate the relative position of the main object to the portal
            const relativePos = new THREE.Vector3().copy(camera.position).applyMatrix4(inverseInMatrix);

            // Rotate the relative position by 180 degrees around the Y-axis
            relativePos.applyAxisAngle(new THREE.Vector3(0, 1, 0), Math.PI);

            // Apply the portal.pair's transformation matrix to get the new camera position
            const newCameraPos = relativePos.applyMatrix4(portal.pair.matrixWorld);
            virtualCamera.position.copy(newCameraPos);

            // Get the relative rotation of the main object to the portal
            const inverseInQuat = portal.quaternion.clone().invert();
            const relativeRot = camera.quaternion.clone().premultiply(inverseInQuat);

            // Rotate the relative rotation by 180 degrees around the Y-axis
            const rot180Y = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 1, 0), Math.PI);
            relativeRot.premultiply(rot180Y);

            // Apply the portal.pair's rotation to get the new camera rotation
            const newCameraRot = portal.pair.quaternion.clone().multiply(relativeRot);
            virtualCamera.quaternion.copy(newCameraRot);

            // Update camera matrices
            virtualCamera.updateMatrixWorld(true);
            
            // Create the Plane object in world space
            const pNormal = new THREE.Vector3().copy(portal.pair.getWorldDirection(new THREE.Vector3()));
            const pPosition = new THREE.Vector3().copy(portal.pair.position);
            const plane = new THREE.Plane().setFromNormalAndCoplanarPoint(pNormal, pPosition);

            // Convert Plane to Vector4
            const clipPlane = new THREE.Vector4(plane.normal.x, plane.normal.y, plane.normal.z, plane.constant);

            // Transform the clip plane into camera space
            const viewMatrix = new THREE.Matrix4().copy(virtualCamera.matrixWorldInverse);
            const clipPlaneCameraSpace = clipPlane.applyMatrix4(viewMatrix).normalize();

            const mainCamera = camera;
            const mainCameraProjectionMatrix = mainCamera.projectionMatrix.clone();

            // Calculate the new oblique projection matrix
            const newProjectionMatrix = this.calculateObliqueMatrix(mainCameraProjectionMatrix, clipPlaneCameraSpace);

            // Apply the new projection matrix to the portal camera
            virtualCamera.projectionMatrix.copy(newProjectionMatrix);

            if (recursionLevel === this.maxRecursion) {
                gl.colorMask(true, true, true, true);
                gl.depthMask(true);
                this.renderer.clear(false, true, false);
                gl.enable(gl.DEPTH_TEST);
                gl.enable(gl.STENCIL_TEST);
                gl.stencilMask(0x00);
                gl.stencilFunc(gl.EQUAL, recursionLevel + 1, 0xff);
                this.renderScene(virtualCamera, this.scene.children);
            } else {
                this.render(virtualCamera, recursionLevel + 1, virtualCamera, [portal, portal.pair]);
            }

            gl.colorMask(false, false, false, false);
            gl.depthMask(false);
            gl.enable(gl.STENCIL_TEST);
            gl.stencilMask(0xff);
            gl.stencilFunc(gl.NOTEQUAL, recursionLevel + 1, 0xFF);
            gl.stencilOp(gl.DECR, gl.KEEP, gl.KEEP);

            this.renderScene(camera, [portal, camera]);
        }

        gl.disable(gl.STENCIL_TEST);
        gl.stencilMask(0x00);
        gl.colorMask(false, false, false, false);
        gl.enable(gl.DEPTH_TEST);
        gl.depthMask(true);
        gl.depthFunc(gl.ALWAYS);
        this.renderer.clear(false, true, false);

        this.renderScene(camera, [...portals, camera]);

        gl.depthFunc(gl.LESS);
        gl.enable(gl.STENCIL_TEST);
        gl.stencilMask(0x00);
        gl.stencilFunc(gl.LEQUAL, recursionLevel, 0xff);
        gl.colorMask(true, true, true, true);
        gl.depthMask(true);
        gl.enable(gl.DEPTH_TEST);

        this.renderScene(camera, this.scene.children);
    }
}

Maybe like decal:

depthTest: true,
depthWrite: false,
polygonOffset: true,
polygonOffsetFactor: - 4,
2 Likes