FitToBox's behavior for objects that have rotation

Trying to get the camera to always look at the front face of my cube with the proper orientation using cameracontrols FitToBox, but it looks like FitToBox is trying to line up against one of the scene’s axis, which means that the camera won’t look straight at the cube is rotated in the scene. Is there a way to prevent FitToBox from aligning against a scene axis?

Sample code (everything works ok if the random rotations are removed):

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Three.js Camera Directly Facing Cube with Camera Controls</title>
    <style>
        body { margin: 0; overflow: hidden; }
        canvas { display: block; }
        #buttonContainer {
            position: absolute;
            top: 10px;
            left: 10px;
            z-index: 100;
        }
    </style>
</head>
<body>
    <div id="buttonContainer">
        <button id="adjustCameraButton">Adjust Camera</button>
    </div>

    <script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
    <script src="https://cdn.jsdelivr.net/npm/camera-controls@1.33.0/dist/camera-controls.js"></script>

    <script>
        // Setup scene, camera, renderer
        const scene = new THREE.Scene();
        const camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 1000);
        const renderer = new THREE.WebGLRenderer();
        renderer.setSize(window.innerWidth, window.innerHeight);
        document.body.appendChild(renderer.domElement);

        // Initialize CameraControls
        CameraControls.install({ THREE });
        const clock = new THREE.Clock();
        const cameraControls = new CameraControls(camera, renderer.domElement);

        // Move the camera further back initially
        camera.position.set(10, 10, 10); // Adjusted initial camera position
        camera.lookAt(0, 0, 0);

        // Randomly rotate the cube
        function getRandomRotation() {
            return Math.random() * Math.PI * 2; // Random rotation between 0 and 2π radians
        }

        // Create a material with different color for the front face
        const materials = [
            new THREE.MeshBasicMaterial({ color: 0x00ff00, wireframe: true }), // Left face
            new THREE.MeshBasicMaterial({ color: 0x00ff00, wireframe: true }), // Right face
            new THREE.MeshBasicMaterial({ color: 0x00ff00, wireframe: true }), // Top face
            new THREE.MeshBasicMaterial({ color: 0x00ff00, wireframe: true }), // Bottom face
            new THREE.MeshBasicMaterial({ color: 0xff0000, transparent: true, opacity: 0.3 }), // Front face (red and semi-transparent)
            new THREE.MeshBasicMaterial({ color: 0x00ff00, wireframe: true })  // Back face
        ];

        // Create parent object and child mesh (with a front face colored red)
        const parentObject = new THREE.Object3D();
        scene.add(parentObject);

        const childMesh = new THREE.Mesh(
            new THREE.BoxGeometry(1, 1, 1),
            materials
        );
        parentObject.add(childMesh);

        // Apply random rotations to the cube
        parentObject.rotation.set(getRandomRotation(), getRandomRotation(), getRandomRotation());
        childMesh.rotation.set(getRandomRotation(), getRandomRotation(), getRandomRotation());

        // Add AxesHelper to visualize the orientation of the objects (small axes)
        const axesHelper = new THREE.AxesHelper(0.2);
        scene.add(axesHelper);

        // Add lights
        const light = new THREE.DirectionalLight(0xffffff, 1);
        light.position.set(10, 10, 10).normalize();
        scene.add(light);

        // Function to adjust the camera using camera-controls (first look at, then fit to box)
        function adjustCameraToFaceFrontWithControls(object, paddingFactor = 1.5) {
            // Get the object's world quaternion (rotation)
            const quaternion = object.getWorldQuaternion(new THREE.Quaternion());

            // Move the camera directly in front of the object's local Z-axis
            const frontDirection = new THREE.Vector3(0, 0, 1); // Local Z-axis (front)
            frontDirection.applyQuaternion(quaternion); // Apply the object's rotation

            // Get the object's local up direction (Y-axis)
            const upDirection = new THREE.Vector3(0, 1, 0); // Local Y-axis (up)
            upDirection.applyQuaternion(quaternion); // Apply the object's rotation

            // Get the object's world position (center)
            const center = new THREE.Vector3();
            object.getWorldPosition(center);

            // Calculate the distance to place the camera
            const boundingBox = new THREE.Box3().setFromObject(object);
            const size = boundingBox.getSize(new THREE.Vector3());
            const maxDim = Math.max(size.x, size.y, size.z);
            const fov = camera.fov * (Math.PI / 180); // Convert FOV to radians
            let cameraDistance = maxDim / (2 * Math.tan(fov / 2)); // Adjust distance based on FOV
            cameraDistance *= paddingFactor; // Add padding

            // Calculate the camera's new position based on the object's front
            const cameraPosition = new THREE.Vector3().copy(center).add(frontDirection.multiplyScalar(cameraDistance));

            // Set camera controls' target and position with a smooth transition (1-second duration)
            cameraControls.setLookAt(
                cameraPosition.x, cameraPosition.y, cameraPosition.z,  // Camera's new position
                center.x, center.y, center.z,                          // Look at the center of the object
                true // Enable transition
            ).then(() => {
                // Once the camera has looked at the object, fit it into view with fitToBox
                cameraControls.fitToBox(object, true);
            });

            // Align camera's up direction with the object's local up direction
            camera.up.copy(upDirection);

            // Update the camera projection matrix to reflect the changes
            camera.updateProjectionMatrix();
        }

        // Button to adjust the camera
        document.getElementById('adjustCameraButton').addEventListener('click', function() {
            adjustCameraToFaceFrontWithControls(childMesh);
        });

        // Render loop
        function animate() {
            const delta = clock.getDelta();
            cameraControls.update(delta); // Update the camera controls
            renderer.render(scene, camera);
            requestAnimationFrame(animate);
        }
        animate();
    </script>
</body>
</html>