Image rotation using AR tags

Android/Java
1

I am trying to rotate an image with reference to an AR tag. I need the AR tag to be in the top right corner of the white box and the full image to be rotated accordingly like the image below.

image2 (2)
image2 (2)1200×1000 69.4 KB

protected Mat processImage(Mat image){
//detect AR
Dictionary dictionary = Aruco.getPredefinedDictionary(Aruco.DICT_5X5_250);
List corners = new ArrayList<>();
Mat markerIds = new Mat();
Aruco.detectMarkers(image, dictionary, corners, markerIds);

    //Get camera matrix
    Mat cameraMatrix = new Mat(3,3, CvType.CV_64F);
    cameraMatrix.put(0,0,api.getNavCamIntrinsics()[0]);
    Mat cameraCoefficients = new Mat(1,5, CvType.CV_64F);
    cameraCoefficients.put(0,0,api.getNavCamIntrinsics()[1]);
    cameraCoefficients.convertTo(cameraCoefficients,CvType.CV_64F);

    //Undistort image
    Mat undistortedImage = new Mat();
    Calib3d.undistort(image, undistortedImage, cameraMatrix, cameraCoefficients);

    if(!markerIds.empty()){
        // Get the first detected marker's corners
        MatOfPoint2f markerCorners = new MatOfPoint2f(corners.get(0).clone());

        // Top-left and top-right corners of the marker
        org.opencv.core.Point[] cornerPoints = markerCorners.toArray();
        org.opencv.core.Point topLeft = cornerPoints[0];
        org.opencv.core.Point topRight = cornerPoints[1];

        // Calculate the angle to rotate
        double deltaX = topRight.x - topLeft.x;
        double deltaY = topRight.y - topLeft.y;
        double angle = Math.toDegrees(Math.atan2(deltaY, deltaX));

        // Print the angle
        Log.i(TAG, "Rotation angle: " + angle);

        // Rotate the image
        Mat rotatedImage = rotateImage(undistortedImage, angle);
        return rotatedImage;
    }
    Log.i(TAG, "no AR tag");
    return undistortedImage;
}

private Mat rotateImage(Mat image, double angle) {
    // Get the image center
    org.opencv.core.Point center = new org.opencv.core.Point(image.width() / 2.0, image.height() / 2.0);

    // Get the rotation matrix
    Mat rotationMatrix = Imgproc.getRotationMatrix2D(center, angle, 1.0);

    // Determine the bounding box size after rotation
    Rect bbox = new RotatedRect(center, image.size(), angle).boundingRect();

    // Adjust the rotation matrix to take into account translation
    rotationMatrix.put(0, 2, rotationMatrix.get(0, 2)[0] + bbox.width / 2.0 - center.x);
    rotationMatrix.put(1, 2, rotationMatrix.get(1, 2)[0] + bbox.height / 2.0 - center.y);

    // Rotate the image
    Mat rotatedImage = new Mat();
    Imgproc.warpAffine(image, rotatedImage, rotationMatrix, bbox.size());

    return rotatedImage;
}

This code sometimes works (like the image above) but there are images that are very far off.
How can I fix this issue?