Problems with fisheye stereo calibration

Hello all,

First of all I want to acknowledge there are multiple posts on the topic I haved checked, I will cite them throughout this post to track my tests.

My goal is to correctly calibrate a stereo camera, I have been able to calibrate each camera separately but fail using stereoCalibrate(). The camera has a wide fov (125º), so I am using the fiseye model since its the one that got me better results for the single camera calibration step.

Basically the main error I am encoutering is the following:

modules\calib3d\src\fisheye.cpp:1453: error: (-215:Assertion failed) fabs(norm_u1) > 0 in function 'cv::internal::InitExtrinsics'

there are multiple threads talking about this error, as well as posts in GitHub and other pages. Apparently this is common in two situations:

  • The calibration images are to close to the border so some corners get cut off and produce errors. To discard this images I ussed the approach I found in the stereopi-fisheye-robot repository (I cant put more than 2 links)
  • The error persisted, so I researched a bit more and found this post claiming the same error was happening when some chessboard poitns got flipped, despite this not being apparent in my case I incuded the solution in the thread.
  • To avoid further point flipping I also tested ChArUco calibration and boards, but It led me to the same errors. Note that I have attempted multiple times to re-take calibration images. I also changed my chessboard pattern for it not to be square (meaning different number of squares on each axis) to avoid confussion during the matching.
  • I have also read this thread where some initialization problems with the intrinsics are discussed, I have attempted to set an initial guess since I have the camera parameters (focal length and baseline), but I guet an ill condicioned matrix error.
  • The previous thread also suggests the use of the cv2.fisheye.CALIB_RECOMPUTE_EXTRINSIC flag, which I also tested and fixed the error, however the results are really bad.

Here are some examples of images:

Valid image (I’ve censored everything non-relevant) on top and rectified image

Here is my (rectangular) chessboad code, I can provide the charuco version also if needed.

import cv2
import numpy as np
import os, sys
import glob

# Chessboard parameters
chessboard_size = (6, 7)
square_size = 42.5

frame_size = (1280, 720)

# Directories
images_dir = "../images_generation/Chessboard_Big_rect2"
image_dir_L = f"{images_dir}/images_separated_L"
image_dir_R = f"{images_dir}/images_separated_R"
output_dir = "params_chessboard_fisheye_stereo2"
os.makedirs(output_dir, exist_ok=True)

debug_dir = "debug"
debug_stereo = "debug_stereo"
os.makedirs(debug_dir, exist_ok=True)
os.makedirs(debug_stereo, exist_ok=True)

# Prepare object points
objp = np.zeros((chessboard_size[0] * chessboard_size[1], 1, 3), np.float32)
objp[:, 0, :2] = np.mgrid[0:chessboard_size[0], 0:chessboard_size[1]].T.reshape(-1, 2) * square_size

objpoints = []  # 3D points
imgpoints_L = []  # 2D points for left camera
imgpoints_R = []  # 2D points for right camera

# Get image files
imagesL = sorted(glob.glob(f"{image_dir_L}/*.png"))
imagesR = sorted(glob.glob(f"{image_dir_R}/*.png"))

loadedX, loadedY = cv2.imread(imagesL[0]).shape[1], cv2.imread(imagesL[0]).shape[0]

if len(imagesL) != len(imagesR) or len(imagesL) == 0 or len(imagesR) == 0:
    print("Error: Mismatch in number of left and right images.")
    exit()

# Find chessboard corners
for i, (imgL_path, imgR_path) in enumerate(zip(imagesL, imagesR)):
    imgL = cv2.imread(imgL_path)
    imgR = cv2.imread(imgR_path)
    grayL = cv2.cvtColor(imgL, cv2.COLOR_BGR2GRAY)
    grayR = cv2.cvtColor(imgR, cv2.COLOR_BGR2GRAY)

    retL, cornersL = cv2.findChessboardCorners(grayL, chessboard_size, None)
    retR, cornersR = cv2.findChessboardCorners(grayR, chessboard_size, None)

    if retL and retR:

        minRx, maxRx = cornersR[:, :, 0].min(), cornersR[:, :, 0].max()
        minRy, maxRy = cornersR[:, :, 1].min(), cornersR[:, :, 1].max()
        minLx, maxLx = cornersL[:, :, 0].min(), cornersL[:, :, 0].max()
        minLy, maxLy = cornersL[:, :, 1].min(), cornersL[:, :, 1].max()
        
        border_threshold_x = loadedX / 7 # higher less agressive
        border_threshold_y = loadedY / 7
        
        x_thresh_bad = (minRx < border_threshold_x or minLx < border_threshold_x)
        y_thresh_bad = (minRy < border_threshold_y or minLy < border_threshold_y)

        # Debug drawing
        imgL_debug = imgL.copy()
        imgR_debug = imgR.copy()
        cv2.drawChessboardCorners(imgL_debug, chessboard_size, cornersL, retL)
        cv2.drawChessboardCorners(imgR_debug, chessboard_size, cornersR, retR)
        img_debug = np.hstack((imgL_debug, imgR_debug))

        if x_thresh_bad or y_thresh_bad:
            print("Chessboard too close to the side! Image ignored")
            cv2.imwrite(f"{debug_dir}/rejected_{i}.png", img_debug)
            continue

        cv2.imwrite(f"{debug_dir}/detected_{i}.png", img_debug)

        # Refine corners for better accuracy
        criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.1)
        cornersL = cv2.cornerSubPix(grayL, cornersL, (11, 11), (-1, -1), criteria)
        cornersR = cv2.cornerSubPix(grayR, cornersR, (11, 11), (-1, -1), criteria)
        diff = cornersL - cornersR
        lengths = np.linalg.norm(diff[:, :, 1], axis=-1)
        total_diff = np.sum(lengths, axis=0)

        if total_diff > 2000.0:
            print(f"THIS STEREO PAIR IS BROKEN!!! Diff is: {total_diff}")
            cornersR = np.flipud(cornersR)  # Flip checkerboard to correct it

        objpoints.append(objp)
        imgpoints_L.append(cornersL)
        imgpoints_R.append(cornersR)

print(f"Found {len(objpoints)} valid image pairs for calibration")

calibration_flags = (
    cv2.fisheye.CALIB_RECOMPUTE_EXTRINSIC 
    # + cv2.fisheye.CALIB_CHECK_COND 
    # + cv2.fisheye.CALIB_FIX_SKEW
)

K_L = np.zeros((3, 3))
D_L = np.zeros((4, 1))
K_R = np.zeros((3, 3))
D_R = np.zeros((4, 1))
R = np.eye(3, dtype=np.float64)
T = np.zeros((3, 1), dtype=np.float64)

# # Added guesses
# K_L = np.zeros((3, 3))
# D_L = np.zeros((4, 1))
# K_R = np.zeros((3, 3))
# D_R = np.zeros((4, 1))
# R = np.eye(3, dtype=np.float64)

# # Initialize focal length (assuming square pixels)
# pixel_size = 3e-6 # 3um / px
# focal_length = 2.4e-3 / pixel_size
# K_L[0, 0] = K_L[1, 1] = focal_length
# K_R[0, 0] = K_R[1, 1] = focal_length
# K_L[2, 2] = K_R[2, 2] = 1  # Homogeneous coordinates

# # Set baseline (assuming translation along the x-axis)
# T = np.array([[baseline], [0], [0]], dtype=np.float64)

ret, KL, DL, KR, DR, R, T, E, F = cv2.fisheye.stereoCalibrate(
    objpoints, imgpoints_L, imgpoints_R,
    K_L, D_L,
    K_R, D_R,
    frame_size,
    R, T,
    flags=calibration_flags,
    criteria=(cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 1e-6)
)

# Save parameters
np.save(os.path.join(output_dir, "K_L.npy"), KL)
np.save(os.path.join(output_dir, "D_L.npy"), DL)
np.save(os.path.join(output_dir, "K_R.npy"), KR)
np.save(os.path.join(output_dir, "D_R.npy"), DR)
np.save(os.path.join(output_dir, "R.npy"), R)
np.save(os.path.join(output_dir, "T.npy"), T)
np.save(os.path.join(output_dir, "E.npy"), E)
np.save(os.path.join(output_dir, "F.npy"), F)

print("\nCalibration parameters saved successfully.")

# Stereo rectify images for debugging purposes
for i, (imgL_path, imgR_path) in enumerate(zip(imagesL, imagesR)):
    imgL = cv2.imread(imgL_path)
    imgR = cv2.imread(imgR_path)
    grayL = cv2.cvtColor(imgL, cv2.COLOR_BGR2GRAY)
    grayR = cv2.cvtColor(imgR, cv2.COLOR_BGR2GRAY)

    R_L, R_R, P_L, P_R, Q, validRoi_L, validRoi_R = cv2.fisheye.stereoRectify(
        KL, DL, KR, DR, frame_size, R, T
    )

    xmapL, ymapL = cv2.fisheye.initUndistortRectifyMap(KL, DL, R_L, P_L, frame_size, cv2.CV_16SC2)
    xmapR, ymapR = cv2.fisheye.initUndistortRectifyMap(KR, DR, R_R, P_R, frame_size, cv2.CV_16SC2)

    rectifiedL = cv2.remap(grayL, xmapL, ymapL, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT)
    rectifiedR = cv2.remap(grayR, xmapR, ymapR, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT)

    combined_rect = np.hstack((rectifiedL, rectifiedR))
    cv2.imwrite(f"{debug_stereo}/stereo_rectified_{i}.png", combined_rect)

Thanks,

J

I am adding additional comments on my post, but not marking it as resolved since I dont think it really is. As most of the posts with simmilar problems as the ones I described in my OP.

I was able to obtain “good” looking rectified images afted selection calibration pictures one by one, for some reason good looking (and good corner-detected) calibration images mess up with the end result big time. Still the calibration is not perfect but I assume I will be able to obtain better results by taking more calibration images and nit-picking them…

I will post my results and findings.

J