Hello all,
First of all I want to acknowledge there are multiple posts on the topic I haved checked, I will cite them throughout this post to track my tests.
My goal is to correctly calibrate a stereo camera, I have been able to calibrate each camera separately but fail using stereoCalibrate(). The camera has a wide fov (125º), so I am using the fiseye model since its the one that got me better results for the single camera calibration step.
Basically the main error I am encoutering is the following:
modules\calib3d\src\fisheye.cpp:1453: error: (-215:Assertion failed) fabs(norm_u1) > 0 in function 'cv::internal::InitExtrinsics'
there are multiple threads talking about this error, as well as posts in GitHub and other pages. Apparently this is common in two situations:
- The calibration images are to close to the border so some corners get cut off and produce errors. To discard this images I ussed the approach I found in the stereopi-fisheye-robot repository (I cant put more than 2 links)
- The error persisted, so I researched a bit more and found this post claiming the same error was happening when some chessboard poitns got flipped, despite this not being apparent in my case I incuded the solution in the thread.
- To avoid further point flipping I also tested ChArUco calibration and boards, but It led me to the same errors. Note that I have attempted multiple times to re-take calibration images. I also changed my chessboard pattern for it not to be square (meaning different number of squares on each axis) to avoid confussion during the matching.
- I have also read this thread where some initialization problems with the intrinsics are discussed, I have attempted to set an initial guess since I have the camera parameters (focal length and baseline), but I guet an ill condicioned matrix error.
- The previous thread also suggests the use of the cv2.fisheye.CALIB_RECOMPUTE_EXTRINSIC flag, which I also tested and fixed the error, however the results are really bad.
Here are some examples of images:
Valid image (I’ve censored everything non-relevant) on top and rectified image
Here is my (rectangular) chessboad code, I can provide the charuco version also if needed.
import cv2
import numpy as np
import os, sys
import glob
# Chessboard parameters
chessboard_size = (6, 7)
square_size = 42.5
frame_size = (1280, 720)
# Directories
images_dir = "../images_generation/Chessboard_Big_rect2"
image_dir_L = f"{images_dir}/images_separated_L"
image_dir_R = f"{images_dir}/images_separated_R"
output_dir = "params_chessboard_fisheye_stereo2"
os.makedirs(output_dir, exist_ok=True)
debug_dir = "debug"
debug_stereo = "debug_stereo"
os.makedirs(debug_dir, exist_ok=True)
os.makedirs(debug_stereo, exist_ok=True)
# Prepare object points
objp = np.zeros((chessboard_size[0] * chessboard_size[1], 1, 3), np.float32)
objp[:, 0, :2] = np.mgrid[0:chessboard_size[0], 0:chessboard_size[1]].T.reshape(-1, 2) * square_size
objpoints = [] # 3D points
imgpoints_L = [] # 2D points for left camera
imgpoints_R = [] # 2D points for right camera
# Get image files
imagesL = sorted(glob.glob(f"{image_dir_L}/*.png"))
imagesR = sorted(glob.glob(f"{image_dir_R}/*.png"))
loadedX, loadedY = cv2.imread(imagesL[0]).shape[1], cv2.imread(imagesL[0]).shape[0]
if len(imagesL) != len(imagesR) or len(imagesL) == 0 or len(imagesR) == 0:
print("Error: Mismatch in number of left and right images.")
exit()
# Find chessboard corners
for i, (imgL_path, imgR_path) in enumerate(zip(imagesL, imagesR)):
imgL = cv2.imread(imgL_path)
imgR = cv2.imread(imgR_path)
grayL = cv2.cvtColor(imgL, cv2.COLOR_BGR2GRAY)
grayR = cv2.cvtColor(imgR, cv2.COLOR_BGR2GRAY)
retL, cornersL = cv2.findChessboardCorners(grayL, chessboard_size, None)
retR, cornersR = cv2.findChessboardCorners(grayR, chessboard_size, None)
if retL and retR:
minRx, maxRx = cornersR[:, :, 0].min(), cornersR[:, :, 0].max()
minRy, maxRy = cornersR[:, :, 1].min(), cornersR[:, :, 1].max()
minLx, maxLx = cornersL[:, :, 0].min(), cornersL[:, :, 0].max()
minLy, maxLy = cornersL[:, :, 1].min(), cornersL[:, :, 1].max()
border_threshold_x = loadedX / 7 # higher less agressive
border_threshold_y = loadedY / 7
x_thresh_bad = (minRx < border_threshold_x or minLx < border_threshold_x)
y_thresh_bad = (minRy < border_threshold_y or minLy < border_threshold_y)
# Debug drawing
imgL_debug = imgL.copy()
imgR_debug = imgR.copy()
cv2.drawChessboardCorners(imgL_debug, chessboard_size, cornersL, retL)
cv2.drawChessboardCorners(imgR_debug, chessboard_size, cornersR, retR)
img_debug = np.hstack((imgL_debug, imgR_debug))
if x_thresh_bad or y_thresh_bad:
print("Chessboard too close to the side! Image ignored")
cv2.imwrite(f"{debug_dir}/rejected_{i}.png", img_debug)
continue
cv2.imwrite(f"{debug_dir}/detected_{i}.png", img_debug)
# Refine corners for better accuracy
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.1)
cornersL = cv2.cornerSubPix(grayL, cornersL, (11, 11), (-1, -1), criteria)
cornersR = cv2.cornerSubPix(grayR, cornersR, (11, 11), (-1, -1), criteria)
diff = cornersL - cornersR
lengths = np.linalg.norm(diff[:, :, 1], axis=-1)
total_diff = np.sum(lengths, axis=0)
if total_diff > 2000.0:
print(f"THIS STEREO PAIR IS BROKEN!!! Diff is: {total_diff}")
cornersR = np.flipud(cornersR) # Flip checkerboard to correct it
objpoints.append(objp)
imgpoints_L.append(cornersL)
imgpoints_R.append(cornersR)
print(f"Found {len(objpoints)} valid image pairs for calibration")
calibration_flags = (
cv2.fisheye.CALIB_RECOMPUTE_EXTRINSIC
# + cv2.fisheye.CALIB_CHECK_COND
# + cv2.fisheye.CALIB_FIX_SKEW
)
K_L = np.zeros((3, 3))
D_L = np.zeros((4, 1))
K_R = np.zeros((3, 3))
D_R = np.zeros((4, 1))
R = np.eye(3, dtype=np.float64)
T = np.zeros((3, 1), dtype=np.float64)
# # Added guesses
# K_L = np.zeros((3, 3))
# D_L = np.zeros((4, 1))
# K_R = np.zeros((3, 3))
# D_R = np.zeros((4, 1))
# R = np.eye(3, dtype=np.float64)
# # Initialize focal length (assuming square pixels)
# pixel_size = 3e-6 # 3um / px
# focal_length = 2.4e-3 / pixel_size
# K_L[0, 0] = K_L[1, 1] = focal_length
# K_R[0, 0] = K_R[1, 1] = focal_length
# K_L[2, 2] = K_R[2, 2] = 1 # Homogeneous coordinates
# # Set baseline (assuming translation along the x-axis)
# T = np.array([[baseline], [0], [0]], dtype=np.float64)
ret, KL, DL, KR, DR, R, T, E, F = cv2.fisheye.stereoCalibrate(
objpoints, imgpoints_L, imgpoints_R,
K_L, D_L,
K_R, D_R,
frame_size,
R, T,
flags=calibration_flags,
criteria=(cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 1e-6)
)
# Save parameters
np.save(os.path.join(output_dir, "K_L.npy"), KL)
np.save(os.path.join(output_dir, "D_L.npy"), DL)
np.save(os.path.join(output_dir, "K_R.npy"), KR)
np.save(os.path.join(output_dir, "D_R.npy"), DR)
np.save(os.path.join(output_dir, "R.npy"), R)
np.save(os.path.join(output_dir, "T.npy"), T)
np.save(os.path.join(output_dir, "E.npy"), E)
np.save(os.path.join(output_dir, "F.npy"), F)
print("\nCalibration parameters saved successfully.")
# Stereo rectify images for debugging purposes
for i, (imgL_path, imgR_path) in enumerate(zip(imagesL, imagesR)):
imgL = cv2.imread(imgL_path)
imgR = cv2.imread(imgR_path)
grayL = cv2.cvtColor(imgL, cv2.COLOR_BGR2GRAY)
grayR = cv2.cvtColor(imgR, cv2.COLOR_BGR2GRAY)
R_L, R_R, P_L, P_R, Q, validRoi_L, validRoi_R = cv2.fisheye.stereoRectify(
KL, DL, KR, DR, frame_size, R, T
)
xmapL, ymapL = cv2.fisheye.initUndistortRectifyMap(KL, DL, R_L, P_L, frame_size, cv2.CV_16SC2)
xmapR, ymapR = cv2.fisheye.initUndistortRectifyMap(KR, DR, R_R, P_R, frame_size, cv2.CV_16SC2)
rectifiedL = cv2.remap(grayL, xmapL, ymapL, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT)
rectifiedR = cv2.remap(grayR, xmapR, ymapR, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT)
combined_rect = np.hstack((rectifiedL, rectifiedR))
cv2.imwrite(f"{debug_stereo}/stereo_rectified_{i}.png", combined_rect)
Thanks,
J