Does anybody know of any good articles or example code where stereo calibration is done on a fisheye lens? Does stereo calibration needs to be run after chessboard calibration before creating calibration matrices or on the fly when running undistort methods? I appreciate any help and suggestions
I have successfully calibrated single fisheye lens and run undistorting script on the sample image to stretch out all the corners. In the next step I modified the code to run same calibration for both lenses then I did fisheye stereoCalibrate and stereoRectify. Unfortunately I’m having hard time locating where I made a mistake that leads to this warped image pairs, I apprieciate any help
import numpy as np
import cv2 as cv
import glob
####@@@@#### Detect chessboard corners ####@@@@####
CHECKERBOARD = (9,6)
frameSize = (1920,2160)
calibration_flags = cv.fisheye.CALIB_RECOMPUTE_EXTRINSIC + cv.fisheye.CALIB_CHECK_COND + cv.fisheye.CALIB_FIX_SKEW
# termination criteria
# term_criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)
subpix_criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.1)
# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
# 1
# objp = np.zeros((CHECKERBOARD[0] * CHECKERBOARD[1], 3), np.float64)
# objp[:,:2] = np.mgrid[0:CHECKERBOARD[0],0:CHECKERBOARD[1]].T.reshape(-1,2)
# 2
# objp = np.zeros((1, CHECKERBOARD[0]*CHECKERBOARD[1], 3), np.float64)
# objp[0,:,:2] = np.mgrid[0:CHECKERBOARD[0], 0:CHECKERBOARD[1]].T.reshape(-1, 2)
# 3
objp = np.zeros( (CHECKERBOARD[0]*CHECKERBOARD[1], 1, 3) , np.float64)
objp[:,0,:2] = np.mgrid[0:CHECKERBOARD[0], 0:CHECKERBOARD[1]].T.reshape(-1, 2)
# size_of_chessboard_squares_mm = 20
# objp = objp * size_of_chessboard_squares_mm
# Arrays to store object points and image points from all the images.
objpoints = [] # 3d point in real world space
imgpointsL = [] # 2d points in image plane.
imgpointsR = [] # 2d points in image plane.
imagesLeft = sorted(glob.glob('images/left/*.jpg'))
imagesRight = sorted(glob.glob('images/right/*.jpg'))
for imgLeft, imgRight in zip(imagesLeft, imagesRight):
imgL = cv.imread(imgLeft)
imgR = cv.imread(imgRight)
grayL = cv.cvtColor(imgL, cv.COLOR_BGR2GRAY)
grayR = cv.cvtColor(imgR, cv.COLOR_BGR2GRAY)
# Find the chess board corners
# retL, cornersL = cv.findChessboardCorners(grayL, chessboardSize, None)
retL, cornersL = cv.findChessboardCorners(grayL, CHECKERBOARD, cv.CALIB_CB_ADAPTIVE_THRESH+cv.CALIB_CB_FAST_CHECK+cv.CALIB_CB_NORMALIZE_IMAGE)
# retR, cornersR = cv.findChessboardCorners(grayR, chessboardSize, None)
retR, cornersR = cv.findChessboardCorners(grayR, CHECKERBOARD, cv.CALIB_CB_ADAPTIVE_THRESH+cv.CALIB_CB_FAST_CHECK+cv.CALIB_CB_NORMALIZE_IMAGE)
# If found, add object points, image points (after refining them)
if retL and retR == True:
objpoints.append(objp)
# cornersL = cv.cornerSubPix(grayL, cornersL, (12,11), (-1,-1), term_criteria)
cornersL = cv.cornerSubPix(grayL, cornersL, (3,3), (-1,-1), subpix_criteria)
imgpointsL.append(cornersL)
# cornersR = cv.cornerSubPix(grayR, cornersR, (11,11), (-1,-1), term_criteria)
cornersR = cv.cornerSubPix(grayR, cornersR, (3,3), (-1,-1), subpix_criteria)
imgpointsR.append(cornersR)
# Draw and display the corners
cv.drawChessboardCorners(imgL, CHECKERBOARD, cornersL, retL)
cv.imshow('img left', imgL)
cv.drawChessboardCorners(imgR, CHECKERBOARD, cornersR, retR)
cv.imshow('img right', imgR)
cv.waitKey(1000)
cv.destroyAllWindows()
####@@@@#### CALIBRATION ####@@@@####
N_OK = len(objpoints)
rvecs = [np.zeros((1, 1, 3), dtype=np.float64) for i in range(N_OK)]
tvecs = [np.zeros((1, 1, 3), dtype=np.float64) for i in range(N_OK)]
R = np.zeros((1, 1, 3), dtype=np.float64)
T = np.zeros((1, 1, 3), dtype=np.float64)
K_left = np.zeros((3, 3))
D_left = np.zeros((4, 1))
rms, _, _, _, _ = \
cv.fisheye.calibrate(
objpoints,
imgpointsL,
grayL.shape[::-1],
K_left,
D_left,
rvecs,
tvecs,
calibration_flags,
(cv.TERM_CRITERIA_EPS+cv.TERM_CRITERIA_MAX_ITER, 30, 1e-6)
)
K_right = np.zeros((3, 3))
D_right = np.zeros((4, 1))
rms, _, _, _, _ = \
cv.fisheye.calibrate(
objpoints,
imgpointsR,
grayL.shape[::-1],
K_right,
D_right,
rvecs,
tvecs,
calibration_flags,
(cv.TERM_CRITERIA_EPS+cv.TERM_CRITERIA_MAX_ITER, 30, 1e-6)
)
print("Found " + str(N_OK) + " valid images for calibration")
print("RMS=" + str(rms))
print("DIM=" + str(frameSize[::-1]))
print("K_left=np.array(" + str(K_left.tolist()) + ")")
print("D_left=np.array(" + str(D_left.tolist()) + ")")
print("K_right=np.array(" + str(K_right.tolist()) + ")")
print("D_right=np.array(" + str(D_right.tolist()) + ")")
print("calibrating both fisheye lenses")
####@@@@#### Stereo Calibration ####@@@@####
objpoints = np.array([objp]*len(imgpointsL), dtype=np.float64)
imgpointsL = np.asarray(imgpointsL, dtype=np.float64)
imgpointsR = np.asarray(imgpointsR, dtype=np.float64)
objpoints = np.reshape(objpoints, (N_OK, 1, CHECKERBOARD[0]*CHECKERBOARD[1], 3))
imgpointsL = np.reshape(imgpointsL, (N_OK, 1, CHECKERBOARD[0]*CHECKERBOARD[1], 2))
imgpointsR = np.reshape(imgpointsR, (N_OK, 1, CHECKERBOARD[0]*CHECKERBOARD[1], 2))
(rms, K1, D1, K2, D2, R, T) = cv.fisheye.stereoCalibrate(
objpoints,
imgpointsL,
imgpointsR,
K_left,
D_left,
K_right,
D_right,
grayL.shape[::-1],
R,
T,
calibration_flags
)
print("\nSTEREO RMS=" + str(rms))
print("K1=np.array(" + str(K1.tolist()) + ")")
print("D1=np.array(" + str(D1.tolist()) + ")")
print("K2=np.array(" + str(K2.tolist()) + ")")
print("D2=np.array(" + str(D2.tolist()) + ")")
# retL, cameraMatrixL, distL, rvecsL, tvecsL = cv.calibrateCamera(objpoints, imgpointsL, frameSize, None, None)
# heightL, widthL, channelsL = imgL.shape
# newCameraMatrixL, roi_L = cv.getOptimalNewCameraMatrix(cameraMatrixL, distL, (widthL, heightL), 1, (widthL, heightL))
# retR, cameraMatrixR, distR, rvecsR, tvecsR = cv.calibrateCamera(objpoints, imgpointsR, frameSize, None, None)
# heightR, widthR, channelsR = imgR.shape
# newCameraMatrixR, roi_R = cv.getOptimalNewCameraMatrix(cameraMatrixR, distR, (widthR, heightR), 1, (widthR, heightR))
# flags = 0
# flags |= cv.CALIB_FIX_INTRINSIC
# Here we fix the intrinsic camara matrixes so that only Rot, Trns, Emat and Fmat are calculated.
# Hence intrinsic parameters are the same
# criteria_stereo = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)
# This step is performed to transformation between the two cameras and calculate Essential and Fundamenatl matrix
# retStereo, newCameraMatrixL, distL, newCameraMatrixR, distR, rot, trans, essentialMatrix, fundamentalMatrix = cv.stereoCalibrate(objpoints, imgpointsL, imgpointsR, newCameraMatrixL, distL, newCameraMatrixR, distR, grayL.shape[::-1], criteria_stereo, flags)
####@@@@#### Stereo Rectification ####@@@@####
R1 = np.zeros([3,3])
R2 = np.zeros([3,3])
P1 = np.zeros([3,4])
P2 = np.zeros([3,4])
Q = np.zeros([4,4])
rectL, rectR, projMatrixL, projMatrixR, Q = cv.fisheye.stereoRectify(
K1,
D1,
K2,
D2,
grayL.shape[::-1],
R,
T,
cv.fisheye.CALIB_ZERO_DISPARITY,
# balance=0.5,
# fov_scale=0.6
R2, P1, P2, Q,
cv.CALIB_ZERO_DISPARITY, (0,0), 0, 0
)
stereoMapL = cv.fisheye.initUndistortRectifyMap(K_left, D_left, rectL, projMatrixL, grayL.shape[::-1], cv.CV_16SC2)
stereoMapR = cv.fisheye.initUndistortRectifyMap(K_right, D_right, rectR, projMatrixR, grayR.shape[::-1], cv.CV_16SC2)
print("Saving!")
cv_file = cv.FileStorage('stereoMap.xml', cv.FILE_STORAGE_WRITE)
cv_file.write('stereoMapL_x',stereoMapL[0])
cv_file.write('stereoMapL_y',stereoMapL[1])
cv_file.write('stereoMapR_x',stereoMapR[0])
cv_file.write('stereoMapR_y',stereoMapR[1])
cv_file.release()
how do you judge that?
calibration results please, and samples of the pictures used for calibration (overview of the set of thumbnails perhaps).
the matrix and distortion coefficients
turns out it was a bad calibration, I had first batch of images to calibrate (around 7) but at least 10 is needed so I made another 10 but this time chessboard was rotated 180 degs! This completely breaks the calibration results. I was getting RMS around 1+ for individual lens and 4.5 for stereo calibration! After removing bad images I’m getting RMS around 0.6 in both single and stereo calibration.
low reprojection error is necessary but not sufficient.
calibration pics need to show perspective foreshortening and they need to cover the corners of the view in particular.
without the first, the focal length is garbage.
without the second, the distortion coefficients are garbage.
I just discovered this strange behaviour, I made a mistake here
stereoMapL = cv.fisheye.initUndistortRectifyMap(K_left, D_left, R1, K_left, grayL.shape[::-1], cv.CV_16SC2)
stereoMapR = cv.fisheye.initUndistortRectifyMap(K_right, D_right, R2, K_left, grayR.shape[::-1], cv.CV_16SC2)
where I used K_left in both of the initUndistortRectifyMap (left and right) and the video turned out to be perfect (I think!) then I corrected the second line and used K_right instead of K_left what I thought should be correct broke the video to the point that watching it almost hurts my brain, any idea why?
DIM=(2160, 1920)
RMS_left=0.6349757465598553
RMS_right=0.6677081416035834
K_left=[[575.7047317639458, 0.0, 1079.9445516653223], [0.0, 576.0598980803733, 1080.485186341297], [0.0, 0.0, 1.0]]
D_left=[[0.013671545923773157], [-0.000773092565207666], [0.0033731035619686597], [-0.0011046345960252903]]
K_right=[[575.7828397557032, 0.0, 869.9354676964672], [0.0, 575.7822659178688, 1091.530528530801], [0.0, 0.0, 1.0]]
D_right=[[0.019173635688566008], [-0.008775671298404274], [0.007914251615504821], [-0.0020588786543884545]]
Calibrating both fisheye lenses
STEREO RMS=0.7349233418669257
K1=[[575.7047317639458, 0.0, 1079.9445516653223], [0.0, 576.0598980803733, 1080.485186341297], [0.0, 0.0, 1.0]]
D1=[[0.013671545923773157], [-0.000773092565207666], [0.0033731035619686597], [-0.0011046345960252903]]
K2=[[575.7828397557032, 0.0, 869.9354676964672], [0.0, 575.7822659178688, 1091.530528530801], [0.0, 0.0, 1.0]]
D2=[[0.019173635688566008], [-0.008775671298404274], [0.007914251615504821], [-0.0020588786543884545]]
R=[[0.9999959115798861, 0.0008342703617784742, 0.002735108128766923], [-0.000818260419370479, 0.9999825573445704, -0.005849397960390626], [-0.002739940400570883, 0.0058471360148698755, 0.9999791516461854]]
T=[[-2.7121875129251327], [-0.043495329411315974], [0.019486853759296718]]
@crackwitz does it mater for calibration if a picture (my 3rd post above) that comes straight from the camera doesn’t actually have an optical center instead it’s offset to the side?
that’s what the calibration is for. it calculates the optical center from pictures.
if you mess with the pictures (by cropping), you can either redo the calibration, or you know the model and how to calculate new model parameters from old ones and the knowledge of what you changed.