Location second camera seems not right cv2.stereoCalibrate()

I am debugging my stereo calibration code, however, the location of the second camera seems to be in the wrong location. I perform a dot product of translationmatrix ([[R, T], [0, 0, 0, 1]]) with the location vector of the first camera ([X1, Y1, Z1, 1]) as given in the stereoCalibrate function (OpenCV: Camera Calibration and 3D Reconstruction).


  • The RMSE for the calibration of both cameras is lower than 1 pixel.
  • The location of the camera is [x, y, z] = [376, -24, 101] in mm with the first camera at the center of the coordinate system; while the real location of the second camera is roughly 190 mm to the right of the first camera and on the same y and z location.

The rotation matrix is

array([[ 8.63219239e-01,  2.41569153e-02, -5.04250918e-01],
       [-1.47503745e-04,  9.98866480e-01,  4.75997246e-02],
       [ 5.04829202e-01, -4.10146192e-02,  8.62244326e-01]])

The translation vector is


Why is the location from the second camera so far of from where it should be?

it is a required criterion, not a sufficient criterion. on its own, it means nothing. low RMSE is easy to achieve with too little, or degenerate, calibration data.

please review the MRE article.

Thank you for your reply. So it means that a low RMSE value does not always mean a good calibration? For instance when I use a small amount of images, the RMSE could be low while the calibration is not “good”?

The resolution of the cameras is 3088x2064, could the low number of chessboard corner points be a reason for a bad calibration?

With the pictures and code below, I find the following location of the second camera [X2, Y2, Z2] = [389.0419239 159.77362514 110.93641357]. So how could this location be explained?

The pictures used can be found here:
Camera 1 calibration (left)
Camera 2 calibration (right)
Synced camera 1
Synced camera 2

Below you can find my code:

import cv2
import numpy as np
import glob

rows = 6 #number of checkerboard rows.
columns = 9 #number of checkerboard columns.
world_scaling = 24.5 #change this to the real world square size. Or not.
_show = True

def calibrate_camera(images_folder):
  images_names = sorted(glob.glob(images_folder))
  images = []
  for imname in images_names:
    im = cv2.imread(imname, 1)

  #criteria used by checkerboard pattern detector.
  #Change this if the code can't find the checkerboard
  criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)

  #coordinates of squares in the checkerboard world space
  objp = np.zeros((rows*columns,3), np.float32)
  objp[:,:2] = np.mgrid[0:rows,0:columns].T.reshape(-1,2)
  objp = world_scaling* objp

  #frame dimensions. Frames should be the same size.
  width = images[0].shape[1]
  height = images[0].shape[0]

  #Pixel coordinates of checkerboards
  imgpoints = [] # 2d points in image plane.

  #coordinates of the checkerboard in checkerboard world space.
  objpoints = [] # 3d point in real world space
  for frame in images:
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    #find the checkerboard
    ret, corners = cv2.findChessboardCorners(gray, (rows, columns), None)

    if ret == True:
      #Convolution size used to improve corner detection. Don't make this too large.
      conv_size = (11, 11)

      #opencv2 can attempt to improve the checkerboard coordinates
      corners = cv2.cornerSubPix(gray, corners, conv_size, (-1, -1), criteria)
      if _show:
        # draw chessboard corners on frame
        cv2.drawChessboardCorners(frame, (rows,columns), corners, ret)

        # resize frame to fit to screen
        res_frame = cv2.resize(frame, (1080,720))
        cv2.imshow('img', res_frame)
        k = cv2.waitKey(100)

      # append corner locations to imgpoints

  # perform camera calibration
  ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, (width, height), None, None)
  print(f'rmse single camera ({images_folder}): {ret}')

  return mtx, dist, rvecs, tvecs

def stereo_calibrate(mtx1, dist1, mtx2, dist2, frames_1, frames_2):
  #read the synched frames
  c1_images_names = glob.glob(frames_1)
  c2_images_names = glob.glob(frames_2)

  c1_images = []
  c2_images = []
  for im1, im2 in zip(c1_images_names, c2_images_names):
    _im = cv2.imread(im1, 1)

    _im = cv2.imread(im2, 1)

  # criteria for stereo calibration
  criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 0.0001)

  #coordinates of squares in the checkerboard world space
  objp = np.zeros((rows*columns,3), np.float32)
  objp[:,:2] = np.mgrid[0:rows,0:columns].T.reshape(-1,2)
  objp = world_scaling* objp

  #frame dimensions. Frames should be the same size.
  width = c1_images[0].shape[1]
  height = c1_images[0].shape[0]

  #Pixel coordinates of checkerboards
  imgpoints_left = [] # 2d points in image plane.
  imgpoints_right = []

  #coordinates of the checkerboard in checkerboard world space.
  objpoints = [] # 3d point in real world space
  count = 0
  for frame1, frame2 in zip(c1_images, c2_images):
    gray1 = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
    gray2 = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
    c_ret1, corners1 = cv2.findChessboardCorners(gray1, (rows, columns), None)
    c_ret2, corners2 = cv2.findChessboardCorners(gray2, (rows, columns), None)

    if c_ret1 == True and c_ret2 == True:
      corners1 = cv2.cornerSubPix(gray1, corners1, (11, 11), (-1, -1), criteria)
      corners2 = cv2.cornerSubPix(gray2, corners2, (11, 11), (-1, -1), criteria)

      if count == 0:
        corner_point = [corners1[0], corners2[1]]

      if _show:
        cv2.drawChessboardCorners(frame1, (rows, columns), corners1, c_ret1)
        res_frame = cv2.resize(frame1, (1080,720))
        cv2.imshow('img', res_frame)

        cv2.drawChessboardCorners(frame2, (rows, columns), corners2, c_ret2)
        res_frame = cv2.resize(frame2, (1080,720))
        cv2.imshow('img2', res_frame)
        k = cv2.waitKey(100)

      count += 1

  stereocalibration_flags = cv2.CALIB_FIX_INTRINSIC

  # stereo calibrate system
  ret, CM1, dist1, CM2, dist2, R, T, E, F = cv2.stereoCalibrate(objpoints, imgpoints_left, imgpoints_right, mtx1, dist1,
                                                                mtx2, dist2, (width, height), criteria = criteria, flags = stereocalibration_flags)

  print(f"rmse stereo: {ret}")

  return R, T, corner_point

if __name__ == "__main__":
  mtx1, dist1, rvecs1, tvecs1 = calibrate_camera(images_folder = 'images/stereoLeft/*')
  mtx2, dist2, rvecs2, tvecs2 = calibrate_camera(images_folder = 'images/stereoRight/*')

  R, T, corner_point = stereo_calibrate(mtx1, dist1, mtx2, dist2, 'images/synced/stereoLeft/*', 'images/synced/stereoRight/*')

  transformation_matrix = np.empty((4,4))
  transformation_matrix[:3, :3] = R
  transformation_matrix[:3, 3] = T.T[0]
  transformation_matrix[3, :] = [0, 0, 0, 1]

  location_cam2 = np.dot(transformation_matrix, [[0], [0], [0], [1]])

  print("location camera 2 [x,y,z]: ", location_cam2[:3].T)

I have found the solution to my problem.

In the function “stereo_calibrate” the images that are imported are not sorted. Replacing:

  c1_images_names = glob.glob(frames_1)
  c2_images_names = glob.glob(frames_2)


  c1_images_names = sorted(glob.glob(frames_1))
  c2_images_names = sorted(glob.glob(frames_2))

Fixed my problem. The norm of the translation vector is now at around 180mm which could be right for the distance between camera1 and camera2.

1 Like